viz/preference/make_master.py

import json
import numpy as np
import matplotlib; matplotlib.use("Agg")
import matplotlib.pyplot as plt

J = {
 "pref": json.load(open("/mnt/localssd/pref_results2.json")),
 "iqa":  json.load(open("/mnt/localssd/pref_results_iqa.json")),
 "panel":json.load(open("/mnt/localssd/pref_results_panel.json")),
 "aes":  json.load(open("/mnt/localssd/pref_results_aes.json")),
 "qal":  json.load(open("/mnt/localssd/pref_results_qalign.json")),
}
# (display, src, kind, key, type, lower_better)
REG = [
 ("ImageReward","pref","models","ImageReward-v1.0","prompt-pref",False),
 ("PickScore","pref","models","PickScore_v1","prompt-pref",False),
 ("NIQE","panel","metrics","niqe","unsupervised-IQA",True),
 ("BRISQUE","panel","metrics","brisque","unsupervised-IQA",True),
 ("PI","panel","metrics","pi","unsupervised-IQA",True),
 ("IL-NIQE","panel","metrics","ilniqe","unsupervised-IQA",True),
 ("CLIP-IQA","iqa","metrics","clipiqa","learned-IQA",False),
 ("CLIP-IQA+","iqa","metrics","clipiqa+","learned-IQA",False),
 ("NIMA(AVA)","panel","metrics","nima","learned-IQA",False),
 ("PaQ-2-PiQ","panel","metrics","paq2piq","learned-IQA",False),
 ("HyperIQA","panel","metrics","hyperiqa","learned-IQA",False),
 ("DBCNN","panel","metrics","dbcnn","learned-IQA",False),
 ("TReS","panel","metrics","tres","learned-IQA",False),
 ("ARNIQA","panel","metrics","arniqa","learned-IQA",False),
 ("QualiCLIP","panel","metrics","qualiclip","learned-IQA",False),
 ("MUSIQ","panel","metrics","musiq","learned-IQA",False),
 ("MANIQA","panel","metrics","maniqa","learned-IQA",False),
 ("TOPIQ-NR","panel","metrics","topiq_nr","learned-IQA",False),
 ("Q-Align-qual","qal","metrics","q_align_quality","learned-IQA",False),
 ("LAION-aes","iqa","metrics","laion_aes","aesthetic",False),
 ("Aes-v2.5","aes","metrics","aes_v2.5","aesthetic",False),
 ("Q-Align-aes","qal","metrics","q_align_aesthetic","aesthetic",False),
]
def summ(src,kind,key,dom):
    return J[src]["domains"][dom][kind][key]["summary"]

rows=[]
for disp,src,kind,key,typ,lb in REG:
    ps=summ(src,kind,key,"pixel"); ls=summ(src,kind,key,"latent")
    rows.append(dict(disp=disp,typ=typ,lb=lb,
        pix_wr=ps["win_rate"],pix_p=ps["binom_p_two_sided"],
        lat_wr=ls["win_rate"],lat_p=ls["binom_p_two_sided"]))

# ---- text table ----
def stars(p): return "***" if p<1e-3 else "**" if p<1e-2 else "*" if p<5e-2 else "ns"
print("="*104)
print(f"{'scorer':<14}{'type':<18}{'LB':<4}{'PIXEL win%':>11} {'p':>9} {'':>3}   {'LATENT win%':>12} {'p':>9} {'':>3}")
print("-"*104)
for r in rows:
    print(f"{r['disp']:<14}{r['typ']:<18}{('Y' if r['lb'] else ''):<4}"
          f"{r['pix_wr']:>10.1f} {r['pix_p']:>9.1e} {stars(r['pix_p']):>3}   "
          f"{r['lat_wr']:>11.1f} {r['lat_p']:>9.1e} {stars(r['lat_p']):>3}")
print("="*104)

# ---- summary counts ----
def classify(wr,p):
    if p>=0.05: return "tie"
    return "gan_better" if wr>50 else "gan_worse"
for dom,wk,pk in [("PIXEL","pix_wr","pix_p"),("LATENT","lat_wr","lat_p")]:
    c={"gan_better":0,"tie":0,"gan_worse":0}
    for r in rows: c[classify(r[wk],r[pk])]+=1
    print(f"{dom}: of {len(rows)} scorers -> GAN significantly BETTER={c['gan_better']}, "
          f"TIE(ns)={c['tie']}, GAN significantly WORSE={c['gan_worse']}")

json.dump({"rows":rows}, open("/mnt/localssd/pref_master_table.json","w"), indent=2)

# ---- figure: horizontal grouped bars ----
TCOL={"prompt-pref":"#4a3aa7","unsupervised-IQA":"#008300","learned-IQA":"#256abf","aesthetic":"#eb6834"}
BLUE="#2a78d6"; ORANGE="#eb6834"; INK="#0b0b0b"; INK2="#52514e"; SURF="#fcfcfb"; GRID="#e6e5e2"
plt.rcParams.update({"figure.facecolor":SURF,"axes.facecolor":SURF,"text.color":INK,
  "axes.labelcolor":INK,"axes.edgecolor":INK2,"xtick.color":INK2,"ytick.color":INK2,
  "font.size":10,"axes.spines.top":False,"axes.spines.right":False})
# order rows grouped by type (already grouped in REG order); build y positions with gaps between groups
order=rows
ypos=[]; y=0; labels=[]; sep=[]; lasttyp=None
for r in order:
    if lasttyp is not None and r['typ']!=lasttyp:
        y-=0.7; sep.append(y+0.35)
    ypos.append(y); labels.append(r['disp']+("  (LB)" if r['lb'] else "")); y-=1; lasttyp=r['typ']
ypos=np.array(ypos); h=0.38
fig,ax=plt.subplots(figsize=(11,12))
for r,yp in zip(order,ypos):
    ax.barh(yp+h/1.6, r['pix_wr'], height=h, color=BLUE, zorder=3)
    ax.barh(yp-h/1.6, r['lat_wr'], height=h, color=ORANGE, zorder=3)
    ax.text(r['pix_wr']+0.8, yp+h/1.6, f"{r['pix_wr']:.0f}{'' if r['pix_p']<0.05 else ' ns'}",
            va="center", ha="left", fontsize=8, color=INK,
            fontweight="bold" if r['pix_p']<0.05 else "normal")
    ax.text(r['lat_wr']+0.8, yp-h/1.6, f"{r['lat_wr']:.0f}{'' if r['lat_p']<0.05 else ' ns'}",
            va="center", ha="left", fontsize=8, color=INK,
            fontweight="bold" if r['lat_p']<0.05 else "normal")
ax.axvline(50, ls="--", lw=1.6, color=INK2, zorder=2)
ax.text(50,ypos.max()+1.2,"50% = tie",ha="center",va="bottom",color=INK2,fontsize=9)
ax.set_yticks(ypos); ax.set_yticklabels(labels, fontsize=9)
# type group labels on right
seen=set()
for r,yp in zip(order,ypos):
    if r['typ'] not in seen:
        seen.add(r['typ'])
        ax.text(103, yp+0.5, r['typ'], color=TCOL[r['typ']], fontsize=10, fontweight="bold", va="top")
ax.set_xlim(0,108); ax.set_xlabel("GAN win-rate  (% of 1065 matched pairs where GAN is better, polarity-correct)")
ax.set_ylim(ypos.min()-1, ypos.max()+2)
from matplotlib.patches import Patch
ax.legend(handles=[Patch(color=BLUE,label="PIXEL (DeCo baseline vs gan_dinov2_text@25k)"),
                   Patch(color=ORANGE,label="LATENT (SANA L1 vs L2+GAN)")],
          frameon=False, loc="lower right", fontsize=9)
ax.set_title("GAN vs no-GAN: image-only quality/aesthetic + preference battery (22 scorer-heads, n=1065/domain)\n"
             "bars right of 50% = GAN preferred; 'ns' = not significant (binomial p>=0.05); LB = lower-better metric (polarity-flipped)",
             fontsize=11, fontweight="bold")
ax.xaxis.grid(True,color=GRID,zorder=0); ax.set_axisbelow(True)
fig.tight_layout()
fig.savefig("/mnt/localssd/pref_iqa_panel.png", dpi=150, facecolor=SURF, bbox_inches="tight")
print("saved /mnt/localssd/pref_iqa_panel.png")