File size: 6,194 Bytes
00db6c4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 | import json
import numpy as np
import matplotlib; matplotlib.use("Agg")
import matplotlib.pyplot as plt
J = {
"pref": json.load(open("/mnt/localssd/pref_results2.json")),
"iqa": json.load(open("/mnt/localssd/pref_results_iqa.json")),
"panel":json.load(open("/mnt/localssd/pref_results_panel.json")),
"aes": json.load(open("/mnt/localssd/pref_results_aes.json")),
"qal": json.load(open("/mnt/localssd/pref_results_qalign.json")),
}
# (display, src, kind, key, type, lower_better)
REG = [
("ImageReward","pref","models","ImageReward-v1.0","prompt-pref",False),
("PickScore","pref","models","PickScore_v1","prompt-pref",False),
("NIQE","panel","metrics","niqe","unsupervised-IQA",True),
("BRISQUE","panel","metrics","brisque","unsupervised-IQA",True),
("PI","panel","metrics","pi","unsupervised-IQA",True),
("IL-NIQE","panel","metrics","ilniqe","unsupervised-IQA",True),
("CLIP-IQA","iqa","metrics","clipiqa","learned-IQA",False),
("CLIP-IQA+","iqa","metrics","clipiqa+","learned-IQA",False),
("NIMA(AVA)","panel","metrics","nima","learned-IQA",False),
("PaQ-2-PiQ","panel","metrics","paq2piq","learned-IQA",False),
("HyperIQA","panel","metrics","hyperiqa","learned-IQA",False),
("DBCNN","panel","metrics","dbcnn","learned-IQA",False),
("TReS","panel","metrics","tres","learned-IQA",False),
("ARNIQA","panel","metrics","arniqa","learned-IQA",False),
("QualiCLIP","panel","metrics","qualiclip","learned-IQA",False),
("MUSIQ","panel","metrics","musiq","learned-IQA",False),
("MANIQA","panel","metrics","maniqa","learned-IQA",False),
("TOPIQ-NR","panel","metrics","topiq_nr","learned-IQA",False),
("Q-Align-qual","qal","metrics","q_align_quality","learned-IQA",False),
("LAION-aes","iqa","metrics","laion_aes","aesthetic",False),
("Aes-v2.5","aes","metrics","aes_v2.5","aesthetic",False),
("Q-Align-aes","qal","metrics","q_align_aesthetic","aesthetic",False),
]
def summ(src,kind,key,dom):
return J[src]["domains"][dom][kind][key]["summary"]
rows=[]
for disp,src,kind,key,typ,lb in REG:
ps=summ(src,kind,key,"pixel"); ls=summ(src,kind,key,"latent")
rows.append(dict(disp=disp,typ=typ,lb=lb,
pix_wr=ps["win_rate"],pix_p=ps["binom_p_two_sided"],
lat_wr=ls["win_rate"],lat_p=ls["binom_p_two_sided"]))
# ---- text table ----
def stars(p): return "***" if p<1e-3 else "**" if p<1e-2 else "*" if p<5e-2 else "ns"
print("="*104)
print(f"{'scorer':<14}{'type':<18}{'LB':<4}{'PIXEL win%':>11} {'p':>9} {'':>3} {'LATENT win%':>12} {'p':>9} {'':>3}")
print("-"*104)
for r in rows:
print(f"{r['disp']:<14}{r['typ']:<18}{('Y' if r['lb'] else ''):<4}"
f"{r['pix_wr']:>10.1f} {r['pix_p']:>9.1e} {stars(r['pix_p']):>3} "
f"{r['lat_wr']:>11.1f} {r['lat_p']:>9.1e} {stars(r['lat_p']):>3}")
print("="*104)
# ---- summary counts ----
def classify(wr,p):
if p>=0.05: return "tie"
return "gan_better" if wr>50 else "gan_worse"
for dom,wk,pk in [("PIXEL","pix_wr","pix_p"),("LATENT","lat_wr","lat_p")]:
c={"gan_better":0,"tie":0,"gan_worse":0}
for r in rows: c[classify(r[wk],r[pk])]+=1
print(f"{dom}: of {len(rows)} scorers -> GAN significantly BETTER={c['gan_better']}, "
f"TIE(ns)={c['tie']}, GAN significantly WORSE={c['gan_worse']}")
json.dump({"rows":rows}, open("/mnt/localssd/pref_master_table.json","w"), indent=2)
# ---- figure: horizontal grouped bars ----
TCOL={"prompt-pref":"#4a3aa7","unsupervised-IQA":"#008300","learned-IQA":"#256abf","aesthetic":"#eb6834"}
BLUE="#2a78d6"; ORANGE="#eb6834"; INK="#0b0b0b"; INK2="#52514e"; SURF="#fcfcfb"; GRID="#e6e5e2"
plt.rcParams.update({"figure.facecolor":SURF,"axes.facecolor":SURF,"text.color":INK,
"axes.labelcolor":INK,"axes.edgecolor":INK2,"xtick.color":INK2,"ytick.color":INK2,
"font.size":10,"axes.spines.top":False,"axes.spines.right":False})
# order rows grouped by type (already grouped in REG order); build y positions with gaps between groups
order=rows
ypos=[]; y=0; labels=[]; sep=[]; lasttyp=None
for r in order:
if lasttyp is not None and r['typ']!=lasttyp:
y-=0.7; sep.append(y+0.35)
ypos.append(y); labels.append(r['disp']+(" (LB)" if r['lb'] else "")); y-=1; lasttyp=r['typ']
ypos=np.array(ypos); h=0.38
fig,ax=plt.subplots(figsize=(11,12))
for r,yp in zip(order,ypos):
ax.barh(yp+h/1.6, r['pix_wr'], height=h, color=BLUE, zorder=3)
ax.barh(yp-h/1.6, r['lat_wr'], height=h, color=ORANGE, zorder=3)
ax.text(r['pix_wr']+0.8, yp+h/1.6, f"{r['pix_wr']:.0f}{'' if r['pix_p']<0.05 else ' ns'}",
va="center", ha="left", fontsize=8, color=INK,
fontweight="bold" if r['pix_p']<0.05 else "normal")
ax.text(r['lat_wr']+0.8, yp-h/1.6, f"{r['lat_wr']:.0f}{'' if r['lat_p']<0.05 else ' ns'}",
va="center", ha="left", fontsize=8, color=INK,
fontweight="bold" if r['lat_p']<0.05 else "normal")
ax.axvline(50, ls="--", lw=1.6, color=INK2, zorder=2)
ax.text(50,ypos.max()+1.2,"50% = tie",ha="center",va="bottom",color=INK2,fontsize=9)
ax.set_yticks(ypos); ax.set_yticklabels(labels, fontsize=9)
# type group labels on right
seen=set()
for r,yp in zip(order,ypos):
if r['typ'] not in seen:
seen.add(r['typ'])
ax.text(103, yp+0.5, r['typ'], color=TCOL[r['typ']], fontsize=10, fontweight="bold", va="top")
ax.set_xlim(0,108); ax.set_xlabel("GAN win-rate (% of 1065 matched pairs where GAN is better, polarity-correct)")
ax.set_ylim(ypos.min()-1, ypos.max()+2)
from matplotlib.patches import Patch
ax.legend(handles=[Patch(color=BLUE,label="PIXEL (DeCo baseline vs gan_dinov2_text@25k)"),
Patch(color=ORANGE,label="LATENT (SANA L1 vs L2+GAN)")],
frameon=False, loc="lower right", fontsize=9)
ax.set_title("GAN vs no-GAN: image-only quality/aesthetic + preference battery (22 scorer-heads, n=1065/domain)\n"
"bars right of 50% = GAN preferred; 'ns' = not significant (binomial p>=0.05); LB = lower-better metric (polarity-flipped)",
fontsize=11, fontweight="bold")
ax.xaxis.grid(True,color=GRID,zorder=0); ax.set_axisbelow(True)
fig.tight_layout()
fig.savefig("/mnt/localssd/pref_iqa_panel.png", dpi=150, facecolor=SURF, bbox_inches="tight")
print("saved /mnt/localssd/pref_iqa_panel.png")
|