Upload viz/preference/pref_score.py with huggingface_hub

viz/preference/pref_score.py

@@ -0,0 +1,75 @@
+import os, glob, json, sys
+import ImageReward
+
+DPG = "/mnt/localssd/ELLA_dpg/dpg_bench/prompts"
+
+def prompt_for(idx):
+    p = os.path.join(DPG, f"{idx}.txt")
+    with open(p) as f:
+        return f.read().strip()
+
+def run_domain(name, nogan_dir, gan_dir, model):
+    # indices present in BOTH dirs as {idx}_0.png and having a DPG prompt
+    def idxs(d):
+        out = set()
+        for f in glob.glob(os.path.join(d, "*_0.png")):
+            stem = os.path.basename(f)[:-len("_0.png")]
+            out.add(stem)
+        return out
+    common = sorted(idxs(nogan_dir) & idxs(gan_dir),
+                    key=lambda x: (0, int(x)) if x.isdigit() else (1, x))
+    rows = []
+    for idx in common:
+        if not os.path.exists(os.path.join(DPG, f"{idx}.txt")):
+            continue
+        prompt = prompt_for(idx)
+        ng_img = os.path.join(nogan_dir, f"{idx}_0.png")
+        g_img = os.path.join(gan_dir, f"{idx}_0.png")
+        s_ng = float(model.score(prompt, ng_img))
+        s_g = float(model.score(prompt, g_img))
+        rows.append({"idx": idx, "nogan": s_ng, "gan": s_g, "delta": s_g - s_ng})
+    return rows
+
+def summarize(name, rows):
+    n = len(rows)
+    wins = sum(1 for r in rows if r["gan"] > r["nogan"])
+    ties = sum(1 for r in rows if r["gan"] == r["nogan"])
+    losses = n - wins - ties
+    mean_ng = sum(r["nogan"] for r in rows) / n
+    mean_g = sum(r["gan"] for r in rows) / n
+    mean_d = sum(r["delta"] for r in rows) / n
+    return {
+        "domain": name, "n": n, "wins": wins, "ties": ties, "losses": losses,
+        "win_rate": 100.0 * wins / n,
+        "win_rate_excl_ties": 100.0 * wins / (wins + losses) if (wins + losses) else None,
+        "mean_nogan": mean_ng, "mean_gan": mean_g, "mean_delta": mean_d,
+    }
+
+def main():
+    model = ImageReward.load("ImageReward-v1.0")
+    results = {"model": "ImageReward-v1.0", "domains": {}}
+
+    pixel_rows = run_domain(
+        "pixel",
+        "/mnt/localssd/eval_work/spectrum/gen/baseline",
+        "/mnt/localssd/eval_work/spectrum/gen/gan_dinov2_t035",
+        model)
+    results["domains"]["pixel"] = {"summary": summarize("pixel", pixel_rows),
+                                    "rows": pixel_rows}
+    print("PIXEL done:", json.dumps(results["domains"]["pixel"]["summary"]), flush=True)
+
+    latent_rows = run_domain(
+        "latent",
+        "/mnt/localssd/eval_work_sana/sana_blip3o_sft/imgs_50000",
+        "/mnt/localssd/eval_work_sana/sana_blip3o_gan/imgs_50000",
+        model)
+    results["domains"]["latent"] = {"summary": summarize("latent", latent_rows),
+                                     "rows": latent_rows}
+    print("LATENT done:", json.dumps(results["domains"]["latent"]["summary"]), flush=True)
+
+    with open("/mnt/localssd/pref_results.json", "w") as f:
+        json.dump(results, f, indent=2)
+    print("WROTE /mnt/localssd/pref_results.json", flush=True)
+
+if __name__ == "__main__":
+    main()