Add Unified-Bench old protocol bundle: unified_bench_old_protocol_0p8346/scripts/reproduce_apr4_unified_bagel_score.py

unified_bench_old_protocol_0p8346/scripts/reproduce_apr4_unified_bagel_score.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import modal
+
+from config import DATASET_CACHE_PATH, MODEL_CACHE_PATH, OUTPUT_APR4_PATH
+from images import get_image
+from volumes import dataset_volume, model_volume, output_apr4_volume
+
+
+RUN_NAME = "bagel"
+REF_DIR = f"{DATASET_CACHE_PATH}/unified_bench/Image"
+GEN_DIR = f"{OUTPUT_APR4_PATH}/unified_bench/{RUN_NAME}/images"
+
+image = (
+    get_image("unified_bench")
+    .add_local_dir("eval", remote_path="/workspace/eval")
+    .add_local_dir("src", remote_path="/workspace/src")
+)
+app = modal.App("umm-apr4-bagel-unified-reproduce")
+
+
+def _image_files(path: str) -> list[str]:
+    exts = (".png", ".jpg", ".jpeg", ".webp", ".bmp")
+    return sorted(
+        [str(p) for p in Path(path).iterdir() if p.is_file() and p.suffix.lower() in exts],
+        key=lambda s: int(Path(s).stem) if Path(s).stem.isdigit() else Path(s).stem,
+    )
+
+
+@app.function(
+    image=image,
+    volumes={
+        MODEL_CACHE_PATH: model_volume,
+        DATASET_CACHE_PATH: dataset_volume,
+        OUTPUT_APR4_PATH: output_apr4_volume,
+    },
+    gpu="A100-40GB",
+    timeout=7200,
+)
+def reproduce() -> dict[str, Any]:
+    import sys
+    import torch
+
+    sys.path.insert(0, "/workspace/eval/generation/unified_bench")
+    from scorers import load_scorer
+
+    refs = _image_files(REF_DIR)
+    gens = _image_files(GEN_DIR)
+    if len(refs) != len(gens):
+        raise RuntimeError(f"ref/gen count mismatch: refs={len(refs)} gens={len(gens)}")
+
+    model_paths = {
+        "clip": f"{MODEL_CACHE_PATH}/unified_bench/clip-vit-large-patch14",
+        "dinov2": f"{MODEL_CACHE_PATH}/unified_bench/dinov2-large",
+        "dinov3": f"{MODEL_CACHE_PATH}/unified_bench/dinov3-vitl16",
+        "longclip": f"{MODEL_CACHE_PATH}/unified_bench/longclip-L.pt",
+    }
+    result: dict[str, Any] = {
+        "run_name": RUN_NAME,
+        "ref_dir": REF_DIR,
+        "gen_dir": GEN_DIR,
+        "count": len(refs),
+        "model_paths": model_paths,
+        "averages_by_model": {},
+        "statistics_by_model": {},
+        "sanity": {},
+    }
+
+    for name in ("clip", "dinov2", "dinov3", "longclip"):
+        scorer = load_scorer(name, model_path=model_paths[name], device="cuda")
+        values = []
+        per_image = {}
+        for ref, gen in zip(refs, gens):
+            score = float(scorer.score_pair(ref, gen))
+            values.append(score)
+            per_image[Path(gen).name] = score
+        result["averages_by_model"][name] = sum(values) / len(values)
+        result["statistics_by_model"][name] = {
+            "total_images": len(values),
+            "valid_pairs": len(values),
+            "average_similarity": sum(values) / len(values),
+            "min_similarity": min(values),
+            "max_similarity": max(values),
+        }
+        result.setdefault("similarities_by_model", {})[name] = per_image
+
+        if name == "dinov3":
+            sample_n = min(12, len(refs))
+            ref_shift = [
+                float(scorer.score_pair(refs[i], refs[(i + 1) % sample_n]))
+                for i in range(sample_n)
+            ]
+            result["sanity"]["dinov3_ref_shift_12_avg"] = sum(ref_shift) / len(ref_shift)
+            result["sanity"]["dinov3_ref_shift_12_first"] = ref_shift[:5]
+
+        del scorer
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+
+    avgs = result["averages_by_model"]
+    result["overall_average"] = sum(avgs.values()) / len(avgs)
+    result["summary"] = {
+        "overall_average": result["overall_average"],
+        "averages_by_model": avgs,
+    }
+    print(json.dumps(result, indent=2, ensure_ascii=False), flush=True)
+    return result
+
+
+@app.local_entrypoint()
+def main() -> None:
+    print(json.dumps(reproduce.remote(), indent=2, ensure_ascii=False))