forgebench/code/eval/appeval/appearance.py

"""Appearance metrics on object-masked renders (LOCKED spec).

Rankings (per PI decision): LPIPS primary, SSIM secondary, CLIP tertiary.
PSNR reported but NON-RANKING. NO FID/KID, NO PSNR ranking.

  * LPIPS(net='alex')  : lpips pkg, inputs in [-1,1].
  * SSIM               : skimage structural_similarity, win_size=11,
                         gaussian_weights=True, data_range=1, channel_axis=-1.
  * CLIP-similarity    : cosine of open_clip ViT-B-32 image embeddings.
  * PSNR               : torchmetrics PeakSignalNoiseRatio data_range=1.

Masking: every metric is computed on the object composited over a fixed
WHITE background using the render alpha (alpha>0 => object). Pred and GT use
the identical background, so silhouette disagreement is penalised (it shows up
as object-vs-background) while background pixels never bias the score.
Metrics are computed per view, then averaged over views, then over objects
(the averaging over objects happens in the CLI).
"""

from __future__ import annotations

import numpy as np
import torch

from skimage.metrics import structural_similarity as _ssim
from torchmetrics.image import PeakSignalNoiseRatio

_LPIPS = None
_CLIP = None
_CLIP_PRE = None
_PSNR = None
_DEVICE = "cuda"


def _lazy():
    global _LPIPS, _CLIP, _CLIP_PRE, _PSNR
    if _LPIPS is None:
        import lpips
        _LPIPS = lpips.LPIPS(net="alex").to(_DEVICE).eval()
    if _CLIP is None:
        import open_clip
        import os
        cache = os.environ.get("HF_HOME", None)
        model, _, pre = open_clip.create_model_and_transforms(
            "ViT-B-32", pretrained="openai", cache_dir=cache)
        _CLIP = model.to(_DEVICE).eval()
        _CLIP_PRE = pre
    if _PSNR is None:
        _PSNR = PeakSignalNoiseRatio(data_range=1.0).to(_DEVICE)


def composite_white(rgba: np.ndarray) -> np.ndarray:
    """Composite an (H,W,4) float[0,1] RGBA over white -> (H,W,3) float[0,1]."""
    rgb = rgba[..., :3]
    a = rgba[..., 3:4]
    return rgb * a + (1.0 - a)


def _prep_tensor(rgb):
    """(H,W,3) float[0,1] -> (1,3,H,W) tensor on device."""
    t = torch.as_tensor(rgb, dtype=torch.float32, device=_DEVICE)
    return t.permute(2, 0, 1)[None].contiguous()


@torch.no_grad()
def appearance_metrics(pred_rgba: np.ndarray, ref_rgba: np.ndarray):
    """Compare two RGBA renders. Returns dict lpips/ssim/clip/psnr for one view.

    Both inputs are (H,W,4) float in [0,1]. Composited over white internally.
    """
    _lazy()
    pred = composite_white(np.asarray(pred_rgba, np.float32))
    ref = composite_white(np.asarray(ref_rgba, np.float32))

    pt = _prep_tensor(pred)
    rt = _prep_tensor(ref)

    # LPIPS wants [-1,1]
    lp = float(_LPIPS(pt * 2 - 1, rt * 2 - 1).item())

    # SSIM (skimage, per channel gaussian)
    ss = float(_ssim(pred, ref, win_size=11, gaussian_weights=True,
                     data_range=1.0, channel_axis=-1))

    # PSNR (non-ranking)
    ps = float(_PSNR(pt, rt).item())

    # CLIP cosine
    from PIL import Image
    pi = _CLIP_PRE(Image.fromarray((pred * 255).astype(np.uint8))).unsqueeze(0).to(_DEVICE)
    ri = _CLIP_PRE(Image.fromarray((ref * 255).astype(np.uint8))).unsqueeze(0).to(_DEVICE)
    fe = _CLIP.encode_image(pi)
    ge = _CLIP.encode_image(ri)
    fe = fe / fe.norm(dim=-1, keepdim=True)
    ge = ge / ge.norm(dim=-1, keepdim=True)
    clip = float((fe * ge).sum(-1).item())

    return {"lpips": lp, "ssim": ss, "clip": clip, "psnr": ps}


def average_views(view_dicts):
    """Mean each metric across a list of per-view dicts."""
    if not view_dicts:
        return {}
    keys = view_dicts[0].keys()
    return {k: float(np.mean([d[k] for d in view_dicts])) for k in keys}