forgebench/code/eval/appeval/selfcheck.py

"""Self-test for the appearance+geometry harness. MUST pass before use.

Checks:
  (1) ORACLE     : GT mesh vs itself -> CD~0, F@all~1.0, VolIoU~1.0,
                   LPIPS~0, SSIM~1, CLIP~1, PSNR very high.
  (2) RENDER     : render a VERTEX-COLORED glb (primary path for our methods,
                   and toys4k GT) from the input view; confirm alpha mask is
                   sane, mask IoU vs stored depth mask is high, and rendered
                   colours are NOT flat grey (real colour variance). Also
                   render a UV-textured glb if available to exercise that path.
                   Proof sheets saved to .debug/appeval/.
  (3) MONOTONIC  : a noise-perturbed mesh scores WORSE than the oracle on
                   EVERY metric (CD up, F down, IoU down, NC down; LPIPS up,
                   SSIM down, CLIP down, PSNR down).

Runs on 2 real toys4k objects using their GT canon.glb as the "prediction".
"""

from __future__ import annotations

import os
import sys
from pathlib import Path

import numpy as np
import trimesh
from PIL import Image

sys.path.insert(0, str(Path(__file__).resolve().parent))

import render as R
from geometry import geometry_metrics
from appearance import appearance_metrics, composite_white

EXP = Path("/lp-dev/jonghoon/mv-mesh/exp_faithfulness/toys4k")
DEBUG = Path("/home/nvidia/jonghoon/mv-mesh/.debug/appeval")
OBJECTS = ["apple_002", "car_006"]
UV_GLB = EXP / "gen_2v" / "trellis" / "apple_002.glb"  # optional UV path


def perturb(mesh: trimesh.Trimesh, sigma=0.03, seed=0):
    m = mesh.copy()
    rng = np.random.default_rng(seed)
    m.vertices = m.vertices + rng.normal(0, sigma, size=m.vertices.shape)
    return m


def mask_iou(a, b):
    a = a.astype(bool); b = b.astype(bool)
    return float((a & b).sum() / max((a | b).sum(), 1))


def check_object(obj):
    print(f"\n=== object: {obj} ===")
    gt_path = EXP / "renders" / f"{obj}_canon.glb"
    gt = trimesh.load(str(gt_path), force="mesh", process=False)
    gt_gl = R.prepare_mesh(gt)
    assert gt_gl.mode == "vertex", f"GT should be vertex-coloured, got {gt_gl.mode}"

    # ---- (2) render vertex-coloured mesh from input view ----
    z = np.load(EXP / "renders" / f"{obj}_front.npz")
    K = {k: float(z[k]) for k in ("fx", "fy", "cx", "cy")}
    res = int(z["res"])
    c2w = z["c2w_cv"]
    depth_mask = z["depth_mm"] > 0
    rgba = R.render_input_view(gt_gl, K, c2w, res, res).cpu().numpy()
    alpha = rgba[..., 3] > 0.5
    iou = mask_iou(alpha, depth_mask)
    obj_rgb = rgba[..., :3][alpha]
    col_std = float(obj_rgb.std(0).mean())
    col_mean = obj_rgb.mean(0)
    print(f"  [render] mask IoU vs depth = {iou:.4f} | "
          f"colour mean={np.round(col_mean,3).tolist()} std={col_std:.4f}")
    assert iou > 0.9, f"input-view mask IoU too low ({iou})"
    assert alpha.sum() > 1000, "empty render"
    assert col_std > 0.02, f"render looks flat grey (col_std={col_std}); " \
        "vertex colours not applied!"
    assert not np.allclose(col_mean, [0.6, 0.6, 0.6], atol=0.02), \
        "render is the flat-grey fallback colour"
    DEBUG.mkdir(parents=True, exist_ok=True)
    Image.fromarray((composite_white(rgba) * 255).astype(np.uint8)).save(
        DEBUG / f"selfcheck_{obj}_vertexcolor.png")

    # a couple orbit views for the proof sheet
    cams = R.orbit_cameras()
    orb = R.render_orbit(gt_gl, cams).cpu().numpy()
    sheet = np.concatenate([composite_white(orb[i]) for i in (0, 4, 8, 12, 16, 20)], axis=1)
    Image.fromarray((sheet * 255).astype(np.uint8)).save(
        DEBUG / f"selfcheck_{obj}_orbit.png")

    # ---- (1) ORACLE geometry ----
    go = geometry_metrics(gt, gt)
    print(f"  [oracle-geom] CD_L1={go['cd_l1']:.5f} CD_L2={go['cd_l2']:.6f} "
          f"F@.01={go['f01']:.4f} F@.02={go['f02']:.4f} F@.05={go['f05']:.4f} "
          f"NC={go['normal_consistency']:.4f} VolIoU={go['vol_iou']:.4f}")
    assert go["cd_l1"] < 5e-3, go["cd_l1"]
    assert go["f01"] > 0.99 and go["f02"] > 0.99 and go["f05"] > 0.99
    assert go["normal_consistency"] > 0.99
    assert go["vol_iou"] > 0.99

    # ---- (1) ORACLE appearance (identical renders) ----
    ao = appearance_metrics(rgba, rgba)
    print(f"  [oracle-app ] LPIPS={ao['lpips']:.5f} SSIM={ao['ssim']:.4f} "
          f"CLIP={ao['clip']:.4f} PSNR={ao['psnr']:.2f}")
    assert ao["lpips"] < 1e-4
    assert ao["ssim"] > 0.999
    assert ao["clip"] > 0.999

    # ---- (3) MONOTONICITY: perturbed mesh worse on every metric ----
    pert = perturb(gt, sigma=0.03)
    pert_gl = R.prepare_mesh(pert)
    gp = geometry_metrics(pert, gt)
    pert_rgba = R.render_input_view(pert_gl, K, c2w, res, res).cpu().numpy()
    ap = appearance_metrics(pert_rgba, rgba)
    print(f"  [perturbed  ] CD_L1={gp['cd_l1']:.5f} F@.02={gp['f02']:.4f} "
          f"VolIoU={gp['vol_iou']:.4f} NC={gp['normal_consistency']:.4f} | "
          f"LPIPS={ap['lpips']:.4f} SSIM={ap['ssim']:.4f} CLIP={ap['clip']:.4f} "
          f"PSNR={ap['psnr']:.2f}")
    checks = {
        "CD_L1 up":  gp["cd_l1"] > go["cd_l1"],
        "CD_L2 up":  gp["cd_l2"] > go["cd_l2"],
        "F@.01 down": gp["f01"] < go["f01"],
        "F@.02 down": gp["f02"] < go["f02"],
        "F@.05 down": gp["f05"] < go["f05"],
        "NC down":   gp["normal_consistency"] < go["normal_consistency"],
        "VolIoU down": gp["vol_iou"] < go["vol_iou"],
        "LPIPS up":  ap["lpips"] > ao["lpips"],
        "SSIM down": ap["ssim"] < ao["ssim"],
        "CLIP down": ap["clip"] < ao["clip"],
        "PSNR down": ap["psnr"] < ao["psnr"],
    }
    for name, ok in checks.items():
        assert ok, f"MONOTONICITY FAIL: {name}"
    print(f"  [monotonic  ] all {len(checks)} metrics degrade correctly: PASS")
    return go, ao


def check_uv_path():
    if not UV_GLB.exists():
        print("\n[uv-path] no UV-textured glb available, skipping")
        return
    m = trimesh.load(str(UV_GLB), force="mesh", process=False)
    g = R.prepare_mesh(m)
    print(f"\n[uv-path] {UV_GLB.name} -> mode={g.mode}")
    cams = R.orbit_cameras()
    r = R.render_orbit(g, cams[:1]).cpu().numpy()[0]
    obj = r[..., 3] > 0.5
    assert obj.sum() > 500, "UV render empty"
    col_std = float(r[..., :3][obj].std(0).mean())
    print(f"  UV render obj px={int(obj.sum())} colour std={col_std:.4f} "
          f"(mode={g.mode})")
    Image.fromarray((composite_white(r) * 255).astype(np.uint8)).save(
        DEBUG / "selfcheck_uvpath.png")


def main():
    print("APPEVAL SELF-CHECK")
    for obj in OBJECTS:
        check_object(obj)
    check_uv_path()
    print("\n==================================================")
    print("ALL SELF-CHECKS PASSED")
    print("proof sheets ->", DEBUG)
    print("==================================================")


if __name__ == "__main__":
    main()