File size: 6,777 Bytes
09df272 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 | """Self-test for the appearance+geometry harness. MUST pass before use.
Checks:
(1) ORACLE : GT mesh vs itself -> CD~0, F@all~1.0, VolIoU~1.0,
LPIPS~0, SSIM~1, CLIP~1, PSNR very high.
(2) RENDER : render a VERTEX-COLORED glb (primary path for our methods,
and toys4k GT) from the input view; confirm alpha mask is
sane, mask IoU vs stored depth mask is high, and rendered
colours are NOT flat grey (real colour variance). Also
render a UV-textured glb if available to exercise that path.
Proof sheets saved to .debug/appeval/.
(3) MONOTONIC : a noise-perturbed mesh scores WORSE than the oracle on
EVERY metric (CD up, F down, IoU down, NC down; LPIPS up,
SSIM down, CLIP down, PSNR down).
Runs on 2 real toys4k objects using their GT canon.glb as the "prediction".
"""
from __future__ import annotations
import os
import sys
from pathlib import Path
import numpy as np
import trimesh
from PIL import Image
sys.path.insert(0, str(Path(__file__).resolve().parent))
import render as R
from geometry import geometry_metrics
from appearance import appearance_metrics, composite_white
EXP = Path("/lp-dev/jonghoon/mv-mesh/exp_faithfulness/toys4k")
DEBUG = Path("/home/nvidia/jonghoon/mv-mesh/.debug/appeval")
OBJECTS = ["apple_002", "car_006"]
UV_GLB = EXP / "gen_2v" / "trellis" / "apple_002.glb" # optional UV path
def perturb(mesh: trimesh.Trimesh, sigma=0.03, seed=0):
m = mesh.copy()
rng = np.random.default_rng(seed)
m.vertices = m.vertices + rng.normal(0, sigma, size=m.vertices.shape)
return m
def mask_iou(a, b):
a = a.astype(bool); b = b.astype(bool)
return float((a & b).sum() / max((a | b).sum(), 1))
def check_object(obj):
print(f"\n=== object: {obj} ===")
gt_path = EXP / "renders" / f"{obj}_canon.glb"
gt = trimesh.load(str(gt_path), force="mesh", process=False)
gt_gl = R.prepare_mesh(gt)
assert gt_gl.mode == "vertex", f"GT should be vertex-coloured, got {gt_gl.mode}"
# ---- (2) render vertex-coloured mesh from input view ----
z = np.load(EXP / "renders" / f"{obj}_front.npz")
K = {k: float(z[k]) for k in ("fx", "fy", "cx", "cy")}
res = int(z["res"])
c2w = z["c2w_cv"]
depth_mask = z["depth_mm"] > 0
rgba = R.render_input_view(gt_gl, K, c2w, res, res).cpu().numpy()
alpha = rgba[..., 3] > 0.5
iou = mask_iou(alpha, depth_mask)
obj_rgb = rgba[..., :3][alpha]
col_std = float(obj_rgb.std(0).mean())
col_mean = obj_rgb.mean(0)
print(f" [render] mask IoU vs depth = {iou:.4f} | "
f"colour mean={np.round(col_mean,3).tolist()} std={col_std:.4f}")
assert iou > 0.9, f"input-view mask IoU too low ({iou})"
assert alpha.sum() > 1000, "empty render"
assert col_std > 0.02, f"render looks flat grey (col_std={col_std}); " \
"vertex colours not applied!"
assert not np.allclose(col_mean, [0.6, 0.6, 0.6], atol=0.02), \
"render is the flat-grey fallback colour"
DEBUG.mkdir(parents=True, exist_ok=True)
Image.fromarray((composite_white(rgba) * 255).astype(np.uint8)).save(
DEBUG / f"selfcheck_{obj}_vertexcolor.png")
# a couple orbit views for the proof sheet
cams = R.orbit_cameras()
orb = R.render_orbit(gt_gl, cams).cpu().numpy()
sheet = np.concatenate([composite_white(orb[i]) for i in (0, 4, 8, 12, 16, 20)], axis=1)
Image.fromarray((sheet * 255).astype(np.uint8)).save(
DEBUG / f"selfcheck_{obj}_orbit.png")
# ---- (1) ORACLE geometry ----
go = geometry_metrics(gt, gt)
print(f" [oracle-geom] CD_L1={go['cd_l1']:.5f} CD_L2={go['cd_l2']:.6f} "
f"F@.01={go['f01']:.4f} F@.02={go['f02']:.4f} F@.05={go['f05']:.4f} "
f"NC={go['normal_consistency']:.4f} VolIoU={go['vol_iou']:.4f}")
assert go["cd_l1"] < 5e-3, go["cd_l1"]
assert go["f01"] > 0.99 and go["f02"] > 0.99 and go["f05"] > 0.99
assert go["normal_consistency"] > 0.99
assert go["vol_iou"] > 0.99
# ---- (1) ORACLE appearance (identical renders) ----
ao = appearance_metrics(rgba, rgba)
print(f" [oracle-app ] LPIPS={ao['lpips']:.5f} SSIM={ao['ssim']:.4f} "
f"CLIP={ao['clip']:.4f} PSNR={ao['psnr']:.2f}")
assert ao["lpips"] < 1e-4
assert ao["ssim"] > 0.999
assert ao["clip"] > 0.999
# ---- (3) MONOTONICITY: perturbed mesh worse on every metric ----
pert = perturb(gt, sigma=0.03)
pert_gl = R.prepare_mesh(pert)
gp = geometry_metrics(pert, gt)
pert_rgba = R.render_input_view(pert_gl, K, c2w, res, res).cpu().numpy()
ap = appearance_metrics(pert_rgba, rgba)
print(f" [perturbed ] CD_L1={gp['cd_l1']:.5f} F@.02={gp['f02']:.4f} "
f"VolIoU={gp['vol_iou']:.4f} NC={gp['normal_consistency']:.4f} | "
f"LPIPS={ap['lpips']:.4f} SSIM={ap['ssim']:.4f} CLIP={ap['clip']:.4f} "
f"PSNR={ap['psnr']:.2f}")
checks = {
"CD_L1 up": gp["cd_l1"] > go["cd_l1"],
"CD_L2 up": gp["cd_l2"] > go["cd_l2"],
"F@.01 down": gp["f01"] < go["f01"],
"F@.02 down": gp["f02"] < go["f02"],
"F@.05 down": gp["f05"] < go["f05"],
"NC down": gp["normal_consistency"] < go["normal_consistency"],
"VolIoU down": gp["vol_iou"] < go["vol_iou"],
"LPIPS up": ap["lpips"] > ao["lpips"],
"SSIM down": ap["ssim"] < ao["ssim"],
"CLIP down": ap["clip"] < ao["clip"],
"PSNR down": ap["psnr"] < ao["psnr"],
}
for name, ok in checks.items():
assert ok, f"MONOTONICITY FAIL: {name}"
print(f" [monotonic ] all {len(checks)} metrics degrade correctly: PASS")
return go, ao
def check_uv_path():
if not UV_GLB.exists():
print("\n[uv-path] no UV-textured glb available, skipping")
return
m = trimesh.load(str(UV_GLB), force="mesh", process=False)
g = R.prepare_mesh(m)
print(f"\n[uv-path] {UV_GLB.name} -> mode={g.mode}")
cams = R.orbit_cameras()
r = R.render_orbit(g, cams[:1]).cpu().numpy()[0]
obj = r[..., 3] > 0.5
assert obj.sum() > 500, "UV render empty"
col_std = float(r[..., :3][obj].std(0).mean())
print(f" UV render obj px={int(obj.sum())} colour std={col_std:.4f} "
f"(mode={g.mode})")
Image.fromarray((composite_white(r) * 255).astype(np.uint8)).save(
DEBUG / "selfcheck_uvpath.png")
def main():
print("APPEVAL SELF-CHECK")
for obj in OBJECTS:
check_object(obj)
check_uv_path()
print("\n==================================================")
print("ALL SELF-CHECKS PASSED")
print("proof sheets ->", DEBUG)
print("==================================================")
if __name__ == "__main__":
main()
|