forgebench/code/eval/tests/test_equiv_heldout.py

#!/usr/bin/env python3
"""Unit test: eval_final.masked_view == eval_heldout.heldout_view (the path that produced
the held-out NV numbers / viz_pairs artifact) on FB150 + Omni views, and the far-clip
fix only changes renders of cameras farther than 10 units.
Usage: python test_equiv_heldout.py MESH_DIR_FB150 [OBJ ...]   (server-local paths)"""
import sys, json
from pathlib import Path
import numpy as np

HERE = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(HERE))
import eval_final as EF                                   # noqa: E402  (package, far-fixed renderer)
sys.path.insert(0, str(HERE / "heldout"))
import eval_heldout as EH                                 # noqa: E402  (original module)
import importlib.util                                     # noqa: E402
# 'render' is already bound to the package (far-fixed) module -> load the ORIGINAL renderer explicitly
_spec = importlib.util.spec_from_file_location("render_orig", "/home/nvidia/jonghoon/mv-mesh/metrics/appeval/render.py")
R_ORIG = importlib.util.module_from_spec(_spec); _spec.loader.exec_module(R_ORIG)
assert R_ORIG.render_input_view.__defaults__[1] == 20.0 and EF.R.render_input_view.__defaults__[1] is None
EH.R = R_ORIG                                             # heldout_view now uses the original far=20 renderer

mesh_dir = Path(sys.argv[1]); objs = sys.argv[2:]
ctx = EF.R.get_ctx()
rep = []
for ds in ("fb150", "omni300"):
    idx = json.load(open(EH.DS[ds]["index"]))
    for obj in (objs if ds == "fb150" else ["asparagus_001", "bowl_001"]):
        mp = mesh_dir / f"{obj}.glb" if ds == "fb150" else EH.DS[ds]["exp"] / "renders" / f"{obj}_canon.glb"
        mesh = EF.EA.load_mesh(mp)
        g_new, g_old = EF.R.prepare_mesh(mesh), EH.R.prepare_mesh(mesh)
        for rec in idx[obj][:3] + [dict(tag="front", cam=-1, root=str(EH.DS[ds]["exp"]))]:
            m_old, p_old, r_old = EH.heldout_view(g_old, rec, obj, ctx)
            m_new, p_new, r_new = EF.masked_view(g_new, Path(rec["root"]), obj, rec["tag"], ctx)
            d = max(abs(m_old[k] - m_new[k]) for k in EF.MKEYS)
            rep.append(dict(ds=ds, obj=obj, tag=rec["tag"], cam_dist=round(m_new["cam_dist"], 2),
                            max_metric_diff=d, pix_equal=bool(np.array_equal(p_old, p_new, equal_nan=True)),
                            max_pix_diff=float(np.nanmax(np.abs(p_old - p_new))),
                            lpips_old=m_old["lpips"], lpips_new=m_new["lpips"]))
            print(rep[-1], flush=True)
near = [r for r in rep if r["cam_dist"] < 10]
# pass: identical metrics (<=1e-6; rasteriser/texture sampling is not bit-deterministic across two
# MeshGL instances of large UV-textured meshes, max pixel diff is reported)
ok = all(r["max_metric_diff"] <= 1e-6 for r in near)
print(f"EQUIV (cams < 10 units: {len(near)} views, bit-identical renders {sum(r['pix_equal'] for r in near)}, "
      f"max metric diff {max(r['max_metric_diff'] for r in near):.2e}, max pix diff {max(r['max_pix_diff'] for r in near):.2e}): "
      f"{'PASS' if ok else 'FAIL'}")
far = [r for r in rep if r["cam_dist"] >= 10]
print(f"far cams (>=10 units, far-clip fix may change render): {len(far)} views, "
      f"changed {sum(not r['pix_equal'] for r in far)}")
sys.exit(0 if ok else 1)