forgebench/code/eval/eval_rest658_wrapper.py

#!/usr/bin/env python3
"""FORGE3DBench rest-658 with the FINAL evaluator: thin wrapper around eval_final.py (imported, NOT modified).

rest658 = the 658 objects of the 808-object FORGE3DBench that are not in FB150. They come from the same PSL MVScenes
scenes and were built with the same builder, so they are scored with the FB150 protocol verbatim (eval_final --dataset fb150
code path: masked GT-image IV + held-out-camera NV with visibility >= --min-vis 0.20, geometry vs the canon GT).
The wrapper only
  * accepts --dataset rest658 (runs the fb150 code path),
  * NV filter = eval_final's (vis >= --min-vis, default 0.20) AND px >= --min-px (default 400) AND files present.
    heldout_rest658.json lists ALL 20 non-input scene cams per object (vis, px stored); with the defaults the NV set is
    exactly the FB150 rule (FB150's index was pre-filtered to vis>0 & px>=400 at build time).
  * sets the defaults: --exp  -> rest658 exp_4v / exp_1v   (data/rest658_exp_{4v,1v}.tar),
                       --heldout-root -> rest658_heldout    (data/rest658_heldout.tar.gz.part*),
                       --heldout-index -> heldout_index/heldout_rest658.json,
  * labels summary.json with dataset = "rest658" (protocol_dataset = "fb150").
All other flags are eval_final.py's (--setting, --meshes, --method, --out, --shard/--nshards, --summarize, --pred-is-gt, ...).
"""
import sys
from pathlib import Path

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import eval_final as EF  # noqa: E402

LP = EF.LP
REST_EXP = {"4v": f"{LP}/.debug/forgebench_eval/rest658/exp_4v", "1v": f"{LP}/.debug/forgebench_eval/rest658/exp_1v"}
REST_HELDOUT = f"{LP}/.debug/hub_rest658_heldout/rest658_heldout"
REST_INDEX = HERE / "heldout_index" / "heldout_rest658.json"
MIN_PX = 400

_setup = EF.setup


def setup(a):
    if a.exp is None:
        a.exp = Path(REST_EXP[a.setting])
    if a.heldout_root is None:
        a.heldout_root = Path(REST_HELDOUT)
    if a.heldout_index is None:
        a.heldout_index = REST_INDEX
    meta = _setup(a)
    meta.update(min_px=MIN_PX, dataset="rest658", protocol_dataset="fb150", heldout_index=str(a.heldout_index),
                heldout_root=str(a.heldout_root))
    return meta


EF.setup = setup
_nv_views = EF.nv_views


def nv_views(a, recs):
    return [r for r in _nv_views(a, recs) if r.get("files", True) and r["px"] >= MIN_PX]


EF.nv_views = nv_views

if __name__ == "__main__":
    argv = sys.argv[1:]
    if "--min-px" in argv:
        i = argv.index("--min-px"); MIN_PX = int(argv[i + 1]); del argv[i:i + 2]
    if "--dataset" in argv:
        i = argv.index("--dataset")
        if argv[i + 1] != "rest658":
            sys.exit("eval_rest658_wrapper.py only runs --dataset rest658 (use eval_final.py for the others)")
        argv[i + 1] = "fb150"
    else:
        argv = ["--dataset", "fb150"] + argv
    sys.argv = [sys.argv[0]] + argv
    EF.main()