File size: 2,914 Bytes
2c0472d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | #!/usr/bin/env python3
"""FORGE3DBench rest-658 with the FINAL evaluator: thin wrapper around eval_final.py (imported, NOT modified).
rest658 = the 658 objects of the 808-object FORGE3DBench that are not in FB150. They come from the same PSL MVScenes
scenes and were built with the same builder, so they are scored with the FB150 protocol verbatim (eval_final --dataset fb150
code path: masked GT-image IV + held-out-camera NV with visibility >= --min-vis 0.20, geometry vs the canon GT).
The wrapper only
* accepts --dataset rest658 (runs the fb150 code path),
* NV filter = eval_final's (vis >= --min-vis, default 0.20) AND px >= --min-px (default 400) AND files present.
heldout_rest658.json lists ALL 20 non-input scene cams per object (vis, px stored); with the defaults the NV set is
exactly the FB150 rule (FB150's index was pre-filtered to vis>0 & px>=400 at build time).
* sets the defaults: --exp -> rest658 exp_4v / exp_1v (data/rest658_exp_{4v,1v}.tar),
--heldout-root -> rest658_heldout (data/rest658_heldout.tar.gz.part*),
--heldout-index -> heldout_index/heldout_rest658.json,
* labels summary.json with dataset = "rest658" (protocol_dataset = "fb150").
All other flags are eval_final.py's (--setting, --meshes, --method, --out, --shard/--nshards, --summarize, --pred-is-gt, ...).
"""
import sys
from pathlib import Path
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import eval_final as EF # noqa: E402
LP = EF.LP
REST_EXP = {"4v": f"{LP}/.debug/forgebench_eval/rest658/exp_4v", "1v": f"{LP}/.debug/forgebench_eval/rest658/exp_1v"}
REST_HELDOUT = f"{LP}/.debug/hub_rest658_heldout/rest658_heldout"
REST_INDEX = HERE / "heldout_index" / "heldout_rest658.json"
MIN_PX = 400
_setup = EF.setup
def setup(a):
if a.exp is None:
a.exp = Path(REST_EXP[a.setting])
if a.heldout_root is None:
a.heldout_root = Path(REST_HELDOUT)
if a.heldout_index is None:
a.heldout_index = REST_INDEX
meta = _setup(a)
meta.update(min_px=MIN_PX, dataset="rest658", protocol_dataset="fb150", heldout_index=str(a.heldout_index),
heldout_root=str(a.heldout_root))
return meta
EF.setup = setup
_nv_views = EF.nv_views
def nv_views(a, recs):
return [r for r in _nv_views(a, recs) if r.get("files", True) and r["px"] >= MIN_PX]
EF.nv_views = nv_views
if __name__ == "__main__":
argv = sys.argv[1:]
if "--min-px" in argv:
i = argv.index("--min-px"); MIN_PX = int(argv[i + 1]); del argv[i:i + 2]
if "--dataset" in argv:
i = argv.index("--dataset")
if argv[i + 1] != "rest658":
sys.exit("eval_rest658_wrapper.py only runs --dataset rest658 (use eval_final.py for the others)")
argv[i + 1] = "fb150"
else:
argv = ["--dataset", "fb150"] + argv
sys.argv = [sys.argv[0]] + argv
EF.main()
|