forgebench/code/eval/aggregate_results.py

#!/usr/bin/env python
"""Aggregate evaluate_appforce results into the paper-table row.
usage: python aggregate_results.py DIR [DIR ...]   (each DIR holds results.json or shards/subdirs with results.json)
Prints n, IV LPIPS/PSNR/SSIM, NV LPIPS/PSNR/SSIM, CD-L1, F@0.01 (means over objects that have all parts;
an object appearing in several shards/retries is counted once, preferring a complete record)."""
import glob, json, sys, numpy as np
rows = {}
for d in sys.argv[1:]:
    for f in glob.glob(d + "/**/results.json", recursive=True):
        for r in json.load(open(f)):
            if not isinstance(r, dict) or "object" not in r: continue
            ok = r.get("geometry") and r.get("input_view") and r.get("novel_view")
            if ok: rows[r["object"]] = r
R = list(rows.values())
if not R: sys.exit("no complete records")
m = lambda k, s: np.mean([x[k][s] for x in R])
print(f"n={len(R)}  IV {m('input_view','lpips'):.4f}/{m('input_view','psnr'):.2f}/{m('input_view','ssim'):.3f}"
      f"  NV {m('novel_view','lpips'):.4f}/{m('novel_view','psnr'):.2f}/{m('novel_view','ssim'):.3f}"
      f"  CD {np.mean([x['geometry']['cd_l1'] for x in R]):.4f}  F@.01 {np.mean([x['geometry']['f01'] for x in R]):.3f}")