"""Fast SAM 3D Body's evaluation protocol on saved predictions (bench/eval_hmr.py --save_pred): the MHR mesh is converted to SMPL by the official MHR tool's iterative fit (tools/mhr_smpl_conversion, PyTorchSMPLFitting: Adam on vertex + edge losses, neutral SMPL, per frame) instead of being scored through the barycentric map alone, then scored with the same joints as ours (14 H36M-regressor joints, hip-midpoint aligned; 24 SMPL joints, pelvis aligned; PVE). Their paper's Table 1 converts with this tool ("the procedure described in [MHR]") and reports 3DPW on 14 joints and EMDB on 24 joints. The barycentric target of the fit equals our saved pred mesh (same mhr2smpl_mapping.npz). Also the helpers of the official MHR -> SMPL fit used by bench/eval_standard_protocol.py and bench/eval_ssp3d.py (import_official_fitting, smplx_neutral). The MHR tool is the public MHR repository's tools/mhr_smpl_conversion (docs/evaluation.md); Fast SAM 3D Body's converter (--fast3db_root) is its official code, installed by you. python -m bench.eval_fast3db_protocol --preds $FAB_HOME/bench/preds/ --dataset 3dpw \ --mhr_tools $FAB_HOME/src/MHR/tools/mhr_smpl_conversion --fast3db_root """ import argparse import glob import json import sys import time import types from pathlib import Path import numpy as np import torch from bench.body_models import SMPL, load_smpl_pkl from bench.eval_hmr import H36M_TO_J14, errors from fab import paths # the public MHR repository's MHR -> SMPL conversion tool (github.com/facebookresearch/MHR, tools/mhr_smpl_conversion) MHR_TOOLS_DEFAULT = Path(__import__("os").environ.get("FAB_MHR_TOOLS", paths.home() / "src" / "MHR" / "tools" / "mhr_smpl_conversion")) def import_official_fitting(tools_dir, extra_path=""): """Import the official PyTorchSMPLFitting without pymomentum (the MHR -> SMPL path never calls it).""" class AnyModule(types.ModuleType): # every attribute is a placeholder class (type annotations at import time) def __getattr__(self, k): if k.startswith("__"): raise AttributeError(k) return type(k, (), {}) def stub(name, **attrs): import importlib.machinery m = sys.modules.get(name) or AnyModule(name) m.__spec__ = importlib.machinery.ModuleSpec(name, None) # torch._dynamo's trace rules call find_spec m.__dict__.update(attrs) sys.modules[name] = m if "." in name: # bind as an attribute of the parent ("import a.b as c" reads a.b) parent, child = name.rsplit(".", 1) setattr(sys.modules[parent], child, m) return m for n in ("pymomentum", "pymomentum.solver", "pymomentum.geometry", "pymomentum.torch", "pymomentum.torch.character"): stub(n) stub("mhr") stub("mhr.mhr", MHR=type("MHR", (), {})) try: import sklearn.cluster # noqa: F401 except ImportError: stub("sklearn") stub("sklearn.cluster", KMeans=None) for p in (extra_path, str(tools_dir)): if p and p not in sys.path: sys.path.insert(0, p) from pytorch_fitting import PyTorchSMPLFitting return PyTorchSMPLFitting def smplx_neutral(bm_dir): """The smplx library's SMPL (as the official tool uses) from our chumpy-free neutral model data.""" import smplx from smplx.utils import Struct d = load_smpl_pkl(Path(bm_dir) / "smpl" / "SMPL_NEUTRAL.pkl") data = Struct(v_template=d["v_template"], shapedirs=d["shapedirs"], posedirs=d["posedirs"], J_regressor=d["J_regressor"], weights=d["weights"], kintree_table=d["kintree_table"], f=d["faces"]) return smplx.SMPL(model_path=str(Path(bm_dir) / "smpl"), data_struct=data, gender="neutral") def mlp_convert(pred, fast3db_root, bm_dir, batch=1024): """Fast SAM 3D Body's deployed MHR -> SMPL network (mhr2smpl/experiments/multiview_n30000_e500, single view, as run_publisher.py): barycentric SMPL-topology vertices in SAM 3D Body's y-up frame, 1500-vertex subsample, centred -> global orient (6D) + 21 body joints + betas -> neutral SMPL, returned in the camera frame.""" import importlib.util spec = importlib.util.spec_from_file_location("mv_mhr2smpl", Path(fast3db_root) / "mocap/core/multiview_mhr2smpl.py") mv = importlib.util.module_from_spec(spec) spec.loader.exec_module(mv) d = Path(fast3db_root) / "mhr2smpl/experiments/multiview_n30000_e500" net = mv.MinimalMultiViewNet(input_dim=1500 * 3).cuda().eval() net.load_state_dict(torch.load(d / "best_model.pth", map_location="cuda", weights_only=True)) idx = torch.from_numpy(np.load(d / "sample_idx.npy").astype(np.int64)).cuda() body = SMPL(Path(bm_dir) / "smpl" / "SMPL_NEUTRAL.pkl").cuda() flip = torch.tensor([1.0, -1.0, -1.0], device="cuda") out = [] with torch.no_grad(): for s in range(0, len(pred), batch): v = torch.from_numpy(pred[s : s + batch]).float().cuda() * flip # camera -> SAM 3D Body's y-up frame sub = v[:, idx] x = (sub - sub.mean(1, keepdim=True)).flatten(1) go6d, bp, betas = mv.split_output(net.forward_single_view(x)) bp69 = torch.cat([bp, torch.zeros(len(bp), 69 - bp.shape[1], device="cuda")], 1) vs, _ = body(mv.rot6d_to_aa(go6d), bp69, betas, None) vs = vs - vs[:, idx].mean(1, keepdim=True) + sub.mean(1, keepdim=True) out.append((vs * flip).cpu().numpy()) return np.concatenate(out) def load(path, dataset): parts = [np.load(f) for f in sorted(glob.glob(f"{path}/{dataset}_pred_shard*.npz"))] out = {k: np.concatenate([p[k] for p in parts]) for k in parts[0].files} v = lambda n: out[f"{n}_verts"].astype(np.float32) + out[f"{n}_root"][:, None] # noqa: E731 return out["ids"].astype(str), v("pred"), v("gt") def main(): p = argparse.ArgumentParser() p.add_argument("--preds", required=True, help="dir with _pred_shard*.npz") p.add_argument("--dataset", required=True) p.add_argument("--bm_dir", default=str(paths.body_models_dir())) p.add_argument("--mhr_tools", default=str(MHR_TOOLS_DEFAULT)) p.add_argument("--batch", type=int, default=256) p.add_argument("--max", type=int, default=0) p.add_argument("--out", default="") p.add_argument("--fast3db_root", required=True, help="a checkout of Fast SAM 3D Body's official code") p.add_argument("--dataset_root", default="", help="also score against neutral-model GT (protocol variant)") args = p.parse_args() ids, pred, gt = load(args.preds, args.dataset) if args.max: ids, pred, gt = ids[: args.max], pred[: args.max], gt[: args.max] Fitting = import_official_fitting(args.mhr_tools) import trimesh body = smplx_neutral(args.bm_dir).cuda() for q in body.parameters(): q.requires_grad_(False) edges = torch.from_numpy(trimesh.Trimesh(body.v_template.cpu().numpy(), body.faces, process=False) .edges_unique.copy()).long().cuda() fitter = Fitting(smpl_model=body, smpl_edges=edges, smpl_model_type="smpl", hand_pose_dim=0, device="cuda", batch_size=args.batch) t0 = time.time() fitted = [] for s in range(0, len(pred), 16 * args.batch): # chunks (the solver batches inside) tgt = torch.from_numpy(pred[s : s + 16 * args.batch]).float().cuda() prm = fitter.fit(target_vertices=tgt, single_identity=False, is_tracking=False) with torch.no_grad(): v = body(betas=prm["betas"], body_pose=prm["body_pose"], global_orient=prm["global_orient"], transl=prm["transl"]).vertices fitted.append(v.cpu().numpy()) print(f"fitted {min(s + 16 * args.batch, len(pred))}/{len(pred)} ({time.time() - t0:.0f} s)", flush=True) fitted = np.concatenate(fitted) mlp = mlp_convert(pred, args.fast3db_root, args.bm_dir) j14 = np.load(Path(args.bm_dir) / "smpl" / "J_regressor_h36m.npy")[H36M_TO_J14].astype(np.float64) j24 = SMPL(Path(args.bm_dir) / "smpl" / "SMPL_NEUTRAL.pkl").J_regressor.numpy().astype(np.float64) gts = {"gendered_gt": gt} if args.dataset_root: # GT posed with the NEUTRAL model (protocol variant: some evaluators ignore the gender) from bench.datasets import DATASETS by_id = {s["id"]: s for s in DATASETS[args.dataset](args.dataset_root)} neutral = SMPL(Path(args.bm_dir) / "smpl" / "SMPL_NEUTRAL.pkl") gn = [] for i in ids: s = by_id[i] t = lambda a: torch.from_numpy(np.asarray(a, np.float32))[None] # noqa: E731 with torch.no_grad(): v, _ = neutral(t(s["global_orient"]), t(s["body_pose"]), t(s["betas"]), t(s["transl"])) gn.append(v[0].numpy() @ s["extrinsic"][:3, :3].T + s["extrinsic"][:3, 3]) gts["neutral_gt"] = np.stack(gn) rows = {} fit_err = [] for gname, gv in gts.items(): for mname, pv in (("barycentric", pred), ("official_fit", fitted), ("fast3db_mlp", mlp)): rows[f"{mname}/{gname}"] = [errors(pv[i].astype(np.float64), gv[i].astype(np.float64), j14, j24) for i in range(len(pred))] fit_err = [float(np.linalg.norm(fitted[i] - pred[i], axis=1).mean() * 1000) for i in range(len(pred))] summary = {k: {m: round(float(np.mean([r[m] for r in v])), 2) for m in v[0]} for k, v in rows.items()} summary["n"] = len(pred) summary["fit_vertex_error_mm"] = round(float(np.mean(fit_err)), 2) summary["fit_seconds"] = round(time.time() - t0, 1) print(json.dumps(summary, indent=1), flush=True) if args.out: Path(args.out).write_text(json.dumps(summary, indent=1) + "\n") root = pred.mean(1, keepdims=True) np.savez(Path(args.out).with_suffix(".npz"), ids=ids, root=root.astype(np.float32), fitted=(fitted - root).astype(np.float16), mlp=(mlp - root).astype(np.float16), **{f"{k.replace('/', '__')}__{m}": np.array([r[m] for r in v]) for k, v in rows.items() for m in v[0]}) if __name__ == "__main__": main()