"""2D keypoints on COCO val2017 (GT person boxes, OKS keypoint AP via pycocotools): the 17 COCO keypoints taken from (a) the projection of the 3D MHR keypoints (the parametric output: SAM 3D Body, FAB) and (b) FAB's dense-head heatmaps (an independent 2D output). Default intrinsics (no camera is known for COCO images). The paper's COCO rows: --predictor --body_engine torch (the released call with the PyTorch body). python -m bench.eval_coco2d --root $FAB_DATA/coco_val --fab_ckpt fab-l --predictor --body_engine torch --out --shard 0 --num_shards 4 python -m bench.eval_coco2d --root $FAB_DATA/coco_val --out --merge """ import argparse import glob import json from pathlib import Path import cv2 import numpy as np import torch from fab import paths MHR_TO_COCO = [0, 1, 2, 3, 4, 5, 6, 7, 8, 62, 41, 9, 10, 11, 12, 13, 14] def main(): p = argparse.ArgumentParser() p.add_argument("--root", required=True) p.add_argument("--fab_ckpt", default="", help="FAB model (fab-l / fab-s / fab-h or a path; empty: SAM 3D Body)") p.add_argument("--ckpt", default=str(paths.sam3d_dir() / "model.ckpt"), help="SAM 3D Body (its network weights only for the SAM 3D Body rows)") p.add_argument("--out", required=True) p.add_argument("--shard", type=int, default=0) p.add_argument("--num_shards", type=int, default=1) p.add_argument("--merge", action="store_true") p.add_argument("--fit_smpl", default="", help="FAB: SMPL fitter checkpoint: also score the fitted SMPL's MHR body " "keypoints (regressed from its vertices by the 23-joint regressor --reg23): one-pass (smpl_onepass) and " "the refined fit (smpl_densefit, fab/gn_fit.DenseFit)") p.add_argument("--reg23", default=str(paths.home() / "extras" / "mhr23_regressor.npz"), help="optional (not part of the download): the fit row") p.add_argument("--predictor", action="store_true", help="the released path, FABPredictor(--fab_ckpt) with its " "defaults: rows dense (its keypoints2d), projected (the MHR head's keypoints2d_projected) and fit (the " "fitted SMPL mesh_smpl_fit's MHR body keypoints through --reg23, projected with the default K; only when --reg23 exists)") p.add_argument("--body_engine", default="auto", help="--predictor: FABPredictor's body_engine ('auto' = the default, the bundle's " "TensorRT engine when one was built; 'torch' = the PyTorch body, the paper's COCO row; or an engine path)") p.add_argument("--dense_w2d", default="0.1", help="--fit_smpl: 2D weights of the refined fit (one row each)") p.add_argument("--bm_dir", default=str(paths.body_models_dir())) p.add_argument("--decoder_fp32", action="store_true", help="FAB wrapper: BF16 backbone, FP32 decoder/heads (precision-parity diagnostic)") p.add_argument("--local_windows", default="", help="dense head: also score the soft-argmax within r heatmap cells of each peak (comma-separated r)") args = p.parse_args() windows = [int(r) for r in args.local_windows.split(",") if r] from pycocotools.coco import COCO ann_file = Path(args.root) / "annotations" / "person_keypoints_val2017.json" coco = COCO(str(ann_file)) out = Path(args.out) if args.merge: from pycocotools.cocoeval import COCOeval res = {} for f in sorted(glob.glob(str(out / "shard*.json"))): for name, rows in json.loads(Path(f).read_text()).items(): res.setdefault(name, []).extend(rows) summary = {} for name, rows in res.items(): ev = COCOeval(coco, coco.loadRes(rows), "keypoints") ev.params.imgIds = sorted({r["image_id"] for r in rows}) ev.evaluate(); ev.accumulate(); ev.summarize() summary[name] = {"AP": ev.stats[0], "AP50": ev.stats[1], "AP75": ev.stats[2], "AR": ev.stats[5], "n": len(rows)} (out / "summary.json").write_text(json.dumps(summary, indent=2)) print(json.dumps(summary, indent=2)) return from fab.data.transforms import Compose, GetBBoxCenterScale, TopdownAffine, VisionTransformWrapper from fab.data.utils.prepare_batch import prepare_batch from torchvision.transforms import ToTensor anns = [a for a in coco.loadAnns(coco.getAnnIds(catIds=[1], iscrowd=False)) if a["num_keypoints"] > 0] anns = anns[args.shard :: args.num_shards] if args.predictor: # the released path (FABPredictor defaults), K = the image-diagonal focal as prepare_batch's from fab.predictor import FABPredictor fab = FABPredictor(args.fab_ckpt, detector="", bm_dir=args.bm_dir, fast=True, parts=False, mask=False, body_engine="" if args.body_engine == "torch" else args.body_engine) reg23 = np.load(args.reg23)["reg"].astype(np.float64) if Path(args.reg23).exists() else None body23 = list(range(21)) + [41, 62] rows = {"dense": [], "projected": [], "fit": []} for k, a in enumerate(anns): info = coco.loadImgs(a["image_id"])[0] img = cv2.cvtColor(cv2.imread(str(Path(args.root) / "val2017" / info["file_name"])), cv2.COLOR_BGR2RGB) x, y, w_, h_ = a["bbox"] H, W = img.shape[:2] f = float(np.hypot(H, W)) K = np.array([[f, 0, W / 2], [0, f, H / 2], [0, 0, 1]], np.float64) pp = fab.predict(img, K.astype(np.float32), boxes=np.array([[x, y, x + w_, y + h_]], np.float32))[0] rows["dense"].append((a, np.asarray(pp["keypoints2d"], np.float64))) rows["projected"].append((a, np.asarray(pp["keypoints2d_projected"], np.float64))) if reg23 is not None: q = (reg23 @ np.asarray(pp["mesh_smpl_fit"], np.float64)) @ K.T full70 = np.zeros((70, 2)) full70[body23] = q[:, :2] / np.maximum(q[:, 2:], 1e-3) rows["fit"].append((a, full70)) if k % 500 == 0: print(f"[{k}/{len(anns)}]", flush=True) res = {name: [{"image_id": a["image_id"], "category_id": 1, "score": 1.0, "keypoints": np.concatenate([kp[MHR_TO_COCO], np.full((17, 1), 2.0)], 1).ravel().tolist()} for a, kp in rr] for name, rr in rows.items() if rr} out.mkdir(parents=True, exist_ok=True) (out / f"shard{args.shard}.json").write_text(json.dumps(res)) print("done", {k: len(v) for k, v in res.items()}, flush=True) return transform = Compose([GetBBoxCenterScale(), TopdownAffine(input_size=[512, 512], use_udp=False), VisionTransformWrapper(ToTensor())]) if args.fab_ckpt: from fab.decoder_model import build_fab_decoder # "teacher": SAM 3D Body's own weights through the same one-pass wrapper (path check) state = torch.load(args.fab_ckpt, map_location="cpu", weights_only=False) if args.fab_ckpt != "teacher" else \ {"args": {"arch": "fabt", "size": 512}, "model": {}} ta = state.get("args", {}) model, tb = build_fab_decoder(args.ckpt, ta.get("backbone", "dinov3_vitb16"), ta.get("size", 512), pretrained=False, dec_layers=ta.get("dec_layers", 0)) if ta.get("arch") == "fabt": # fine-tuned SAM 3D Body: its own backbone (weights in the state) model.body.backbone = tb model.body.backbone_dtype = torch.bfloat16 else: del tb w = dict(state["model"]) w.update(state.get("ema") or {}) if "dense_head.surface_idx" in w: from fab.dense_head import dense_hr, dense_mask_depth, dense_mask_query, dense_part_depth, dense_xyz_layers from fab.dense_head import dense_head_kwargs model.add_dense_head(w["dense_head.surface_idx"].tolist(), residual=bool(int(w.get("dense_head.residual", torch.tensor(0)))), parts="dense_head.part_emb" in w, xyz_layers=dense_xyz_layers(w), part_depth=dense_part_depth(w), hr=dense_hr(w), mask_depth=dense_mask_depth(w), mask_query=dense_mask_query(w), **dense_head_kwargs(w)).cuda() model.dense_head.keep_logits = bool(args.local_windows) model.load_state_dict(w, strict=False) model = model.eval() model.body.head_pose.enable_fast() if args.decoder_fp32: model.body.backbone.forward = torch.autocast("cuda", dtype=torch.bfloat16)( model.body.backbone.forward) else: from fab import SAM3DBodyEstimator, load_sam_3d_body sam, cfg = load_sam_3d_body(checkpoint_path=args.ckpt, device="cuda", mhr_path=str(Path(args.ckpt).parent / "assets" / "mhr_model.pt")) est = SAM3DBodyEstimator(sam_3d_body_model=sam, model_cfg=cfg) rows = {"projected": [], "dense": []} fits = None if args.fab_ckpt and args.fit_smpl and model.dense_head is not None: # SMPL fits of the dense head's outputs from bench.body_models import SMPL from bench.mhr_smpl import MHRToSMPL from fab.fast_fit import GraphFit from fab.gn_fit import DenseFit, GraphDenseFit, crop_camera from fab.smpl_fitter import load_fitter body = SMPL(Path(args.bm_dir) / "smpl" / "SMPL_NEUTRAL.pkl").cuda().eval() sidx = model.dense_head.surface_idx reg23 = torch.from_numpy(np.load(args.reg23)["reg"]).float().cuda() one = GraphFit(load_fitter(args.fit_smpl), body, sidx) dense_fits = {float(w): GraphDenseFit(DenseFit(one.fit, body, sidx, reg23, w_2d=float(w))) for w in args.dense_w2d.split(",")} fits = (MHRToSMPL(Path(args.bm_dir) / "mhr").cuda(), one, dense_fits, reg23, crop_camera) body23 = list(range(21)) + [41, 62] # MHR70 body keypoints (their COCO subset scored like the others) for k, a in enumerate(anns): info = coco.loadImgs(a["image_id"])[0] img = cv2.cvtColor(cv2.imread(str(Path(args.root) / "val2017" / info["file_name"])), cv2.COLOR_BGR2RGB) x, y, w_, h_ = a["bbox"] box = np.array([x, y, x + w_, y + h_], np.float32) if args.fab_ckpt: b = prepare_batch(img, transform, box[None], None, None) with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16, enabled=not args.decoder_fp32): o = model(b["img"][0].cuda(), b["bbox_center"][0].cuda(), b["bbox_scale"][0].cuda(), b["ori_img_size"][0].cuda(), b["cam_int"].cuda(), affine=b["affine_trans"][0].cuda(), return_vertices=fits is not None) if fits is not None and "dense_uv" in o: to_smpl, one, dense_fits, reg23, crop_camera = fits with torch.no_grad(): mesh = to_smpl(o["pred_vertices"][0].float())[None] off = o["dense_xyz"][0].float() shift = off[70:84][[2, 3]].mean(0) surf = (off[84:] - shift + mesh[0, sidx])[None] mo = off[:70][body23] kp3 = (mo - mo[[9, 10]].mean(0) + reg23 @ mesh[0])[None] K = b["cam_int"][0].float().cuda() Kc = crop_camera(K[None], b["affine_trans"][0].float().cuda()) cam_t = o["pred_cam_t"][0].float() _, _, v1 = one(surf) v1 = v1[0] + cam_t fitted = [(f"smpl_densefit_w{w:g}", dense(surf, mesh, kp3, (o["dense_uv"][0].float() * 512)[body23][None], Kc)[2][0].clone()) for w, dense in dense_fits.items()] for name, v in [("mesh_reg23", mesh[0] + cam_t), ("smpl_onepass", v1)] + fitted: q = (reg23 @ v) @ K.T # MHR body keypoints of the fitted mesh, projected (image px) full70 = np.zeros((70, 2)) full70[body23] = (q[:, :2] / q[:, 2:].clamp_min(1e-3)).cpu().numpy() rows.setdefault(name, []).append((a, full70)) kp = o["pred_keypoints_2d"][0, :, :2].float().cpu().numpy() if "dense_uv" in o: A = np.vstack([b["affine_trans"][0, 0].numpy().astype(np.float64), [0, 0, 1]]) uv = o["dense_uv"][0, :70].float().cpu().numpy() * 512 full = (np.linalg.inv(A) @ np.concatenate([uv, np.ones((70, 1))], 1).T).T[:, :2] rows["dense"].append((a, full)) for r in windows: # the same heatmaps, local soft-argmax decoding from fab.dense_head import local_soft_argmax lg = getattr(model.dense_head, "last_logits_hr", None) lg, rr = (model.dense_head.last_logits[:, :70], r) if lg is None else (lg[:, :70], 2 * r) uv = local_soft_argmax(lg.float(), rr)[0].cpu().numpy() * 512 full = (np.linalg.inv(A) @ np.concatenate([uv, np.ones((70, 1))], 1).T).T[:, :2] rows.setdefault(f"dense_lw{r}", []).append((a, full)) else: o = est.process_one_image(img, bboxes=box[None], inference_type="body")[0] kp = np.asarray(o["pred_keypoints_2d"])[:, :2] rows["projected"].append((a, kp)) if k % 500 == 0: print(f"[{k}/{len(anns)}]", flush=True) res = {name: [{"image_id": a["image_id"], "category_id": 1, "score": 1.0, "keypoints": np.concatenate([kp[MHR_TO_COCO], np.full((17, 1), 2.0)], 1).ravel().tolist()} for a, kp in rr] for name, rr in rows.items() if rr} out.mkdir(parents=True, exist_ok=True) (out / f"shard{args.shard}.json").write_text(json.dumps(res)) print("done", {k: len(v) for k, v in res.items()}, flush=True) if __name__ == "__main__": main()