Blame for code/scripts/mpjpe.py

# SPDX-FileCopyrightText: © 2026 Tenstorrent Inc.
# SPDX-License-Identifier: Apache-2.0
"""3-D joint MPJPE evaluation for the tt-nn port.

Two modes:

* **default** — torch CPU reference treated as GT.  Reports the MPJPE
  between tt-nn-predicted joints and reference-predicted joints in mm,
  matching the convention used by InterHand2.6M / FreiHAND papers.  The
  number is the joint-space analogue of the regression-vector PCC reported
  by ``scripts/eval.py``.

* **--gt-json /path/to/interhand_test.json** — when an InterHand2.6M-style
  annotation file is available, treats those 3-D joints as GT and reports
  the absolute MPJPE / PA-MPJPE per sample.  The JSON is expected to look
  like::

      [
        {"image": "...", "bbox": [x1, y1, x2, y2], "joints_3d_mm": [[...]],
         "is_right": 1},
        ...
      ]

Run from the repo root:

    python scripts/mpjpe.py \
        --ckpt /path/to/_DATA/hamer_ckpts/checkpoints/hamer.ckpt \
        --mean /path/to/_DATA/data/mano_mean_params.npz \
        --mano /path/to/_DATA/data/mano/MANO_RIGHT.pkl \
        --images /path/to/hamer/example_data
"""
from __future__ import annotations

import argparse
import json
import os
import sys
from pathlib import Path

import numpy as np
import torch

_REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(_REPO_ROOT))

from hamer import Hamer, load_hamer_checkpoint  # noqa: E402
from hamer.preprocess import preprocess  # noqa: E402
from hamer.mano import MANO  # noqa: E402


def split_output(out: torch.Tensor):
    rotmats = out[:, :144].reshape(-1, 16, 3, 3)
    betas = out[:, 144:154]
    cam = out[:, 154:157]
    return rotmats, betas, cam


def root_relative(joints_mm: np.ndarray) -> np.ndarray:
    """Subtract the wrist joint (joint 0) — root-relative MPJPE matches the
    reporting convention of InterHand2.6M and FreiHAND."""
    return joints_mm - joints_mm[..., :1, :]


def mpjpe(pred: np.ndarray, gt: np.ndarray) -> float:
    """Mean per-joint position error in mm — both inputs ``(J, 3)``."""
    return float(np.linalg.norm(pred - gt, axis=-1).mean())


def procrustes_align(pred: np.ndarray, gt: np.ndarray) -> np.ndarray:
    """Similarity-align ``pred`` onto ``gt`` (Procrustes / PA-MPJPE pre-step).
    Returns the aligned ``pred``."""
    mu_p, mu_g = pred.mean(0), gt.mean(0)
    p, g = pred - mu_p, gt - mu_g
    s_p = np.linalg.norm(p)
    s_g = np.linalg.norm(g)
    p_n = p / (s_p + 1e-12)
    g_n = g / (s_g + 1e-12)
    U, _, Vt = np.linalg.svd(p_n.T @ g_n)
    R = (U @ Vt).T
    if np.linalg.det(R) < 0:
        Vt[-1] *= -1
        R = (U @ Vt).T
    s = (s_g / (s_p + 1e-12)) * np.trace((p_n.T @ g_n) @ R.T)
    return s * (pred - mu_p) @ R.T + mu_g


def parse_args():
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument("--ckpt", type=Path, required=True)
    p.add_argument("--mean", type=Path, required=True)
    p.add_argument("--mano", type=Path, required=True)
    p.add_argument("--images", type=Path, required=True, help="dir of input images")
    p.add_argument("--bboxes", type=Path, required=True,
                   help="JSON {image_name: [x1,y1,x2,y2]} for the synthetic-GT mode")
    p.add_argument("--gt-json", type=Path, default=None,
                   help="Optional InterHand2.6M-style GT (image/bbox/joints_3d_mm). "
                        "Without this, the torch CPU reference is treated as GT.")
    p.add_argument("--device-id", type=int, default=int(os.environ.get("HAMER_DEVICE", "0")))
    return p.parse_args()


def load_gt_samples(gt_json: Path):
    return json.loads(gt_json.read_text())


def collect_demo_samples(images_dir: Path, bboxes_path: Path):
    """Build a sample list from a directory + multi-hand bboxes JSON.  GT
    joints come from the torch CPU reference at run time (the
    synthetic-GT mode)."""
    bboxes_map = json.loads(bboxes_path.read_text())
    out = []
    for name, entries in bboxes_map.items():
        p = images_dir / name
        if not p.exists():
            continue
        if isinstance(entries, dict):
            entries = [entries]
        elif entries and not isinstance(entries[0], (dict, list)):
            entries = [entries]
        for i, e in enumerate(entries):
            if isinstance(e, dict):
                bbox, is_right = e["bbox"], int(e.get("is_right", 1))
            else:
                bbox, is_right = e, 1
            out.append({
                "image": str(p),
                "bbox": list(bbox),
                "is_right": is_right,
                "joints_3d_mm": None,
                "label": f"{name}#{i}",
            })
    return out


def main() -> int:
    args = parse_args()
    print("Loading reference + checkpoint…")
    torch.manual_seed(0)
    ref = Hamer(); ref.eval()
    load_hamer_checkpoint(ref, args.ckpt, args.mean)
    mano = MANO(args.mano)

    os.environ["HAMER_USE_TT"] = "1"
    os.environ["HAMER_FORCE_TT"] = "1"
    from hamer import build_from_reference
    print("Opening tt-nn device…")
    tt_model = build_from_reference(ref, device_id=args.device_id)

    samples = load_gt_samples(args.gt_json) if args.gt_json else collect_demo_samples(args.images, args.bboxes)
    if not samples:
        print("ERROR: no samples")
        return 2

    rows = []
    for s in samples:
        x, _ = preprocess(s["image"], bbox_xyxy=tuple(s["bbox"]), is_right=bool(s.get("is_right", 1)))
        with torch.no_grad():
            ref_out = ref(x)
            tt_out = tt_model(x)
        rot_ref, betas_ref, _ = split_output(ref_out)
        rot_tt, betas_tt, _ = split_output(tt_out)
        with torch.no_grad():
            j_ref = mano.joints_21(rot_ref, betas_ref)[0].cpu().numpy() * 1000.0  # → mm
            j_tt = mano.joints_21(rot_tt, betas_tt)[0].cpu().numpy() * 1000.0

        gt = j_ref if s["joints_3d_mm"] is None else np.asarray(s["joints_3d_mm"])
        gt_rr, ref_rr, tt_rr = root_relative(gt), root_relative(j_ref), root_relative(j_tt)
        m_ref = mpjpe(ref_rr, gt_rr)                                    # 0 in synthetic-GT mode
        m_tt = mpjpe(tt_rr, gt_rr)
        m_tt_pa = mpjpe(procrustes_align(tt_rr, gt_rr), gt_rr)
        label = s.get("label") or Path(s["image"]).name
        rows.append((label, m_ref, m_tt, m_tt_pa))

    print()
    print(f"{'image':<24} {'MPJPE(ref)':>11} {'MPJPE(tt)':>10} {'PA-MPJPE(tt)':>13}    [mm]")
    print("-" * 76)
    for name, m_r, m_t, m_t_pa in rows:
        print(f"{name:<24} {m_r:>10.3f}  {m_t:>9.3f}  {m_t_pa:>12.3f}")
    print("-" * 76)
    arr = np.array([r[1:] for r in rows])
    print(f"{'mean':<24} {arr[:, 0].mean():>10.3f}  {arr[:, 1].mean():>9.3f}  {arr[:, 2].mean():>12.3f}")

    if hasattr(tt_model, "close"):
        tt_model.close()
    return 0


if __name__ == "__main__":
    raise SystemExit(main())