Blame for code/hamer/rendering.py

# SPDX-FileCopyrightText: © 2026 Tenstorrent Inc.
# SPDX-License-Identifier: Apache-2.0
"""Render the predicted MANO mesh on the *original* image.

Projection convention derived from ``hamer/utils/renderer.py`` upstream.
The upstream applies a 180° X-rotation to vertices (trimesh) *and* then
renders with pyrender/OpenGL which does a y-flip (NDC y-up → image y-down).
Those two y-negations cancel.  Our equivalent — without pyrender — is to
project the MANO vertices **directly** without any rotation:

    proj_x = focal * (vx + tx) / (vz + tz) + W/2
    proj_y = focal * (vy + ty) / (vz + tz) + H/2

where (tx, ty, tz) = cam_crop_to_full(pred_cam, ...).  Applying only the
rotation (without the matching y-flip) inverts every vertex's y relative
to the hand center, making a downward-facing palm appear upward.

Triangles are rasterized as alpha-blended polygons with simple diffuse
shading from face-normal · view-vector — no ``pyrender`` / ``trimesh``
dependency.  The shading is enough to make finger curl readable instead of
the flat silhouette we used to ship.
"""
from __future__ import annotations

from pathlib import Path
from typing import Optional, Tuple

import numpy as np
import torch

# matplotlib is imported lazily inside ``overlay_mesh`` so the projection
# helpers (``cam_crop_to_full`` / ``project_mesh``) stay importable in a
# runtime that ships no matplotlib (the serving image).

FOCAL_LENGTH = 5000.0
MODEL_IMAGE_SIZE = 256


def cam_crop_to_full(
    cam_bbox: np.ndarray,
    box_center: np.ndarray,
    box_size: float,
    img_w: int,
    img_h: int,
    focal: float,
) -> np.ndarray:
    """Replicates ``hamer.utils.renderer.cam_crop_to_full`` on numpy.

    Returns the 3-D camera translation ``(tx, ty, tz)`` in original-image
    pixel-scaled world coordinates.
    """
    s, tx_b, ty_b = float(cam_bbox[0]), float(cam_bbox[1]), float(cam_bbox[2])
    cx, cy = float(box_center[0]), float(box_center[1])
    bs = box_size * s + 1e-9
    tz = 2.0 * focal / bs
    tx = 2.0 * (cx - img_w / 2.0) / bs + tx_b
    ty = 2.0 * (cy - img_h / 2.0) / bs + ty_b
    return np.array([tx, ty, tz], dtype=np.float64)


def project_mesh(
    verts: torch.Tensor,
    cam_bbox: torch.Tensor,
    box_center: Tuple[float, float],
    box_size: float,
    img_w: int,
    img_h: int,
    is_right: bool = True,
) -> Tuple[np.ndarray, np.ndarray]:
    """Project MANO vertices to ORIGINAL-image pixel coords.

    For a LEFT hand the upstream pipeline runs the (right-hand) model on a
    horizontally-mirrored input crop, then flips ``cam.x`` and ``verts.x``
    back before rendering — see ``demo.py`` lines 136-138 / 178-179.
    Reproduce that here.

    Returns ``(verts_2d, verts_camera_z)`` — the second array is per-vertex
    camera-space z (positive = depth in front of the camera) used for
    back-to-front face sorting in rendering.
    """
    focal = FOCAL_LENGTH / MODEL_IMAGE_SIZE * max(img_w, img_h)
    cam_bbox_np = cam_bbox.detach().cpu().numpy().copy()
    if not is_right:
        cam_bbox_np[1] *= -1.0
    cam_t = cam_crop_to_full(cam_bbox_np, np.array(box_center), box_size, img_w, img_h, focal)

    v = verts.detach().cpu().numpy().astype(np.float64).copy()
    if not is_right:
        v[:, 0] *= -1.0
    # Direct projection — no rotation applied (upstream's 180° X rotation and
    # pyrender's NDC y-flip cancel; applying only the rotation inverts y).
    v_cam = v + cam_t
    z = v_cam[:, 2]
    proj = focal * v_cam[:, :2] / z[:, None]
    proj[:, 0] += img_w / 2.0
    proj[:, 1] += img_h / 2.0
    return proj, z


def overlay_mesh(
    ax,
    image_uint8: np.ndarray,
    verts_2d: np.ndarray,
    verts_z: np.ndarray,
    faces: np.ndarray,
    color=(0.20, 0.85, 1.00),
    alpha: float = 0.65,
    is_right: bool = True,
) -> None:
    """Rasterize the projected mesh on ``image_uint8`` with face-normal
    diffuse shading + back-to-front depth ordering."""
    from matplotlib.collections import PolyCollection  # noqa: WPS433 (visualisation only)

    h, w = image_uint8.shape[:2]
    ax.imshow(image_uint8)
    ax.set_xlim(0, w)
    ax.set_ylim(h, 0)
    ax.set_aspect("equal")
    ax.axis("off")

    tri_z = verts_z[faces].mean(axis=1)               # (F,) camera z per face
    tris2d = verts_2d[faces]                          # (F, 3, 2)

    # Cull triangles whose mean z is behind the camera or way off-screen.
    inside = (
        (tris2d[..., 0] > -50).all(-1)
        & (tris2d[..., 0] < w + 50).all(-1)
        & (tris2d[..., 1] > -50).all(-1)
        & (tris2d[..., 1] < h + 50).all(-1)
        & (tri_z > 0.05)
    )
    tris2d, tri_z = tris2d[inside], tri_z[inside]

    # Winding: without the 180° X rotation, projecting v directly means
    # vy > 0 maps to larger proj_y (lower in image).  A right-hand
    # front-facing triangle that is CCW in 3D world projects to CW in
    # y-down image (cross_z > 0).  The x-flip for left hands reverses
    # winding, so front-facing left-hand triangles have cross_z < 0.
    e1 = tris2d[:, 1] - tris2d[:, 0]
    e2 = tris2d[:, 2] - tris2d[:, 0]
    cross_z = e1[:, 0] * e2[:, 1] - e1[:, 1] * e2[:, 0]
    winding_sign = 1.0 if is_right else -1.0
    facing = np.clip(winding_sign * cross_z / (np.abs(cross_z).max() + 1e-9), -1, 1)
    diffuse = 0.55 + 0.45 * facing

    # Painter's order: render farthest faces first.
    order = np.argsort(-tri_z)
    tris2d = tris2d[order]
    diffuse = diffuse[order]

    facecolors = np.empty((len(tris2d), 4), dtype=np.float32)
    facecolors[:, 0] = color[0] * diffuse
    facecolors[:, 1] = color[1] * diffuse
    facecolors[:, 2] = color[2] * diffuse
    facecolors[:, 3] = alpha

    coll = PolyCollection(
        tris2d,
        facecolors=facecolors,
        edgecolors=(0.05, 0.10, 0.15, 0.06),
        linewidths=0.25,
    )
    ax.add_collection(coll)