Blame for code/hamer/reference.py

# SPDX-FileCopyrightText: © 2026 Tenstorrent Inc.
# SPDX-License-Identifier: Apache-2.0
"""Self-contained torch CPU reference for the HaMeR per-frame regressor used
by HaMeR.

Adapted from ``geopavlakos/hamer`` (Pavlakos et al., ICCV 2023) with upstream
dependencies on ``timm``, ``pytorch_lightning``, ``yacs``, ``smplx`` and
``chumpy`` removed — this file needs only ``torch`` and ``einops`` so it runs
in the Tenstorrent venv without extra packages.  Random weights, deterministic
under ``torch.manual_seed``; used as the accuracy anchor for the tt-nn port.
"""
from __future__ import annotations

from dataclasses import dataclass
from functools import partial
from typing import Optional

import torch
import torch.nn as nn
import torch.nn.functional as F
from einops import rearrange


# --------------------------------------------------------------------------- #
# Config
# --------------------------------------------------------------------------- #
@dataclass
class HamerConfig:
    # Image / patch
    img_size: tuple = (256, 192)
    patch_size: int = 16
    in_chans: int = 3
    # Backbone (ViT-H)
    embed_dim: int = 1280
    depth: int = 32
    num_heads: int = 16
    mlp_ratio: float = 4.0
    qkv_bias: bool = True
    # MANO head
    num_hand_joints: int = 15
    joint_rep_dim: int = 6     # 6-D continuous rotation per joint
    ief_iters: int = 1
    head_dim: int = 1024
    head_depth: int = 6
    head_heads: int = 8
    head_dim_head: int = 64
    head_mlp_dim: int = 1024

    @property
    def npose(self) -> int:
        return self.joint_rep_dim * (self.num_hand_joints + 1)  # 16 × 6 = 96


# --------------------------------------------------------------------------- #
# ViT-H backbone (no timm)
# --------------------------------------------------------------------------- #
class Mlp(nn.Module):
    def __init__(self, dim: int, hidden_dim: int) -> None:
        super().__init__()
        self.fc1 = nn.Linear(dim, hidden_dim)
        self.act = nn.GELU()
        self.fc2 = nn.Linear(hidden_dim, dim)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return self.fc2(self.act(self.fc1(x)))


class Attention(nn.Module):
    def __init__(self, dim: int, num_heads: int, qkv_bias: bool = True) -> None:
        super().__init__()
        self.num_heads = num_heads
        self.head_dim = dim // num_heads
        self.scale = self.head_dim ** -0.5
        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
        self.proj = nn.Linear(dim, dim)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        B, N, C = x.shape
        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
        q, k, v = qkv[0], qkv[1], qkv[2]
        attn = (q * self.scale) @ k.transpose(-2, -1)
        attn = attn.softmax(dim=-1)
        out = (attn @ v).transpose(1, 2).reshape(B, N, C)
        return self.proj(out)


class Block(nn.Module):
    def __init__(self, dim: int, num_heads: int, mlp_ratio: float, qkv_bias: bool) -> None:
        super().__init__()
        norm_layer = partial(nn.LayerNorm, eps=1e-6)
        self.norm1 = norm_layer(dim)
        self.attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias)
        self.norm2 = norm_layer(dim)
        self.mlp = Mlp(dim, int(dim * mlp_ratio))

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = x + self.attn(self.norm1(x))
        x = x + self.mlp(self.norm2(x))
        return x


class PatchEmbed(nn.Module):
    def __init__(self, img_size: tuple, patch_size: int, in_chans: int, embed_dim: int) -> None:
        super().__init__()
        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size, padding=2)
        self.img_h = img_size[0]
        self.img_w = img_size[1]

    def forward(self, x: torch.Tensor):
        x = self.proj(x)
        Hp, Wp = x.shape[2], x.shape[3]
        x = x.flatten(2).transpose(1, 2)
        return x, (Hp, Wp)


class ViT(nn.Module):
    def __init__(self, cfg: HamerConfig) -> None:
        super().__init__()
        self.patch_embed = PatchEmbed(cfg.img_size, cfg.patch_size, cfg.in_chans, cfg.embed_dim)
        num_patches = ((cfg.img_size[0] + 2 * 2 - cfg.patch_size) // cfg.patch_size + 1) * \
                      ((cfg.img_size[1] + 2 * 2 - cfg.patch_size) // cfg.patch_size + 1)
        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, cfg.embed_dim))
        self.blocks = nn.ModuleList([
            Block(cfg.embed_dim, cfg.num_heads, cfg.mlp_ratio, cfg.qkv_bias)
            for _ in range(cfg.depth)
        ])
        self.last_norm = nn.LayerNorm(cfg.embed_dim, eps=1e-6)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        B = x.shape[0]
        x, (Hp, Wp) = self.patch_embed(x)
        x = x + self.pos_embed[:, 1:] + self.pos_embed[:, :1]
        for blk in self.blocks:
            x = blk(x)
        x = self.last_norm(x)
        return x.permute(0, 2, 1).reshape(B, -1, Hp, Wp).contiguous()


# --------------------------------------------------------------------------- #
# MANO transformer decoder head (cross-attn)
# --------------------------------------------------------------------------- #
class _FeedForward(nn.Module):
    def __init__(self, dim: int, hidden_dim: int) -> None:
        super().__init__()
        self.net = nn.Sequential(nn.Linear(dim, hidden_dim), nn.GELU(), nn.Linear(hidden_dim, dim))

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return self.net(x)


class _SelfAttention(nn.Module):
    def __init__(self, dim: int, heads: int, dim_head: int) -> None:
        super().__init__()
        inner = dim_head * heads
        self.heads = heads
        self.scale = dim_head ** -0.5
        self.to_qkv = nn.Linear(dim, inner * 3, bias=False)
        self.to_out = nn.Linear(inner, dim)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        qkv = self.to_qkv(x).chunk(3, dim=-1)
        q, k, v = (rearrange(t, "b n (h d) -> b h n d", h=self.heads) for t in qkv)
        attn = (q @ k.transpose(-1, -2) * self.scale).softmax(dim=-1)
        out = rearrange(attn @ v, "b h n d -> b n (h d)")
        return self.to_out(out)


class _CrossAttention(nn.Module):
    def __init__(self, dim: int, heads: int, dim_head: int, context_dim: Optional[int] = None) -> None:
        super().__init__()
        inner = dim_head * heads
        context_dim = context_dim or dim
        self.heads = heads
        self.scale = dim_head ** -0.5
        self.to_q = nn.Linear(dim, inner, bias=False)
        self.to_kv = nn.Linear(context_dim, inner * 2, bias=False)
        self.to_out = nn.Linear(inner, dim)

    def forward(self, x: torch.Tensor, context: torch.Tensor) -> torch.Tensor:
        k, v = self.to_kv(context).chunk(2, dim=-1)
        q = self.to_q(x)
        q, k, v = (rearrange(t, "b n (h d) -> b h n d", h=self.heads) for t in (q, k, v))
        attn = (q @ k.transpose(-1, -2) * self.scale).softmax(dim=-1)
        out = rearrange(attn @ v, "b h n d -> b n (h d)")
        return self.to_out(out)


class _CrossAttnBlock(nn.Module):
    def __init__(self, dim: int, heads: int, dim_head: int, mlp_dim: int, context_dim: int) -> None:
        super().__init__()
        self.norm_sa = nn.LayerNorm(dim)
        self.sa = _SelfAttention(dim, heads, dim_head)
        self.norm_ca = nn.LayerNorm(dim)
        self.ca = _CrossAttention(dim, heads, dim_head, context_dim=context_dim)
        self.norm_ff = nn.LayerNorm(dim)
        self.ff = _FeedForward(dim, mlp_dim)

    def forward(self, x: torch.Tensor, context: torch.Tensor) -> torch.Tensor:
        x = x + self.sa(self.norm_sa(x))
        x = x + self.ca(self.norm_ca(x), context=context)
        x = x + self.ff(self.norm_ff(x))
        return x


class TransformerDecoder(nn.Module):
    def __init__(self, num_tokens: int, token_dim: int, dim: int, depth: int,
                 heads: int, dim_head: int, mlp_dim: int, context_dim: int) -> None:
        super().__init__()
        self.to_token_embedding = nn.Linear(token_dim, dim)
        self.pos_embedding = nn.Parameter(torch.randn(1, num_tokens, dim) * 0.02)
        self.layers = nn.ModuleList([
            _CrossAttnBlock(dim, heads, dim_head, mlp_dim, context_dim) for _ in range(depth)
        ])

    def forward(self, tokens: torch.Tensor, context: torch.Tensor) -> torch.Tensor:
        x = self.to_token_embedding(tokens) + self.pos_embedding[:, : tokens.shape[1]]
        for blk in self.layers:
            x = blk(x, context)
        return x


def rot6d_to_rotmat(x: torch.Tensor) -> torch.Tensor:
    """Convert 6-D continuous rotation to 3×3 rotation matrix (Zhou et al., CVPR'19)."""
    x = x.reshape(-1, 2, 3).permute(0, 2, 1).contiguous()  # (N, 3, 2): col0=[v0,v1,v2], col1=[v3,v4,v5]
    a1 = x[..., 0]
    a2 = x[..., 1]
    b1 = F.normalize(a1, dim=-1)
    b2 = F.normalize(a2 - (b1 * a2).sum(-1, keepdim=True) * b1, dim=-1)
    b3 = torch.cross(b1, b2, dim=-1)
    return torch.stack((b1, b2, b3), dim=-1)  # (N, 3, 3)


class MANOHead(nn.Module):
    def __init__(self, cfg: HamerConfig) -> None:
        super().__init__()
        self.cfg = cfg
        self.transformer = TransformerDecoder(
            num_tokens=1,
            token_dim=1,
            dim=cfg.head_dim,
            depth=cfg.head_depth,
            heads=cfg.head_heads,
            dim_head=cfg.head_dim_head,
            mlp_dim=cfg.head_mlp_dim,
            context_dim=cfg.embed_dim,
        )
        self.decpose = nn.Linear(cfg.head_dim, cfg.npose)
        self.decshape = nn.Linear(cfg.head_dim, 10)
        self.deccam = nn.Linear(cfg.head_dim, 3)
        # Mean-param buffers (upstream loads from a .npz; zeros give a stable
        # PCC anchor until real checkpoint weights land).
        self.register_buffer("init_hand_pose", torch.zeros(1, cfg.npose))
        self.register_buffer("init_betas", torch.zeros(1, 10))
        self.register_buffer("init_cam", torch.zeros(1, 3))

    def forward(self, features: torch.Tensor) -> torch.Tensor:
        B = features.shape[0]
        context = rearrange(features, "b c h w -> b (h w) c")
        token = torch.zeros(B, 1, 1, device=features.device, dtype=features.dtype)
        pose = self.init_hand_pose.expand(B, -1)
        betas = self.init_betas.expand(B, -1)
        cam = self.init_cam.expand(B, -1)
        for _ in range(self.cfg.ief_iters):
            tok_out = self.transformer(token, context=context).squeeze(1)
            pose = self.decpose(tok_out) + pose
            betas = self.decshape(tok_out) + betas
            cam = self.deccam(tok_out) + cam
        rotmats = rot6d_to_rotmat(pose).view(B, self.cfg.num_hand_joints + 1, 3, 3)
        # Flatten to one tensor for a stable scalar PCC anchor.
        return torch.cat([rotmats.reshape(B, -1), betas, cam], dim=-1)  # (B, 16·9+10+3)=157


class Hamer(nn.Module):
    def __init__(self, cfg: Optional[HamerConfig] = None) -> None:
        super().__init__()
        self.cfg = cfg or HamerConfig()
        self.backbone = ViT(self.cfg)
        self.head = MANOHead(self.cfg)

    def forward(self, image: torch.Tensor) -> torch.Tensor:
        feats = self.backbone(image)
        return self.head(feats)


def build_reference(seed: int = 0) -> Hamer:
    """Random-weight torch CPU reference; deterministic under the given seed."""
    torch.manual_seed(seed)
    model = Hamer()
    model.eval()
    return model


def sample_input(batch: int = 1, seed: int = 0) -> torch.Tensor:
    g = torch.Generator().manual_seed(seed + 1)
    return torch.randn(batch, 3, 256, 192, generator=g)


def build_paired(tt_module, device_id: int = 0, seed: int = 0):
    """Harness hook — returns (ref_model, tt_model, sample_tensor).

    ``tt_module`` is the imported ``models.experimental.hamer.tt.hamer``
    namespace; it must expose ``build_from_reference(ref_model, device_id)``
    returning a callable with the same output contract as ``Hamer.forward``.
    """
    ref = build_reference(seed=seed)
    sample = sample_input(seed=seed)
    tt_model = tt_module.build_from_reference(ref, device_id=device_id)
    return ref, tt_model, sample