fab-body-estimation / fab /decoder_model.py
multimodalart's picture
multimodalart HF Staff
Upload folder using huggingface_hub
83c6c2d verified
Raw History Blame Contribute Delete
15.6 kB
"""FAB-L / FAB-S: SAM 3D Body's promptable decoder and heads on a small student backbone.
The student (a DINOv3 ViT at the teacher's 512 px input, 16 px patches -> the same 32 x 32 token grid) is
projected to the teacher's embedding width and replaces the 840M backbone; everything after the backbone (ray
conditioning, prompt / keypoint tokens, iterative decoder, MHR and camera heads) is SAM 3D Body's own, initialised
from its checkpoint. Trained with token-level feature distillation against the frozen teacher backbone plus the
FAB output / ground-truth losses (train/train_fab.py --arch fabd).
"""
from __future__ import annotations
import copy
from pathlib import Path
import torch
import torch.nn as nn
from .model import BACKBONES
class StudentBackbone(nn.Module):
"""timm DINOv3 ViT -> (B, out_dim, H/16, W/16), the teacher backbone's output format."""
def __init__(self, backbone: str = "dinov3_vitb16", size: int = 512, out_dim: int = 1280, pretrained: bool = True):
super().__init__()
import timm
timm_name, dim = BACKBONES[backbone]
self.vit = timm.create_model(timm_name, pretrained=pretrained, img_size=size, num_classes=0,
dynamic_img_size=False)
self.num_prefix = self.vit.num_prefix_tokens
self.proj = nn.Sequential(nn.LayerNorm(dim), nn.Linear(dim, out_dim))
def forward(self, x, extra_embed=None):
t = self.proj(self.vit.forward_features(x)[:, self.num_prefix:])
b, n, c = t.shape
h = int(round(n ** 0.5))
return t.transpose(1, 2).reshape(b, c, h, n // h)
class FABDecoder(nn.Module):
"""forward(img (B, 3, 512, 512) in [0, 1], bbox_center, bbox_scale, ori_size, cam_int, affine) -> SAM 3D Body's
`mhr` output dict (+ unconditioned "backbone_embeddings" for the feature loss)."""
def __init__(self, sam3db, backbone: str = "dinov3_vitb16", size: int = 512, pretrained: bool = True,
dec_layers: int = 0):
super().__init__()
self.body = sam3db
if dec_layers: # SAM 3D Body's 3-layer estimate is within 0.2 mm of its 6-layer one (3DPW PA-MPJPE)
self.body.decoder.layers = self.body.decoder.layers[:dec_layers]
out_dim = self._teacher_dim(sam3db)
self.body.backbone = StudentBackbone(backbone, size, out_dim, pretrained)
self.body.backbone_dtype = torch.float32 # the student runs under autocast
self.size = size
self.g1_head = None
self.kp2d_head = None
self.dense_head = None
self.contact_head = None
self.vis_head = None
self.dense_point_count = None # inference-only prefix; None retains every training query
self.map_outputs = False # inference: + part_map / mask_map (B, 256, 256) uint8 (fab/seg_maps.py)
def add_dense_head(self, surface_idx, residual=False, parts=False, xyz_layers=0, part_depth=0, hr=0, mask_depth=0,
mask_query="box", xyz_dim=256, xyz_mem="img", p25d=0.0, p25d_border=0.02, p25d_vis=False,
ray_in=False, uv_off=0.0):
"""Direct dense keypoints (fab/dense_head.py): 70 MHR keypoints, 14 evaluation joints and the SMPL
template vertices `surface_idx`, each with heatmap 2D and regressed root-relative 3D; parts: + body part
segmentation (fab/body_parts.py; part_depth > 0: the d6-style head); xyz_layers: transformer layers
before the 3D regression; hr: the first hr points' 2D from a 2x finer heatmap; xyz_dim / xyz_mem / p25d: the
points decoder (saved heads: dense_head.dense_head_kwargs)."""
from .body_parts import N_PARTS
from .dense_head import BodyDenseHead
self.dense_head = BodyDenseHead(surface_idx, img_dim=self._teacher_dim(self.body),
tok_dim=self.body.head_pose.proj.embed_dims, residual=residual,
parts=N_PARTS if parts else 0, xyz_layers=xyz_layers, part_depth=part_depth,
hr=hr, mask_depth=mask_depth, mask_query=mask_query, xyz_dim=xyz_dim,
xyz_mem=xyz_mem, p25d=p25d, p25d_border=p25d_border, p25d_vis=p25d_vis,
ray_in=ray_in, uv_off=uv_off)
return self.dense_head
def add_kp2d_head(self):
"""Independent 2D keypoints: a zero-initialised residual (crop pixels) on the mesh's projected keypoints from
the decoder's final keypoint tokens, supervised by 2D labels; the mesh keeps its own losses (weak agreement)."""
dim = self.body.head_pose.proj.embed_dims
self.kp2d_head = nn.Sequential(nn.LayerNorm(dim), nn.Linear(dim, 256), nn.GELU(), nn.Linear(256, 2))
nn.init.zeros_(self.kp2d_head[-1].weight)
nn.init.zeros_(self.kp2d_head[-1].bias)
return self.kp2d_head
def add_g1_head(self, hidden: int = 512, joints_input: bool = False):
"""In-model retargeting: pose token -> Unitree G1 joint angles (29, URDF order) + root rotation (6D, camera
frame), trained on G1 retargeting targets (the g1_body sidecars, docs/training.md). joints_input: also feed the (detached, pelvis-relative) 3D MHR
keypoints of the mesh path, so the head can learn the deterministic skeleton -> robot mapping."""
dim = self.body.head_pose.proj.embed_dims + (70 * 3 if joints_input else 0)
self.g1_joints_input = joints_input
self.g1_head = nn.Sequential(nn.LayerNorm(dim), nn.Linear(dim, hidden), nn.GELU(), nn.Linear(hidden, hidden),
nn.GELU(), nn.Linear(hidden, 29 + 6))
nn.init.zeros_(self.g1_head[-1].weight)
with torch.no_grad():
self.g1_head[-1].bias.copy_(torch.tensor([0.0] * 29 + [1.0, 0, 0, 0, 1.0, 0]))
return self.g1_head
def add_contact_head(self, attn_dim=0, n_out=8):
"""Explicit contact probabilities (fab/contact_head.py) from the pose token and the dense head's features
at the foot / knee / wrist keypoints; needs the dense head."""
from .contact_head import ContactHead
self.contact_head = ContactHead(self.body.head_pose.proj.embed_dims, 2 * self.dense_head.dim, attn_dim=attn_dim,
img_dim=self._teacher_dim(self.body), n_out=n_out)
self.contact_residual = bool(int(self.dense_head.residual)) # host flag: no device read in the forward (graphs)
return self.contact_head
def add_vis_head(self, attn_dim=128, human=False):
"""Per-keypoint visibility (fab/vis_head.py): visible / occluded / outside the crop for the 70 MHR
keypoints and the 14 joints, from the dense head's features at those points; needs the dense head.
human: + the human-annotation classifier (then vis_logits = human, vis_logits_rendered = rendered labels)."""
from .vis_head import VisHead
self.vis_head = VisHead(self.body.head_pose.proj.embed_dims, 2 * self.dense_head.dim, attn_dim=attn_dim,
img_dim=self._teacher_dim(self.body), human=human)
self.vis_residual = bool(int(self.dense_head.residual))
return self.vis_head
@staticmethod
def _teacher_dim(sam3db):
enc = getattr(sam3db.backbone, "encoder", None)
return int(getattr(enc, "embed_dim", 1280))
def forward(self, img, bbox_center, bbox_scale, ori_size, cam_int, affine=None, return_vertices=True, mask=None,
mask_score=None, mask_box=None):
"""mask (B, 1, S, S) / mask_score (B,): optional person-mask prompt in crop coordinates (default: none).
mask_box (B, 4): the prompt box x0, y0, x1, y1 in crop units, read only by a "+extent" person-mask head."""
b = img.shape[0]
dev = img.device
batch = {
"img": img[:, None], "bbox_center": bbox_center[:, None], "bbox_scale": bbox_scale[:, None],
"ori_img_size": ori_size[:, None].float(), "cam_int": cam_int, "affine_trans": affine[:, None],
"person_valid": torch.ones(b, 1, device=dev),
"mask": torch.zeros(b, 1, 1, self.size, self.size, device=dev) if mask is None else mask[:, None].float(),
"mask_score": torch.zeros(b, 1, device=dev) if mask_score is None else mask_score.reshape(b, 1).float(),
"img_size": torch.full((b, 1, 2), float(self.size), device=dev),
}
self.body._initialize_batch(batch)
self.body.final_vertices = return_vertices
out = self.body.forward_step(batch, decoder_type="body")
mhr = dict(out["mhr"])
mhr["image_embeddings"] = out["image_embeddings"]
mhr["backbone_embeddings"] = out["backbone_embeddings"]
mhr["token"] = mhr.get("pose_token")
if self.kp2d_head is not None and mhr.get("kp_tokens") is not None and affine is not None:
d_crop = self.kp2d_head(mhr["kp_tokens"].float()).float() * self.size # (B, 70, 2) crop pixels
lin = affine[:, :, :2].float() # full image -> crop (similarity, possibly rolled)
d_full = torch.linalg.solve(lin[:, None].expand(-1, d_crop.shape[1], -1, -1), d_crop[..., None])[..., 0]
mhr["pred_keypoints_2d_ind"] = mhr["pred_keypoints_2d"][..., :2].float() + d_full
if self.dense_head is not None and mhr["token"] is not None:
if self.contact_head is not None:
self.dense_head.keep_feats = self.contact_head.points
if self.vis_head is not None: # the visibility points (0-83) include the contact head's (MHR keypoints)
self.dense_head.keep_feats = self.vis_head.points
kw = {}
if self.dense_head.p25d_mix > 0 or self.dense_head.pt_ray is not None: # 2.5D lift / viewing rays: the crop camera
kw.update(cam_int=cam_int, affine=affine)
if self.dense_head.pt_dec is not None: # point transformer memory + the decoder's tokens
kw["dec_tokens"] = torch.cat([mhr["token"].reshape(b, 1, -1), mhr["kp_tokens"]], 1)
if (self.dense_head.mask_extent or self.dense_head.mask_corners) and self.dense_head.mask_dconv is not None:
kw["mask_box"] = mask_box
uv, xyz, parts, *mask = self.dense_head(out["image_embeddings"], mhr["token"].reshape(b, -1), img,
point_count=self.dense_point_count, **kw)
mhr["dense_uv"], mhr["dense_xyz"] = uv, xyz
if self.dense_head.p25d_mix > 0:
mhr["dense_xyz3d"] = self.dense_head.last_xyz3d # the regressed 3D before the 2.5D lift
mhr["dense_abs_z"] = self.dense_head.last_abs_z # the lift's absolute point depths (m)
if parts is not None:
mhr["part_logits"] = parts
if mask:
mhr["mask_logits"] = mask[0] # (B, 1, S/2, S/2) person mask
if self.map_outputs and (parts is not None or mask): # inference: the crop maps inside the (graphed) forward
from .seg_maps import crop_maps_gpu
pm, mm = crop_maps_gpu(parts, mask[0] if mask else None)
if pm is not None:
mhr["part_map"] = pm
if mm is not None:
mhr["mask_map"] = mm
if self.contact_head is not None:
ix = self.contact_head.points
pts = xyz[:, ix].float()
if self.contact_residual: # offsets on the MHR keypoints (the residual head's anchors)
pts = pts + mhr["pred_keypoints_3d"][:, ix].float().detach()
cf = self.dense_head.last_feats if self.vis_head is None else self.dense_head.last_feats[:, ix]
logits, hid = self.contact_head(mhr["token"].reshape(b, -1).float(), cf.float(),
pts, uv[:, ix].float(), mhr["pred_cam_t"].float(),
out["image_embeddings"] if self.contact_head.attn_dim else None)
mhr["contact_logits"], mhr["contact_feat"], mhr["contact_pts"] = logits, hid, pts
if self.vis_head is not None:
vp = xyz[:, :84].float()
if self.vis_residual: # MHR keypoint offsets on the mesh's keypoints (the joints' offsets ~ 0)
vp = torch.cat([vp[:, :70] + mhr["pred_keypoints_3d"].float().detach(), vp[:, 70:]], 1)
vp = vp - vp[:, 72:74].mean(1, keepdim=True) # root-relative at the J14 hips
mhr["vis_logits"] = self.vis_head(mhr["token"].reshape(b, -1).float(), self.dense_head.last_feats.float(),
vp, uv[:, :84].float(),
out["image_embeddings"] if self.vis_head.attn_dim else None)
if isinstance(mhr["vis_logits"], tuple): # human-semantics head: the primary output (deployed /
# benchmarked as vis_logits); the rendered-label classifier stays as vis_logits_rendered
mhr["vis_logits_rendered"], mhr["vis_logits"] = mhr["vis_logits"]
if self.vis_head is not None and self.dense_head.p25d_mix > 0 and self.dense_head.p25d_vis_gate:
# vis-gated 2.5D lift: each keypoint's lift weighted by P(visible) (detached), so occluded / out-of-frame
# keypoints keep the regressed 3D; points without a visibility output (the surface) keep the lift
P = self.dense_head.last_xyz3d.float()
pv = mhr["vis_logits"].float().softmax(-1)[..., 0].detach()
g = torch.cat([pv, torch.ones_like(P[:, pv.shape[1]:, 0])], 1)[..., None]
mhr["dense_xyz"] = P + g * (xyz.float() - P)
if self.g1_head is not None and mhr["token"] is not None:
x = mhr["token"].float().reshape(b, -1)
if getattr(self, "g1_joints_input", False):
kp = mhr["pred_keypoints_3d"].float().detach()
x = torch.cat([x, (kp - kp[:, [9, 10]].mean(1, keepdim=True)).reshape(b, -1) * 5.0], -1)
g = self.g1_head(x)
mhr["g1_q"], mhr["g1_rot6d"] = g[:, :29], g[:, 29:]
return mhr
@property
def head_pose(self):
return self.body.head_pose
@property
def head_camera(self):
return self.body.head_camera
def build_fab_decoder(teacher: str | Path, backbone: str = "dinov3_vitb16", size: int = 512,
device: str = "cuda", pretrained: bool = True, dec_layers: int = 0, teacher_weights: bool = False):
"""-> (student FABDecoder, frozen teacher backbone), built from SAM 3D Body's config and the MHR model in the folder
`teacher` (model_config.yaml, assets/mhr_model.pt). Inference loads every network weight from the FAB checkpoint, so
SAM 3D Body's network weights are not needed; teacher_weights=True also loads <teacher>/model.ckpt (training)."""
from fab import load_sam_3d_body
teacher = Path(teacher)
teacher = teacher.parent if teacher.suffix == ".ckpt" else teacher
sam3db, _ = load_sam_3d_body(checkpoint_path=str(teacher / "model.ckpt") if teacher_weights else "", device=device,
mhr_path=str(teacher / "assets" / "mhr_model.pt"), config_dir=str(teacher))
teacher_backbone = copy.deepcopy(sam3db.backbone).eval()
for q in teacher_backbone.parameters():
q.requires_grad_(False)
model = FABDecoder(sam3db, backbone, size, pretrained, dec_layers).to(device)
return model, teacher_backbone