Spaces:
Running on Zero
Running on Zero
Download fab/decoder_model.py from hugging-apps/fab-body-estimation: direct link, hf CLI and curl.
- Browser
- Download file 15.6 kB
-
https://huggingface.co/spaces/hugging-apps/fab-body-estimation/resolve/main/fab/decoder_model.py
- Command line
-
hf download hf://spaces/hugging-apps/fab-body-estimation/fab/decoder_model.py
-
curl -L -o decoder_model.py https://huggingface.co/spaces/hugging-apps/fab-body-estimation/resolve/main/fab/decoder_model.py
15.6 kB
| """FAB-L / FAB-S: SAM 3D Body's promptable decoder and heads on a small student backbone. | |
| The student (a DINOv3 ViT at the teacher's 512 px input, 16 px patches -> the same 32 x 32 token grid) is | |
| projected to the teacher's embedding width and replaces the 840M backbone; everything after the backbone (ray | |
| conditioning, prompt / keypoint tokens, iterative decoder, MHR and camera heads) is SAM 3D Body's own, initialised | |
| from its checkpoint. Trained with token-level feature distillation against the frozen teacher backbone plus the | |
| FAB output / ground-truth losses (train/train_fab.py --arch fabd). | |
| """ | |
| from __future__ import annotations | |
| import copy | |
| from pathlib import Path | |
| import torch | |
| import torch.nn as nn | |
| from .model import BACKBONES | |
| class StudentBackbone(nn.Module): | |
| """timm DINOv3 ViT -> (B, out_dim, H/16, W/16), the teacher backbone's output format.""" | |
| def __init__(self, backbone: str = "dinov3_vitb16", size: int = 512, out_dim: int = 1280, pretrained: bool = True): | |
| super().__init__() | |
| import timm | |
| timm_name, dim = BACKBONES[backbone] | |
| self.vit = timm.create_model(timm_name, pretrained=pretrained, img_size=size, num_classes=0, | |
| dynamic_img_size=False) | |
| self.num_prefix = self.vit.num_prefix_tokens | |
| self.proj = nn.Sequential(nn.LayerNorm(dim), nn.Linear(dim, out_dim)) | |
| def forward(self, x, extra_embed=None): | |
| t = self.proj(self.vit.forward_features(x)[:, self.num_prefix:]) | |
| b, n, c = t.shape | |
| h = int(round(n ** 0.5)) | |
| return t.transpose(1, 2).reshape(b, c, h, n // h) | |
| class FABDecoder(nn.Module): | |
| """forward(img (B, 3, 512, 512) in [0, 1], bbox_center, bbox_scale, ori_size, cam_int, affine) -> SAM 3D Body's | |
| `mhr` output dict (+ unconditioned "backbone_embeddings" for the feature loss).""" | |
| def __init__(self, sam3db, backbone: str = "dinov3_vitb16", size: int = 512, pretrained: bool = True, | |
| dec_layers: int = 0): | |
| super().__init__() | |
| self.body = sam3db | |
| if dec_layers: # SAM 3D Body's 3-layer estimate is within 0.2 mm of its 6-layer one (3DPW PA-MPJPE) | |
| self.body.decoder.layers = self.body.decoder.layers[:dec_layers] | |
| out_dim = self._teacher_dim(sam3db) | |
| self.body.backbone = StudentBackbone(backbone, size, out_dim, pretrained) | |
| self.body.backbone_dtype = torch.float32 # the student runs under autocast | |
| self.size = size | |
| self.g1_head = None | |
| self.kp2d_head = None | |
| self.dense_head = None | |
| self.contact_head = None | |
| self.vis_head = None | |
| self.dense_point_count = None # inference-only prefix; None retains every training query | |
| self.map_outputs = False # inference: + part_map / mask_map (B, 256, 256) uint8 (fab/seg_maps.py) | |
| def add_dense_head(self, surface_idx, residual=False, parts=False, xyz_layers=0, part_depth=0, hr=0, mask_depth=0, | |
| mask_query="box", xyz_dim=256, xyz_mem="img", p25d=0.0, p25d_border=0.02, p25d_vis=False, | |
| ray_in=False, uv_off=0.0): | |
| """Direct dense keypoints (fab/dense_head.py): 70 MHR keypoints, 14 evaluation joints and the SMPL | |
| template vertices `surface_idx`, each with heatmap 2D and regressed root-relative 3D; parts: + body part | |
| segmentation (fab/body_parts.py; part_depth > 0: the d6-style head); xyz_layers: transformer layers | |
| before the 3D regression; hr: the first hr points' 2D from a 2x finer heatmap; xyz_dim / xyz_mem / p25d: the | |
| points decoder (saved heads: dense_head.dense_head_kwargs).""" | |
| from .body_parts import N_PARTS | |
| from .dense_head import BodyDenseHead | |
| self.dense_head = BodyDenseHead(surface_idx, img_dim=self._teacher_dim(self.body), | |
| tok_dim=self.body.head_pose.proj.embed_dims, residual=residual, | |
| parts=N_PARTS if parts else 0, xyz_layers=xyz_layers, part_depth=part_depth, | |
| hr=hr, mask_depth=mask_depth, mask_query=mask_query, xyz_dim=xyz_dim, | |
| xyz_mem=xyz_mem, p25d=p25d, p25d_border=p25d_border, p25d_vis=p25d_vis, | |
| ray_in=ray_in, uv_off=uv_off) | |
| return self.dense_head | |
| def add_kp2d_head(self): | |
| """Independent 2D keypoints: a zero-initialised residual (crop pixels) on the mesh's projected keypoints from | |
| the decoder's final keypoint tokens, supervised by 2D labels; the mesh keeps its own losses (weak agreement).""" | |
| dim = self.body.head_pose.proj.embed_dims | |
| self.kp2d_head = nn.Sequential(nn.LayerNorm(dim), nn.Linear(dim, 256), nn.GELU(), nn.Linear(256, 2)) | |
| nn.init.zeros_(self.kp2d_head[-1].weight) | |
| nn.init.zeros_(self.kp2d_head[-1].bias) | |
| return self.kp2d_head | |
| def add_g1_head(self, hidden: int = 512, joints_input: bool = False): | |
| """In-model retargeting: pose token -> Unitree G1 joint angles (29, URDF order) + root rotation (6D, camera | |
| frame), trained on G1 retargeting targets (the g1_body sidecars, docs/training.md). joints_input: also feed the (detached, pelvis-relative) 3D MHR | |
| keypoints of the mesh path, so the head can learn the deterministic skeleton -> robot mapping.""" | |
| dim = self.body.head_pose.proj.embed_dims + (70 * 3 if joints_input else 0) | |
| self.g1_joints_input = joints_input | |
| self.g1_head = nn.Sequential(nn.LayerNorm(dim), nn.Linear(dim, hidden), nn.GELU(), nn.Linear(hidden, hidden), | |
| nn.GELU(), nn.Linear(hidden, 29 + 6)) | |
| nn.init.zeros_(self.g1_head[-1].weight) | |
| with torch.no_grad(): | |
| self.g1_head[-1].bias.copy_(torch.tensor([0.0] * 29 + [1.0, 0, 0, 0, 1.0, 0])) | |
| return self.g1_head | |
| def add_contact_head(self, attn_dim=0, n_out=8): | |
| """Explicit contact probabilities (fab/contact_head.py) from the pose token and the dense head's features | |
| at the foot / knee / wrist keypoints; needs the dense head.""" | |
| from .contact_head import ContactHead | |
| self.contact_head = ContactHead(self.body.head_pose.proj.embed_dims, 2 * self.dense_head.dim, attn_dim=attn_dim, | |
| img_dim=self._teacher_dim(self.body), n_out=n_out) | |
| self.contact_residual = bool(int(self.dense_head.residual)) # host flag: no device read in the forward (graphs) | |
| return self.contact_head | |
| def add_vis_head(self, attn_dim=128, human=False): | |
| """Per-keypoint visibility (fab/vis_head.py): visible / occluded / outside the crop for the 70 MHR | |
| keypoints and the 14 joints, from the dense head's features at those points; needs the dense head. | |
| human: + the human-annotation classifier (then vis_logits = human, vis_logits_rendered = rendered labels).""" | |
| from .vis_head import VisHead | |
| self.vis_head = VisHead(self.body.head_pose.proj.embed_dims, 2 * self.dense_head.dim, attn_dim=attn_dim, | |
| img_dim=self._teacher_dim(self.body), human=human) | |
| self.vis_residual = bool(int(self.dense_head.residual)) | |
| return self.vis_head | |
| def _teacher_dim(sam3db): | |
| enc = getattr(sam3db.backbone, "encoder", None) | |
| return int(getattr(enc, "embed_dim", 1280)) | |
| def forward(self, img, bbox_center, bbox_scale, ori_size, cam_int, affine=None, return_vertices=True, mask=None, | |
| mask_score=None, mask_box=None): | |
| """mask (B, 1, S, S) / mask_score (B,): optional person-mask prompt in crop coordinates (default: none). | |
| mask_box (B, 4): the prompt box x0, y0, x1, y1 in crop units, read only by a "+extent" person-mask head.""" | |
| b = img.shape[0] | |
| dev = img.device | |
| batch = { | |
| "img": img[:, None], "bbox_center": bbox_center[:, None], "bbox_scale": bbox_scale[:, None], | |
| "ori_img_size": ori_size[:, None].float(), "cam_int": cam_int, "affine_trans": affine[:, None], | |
| "person_valid": torch.ones(b, 1, device=dev), | |
| "mask": torch.zeros(b, 1, 1, self.size, self.size, device=dev) if mask is None else mask[:, None].float(), | |
| "mask_score": torch.zeros(b, 1, device=dev) if mask_score is None else mask_score.reshape(b, 1).float(), | |
| "img_size": torch.full((b, 1, 2), float(self.size), device=dev), | |
| } | |
| self.body._initialize_batch(batch) | |
| self.body.final_vertices = return_vertices | |
| out = self.body.forward_step(batch, decoder_type="body") | |
| mhr = dict(out["mhr"]) | |
| mhr["image_embeddings"] = out["image_embeddings"] | |
| mhr["backbone_embeddings"] = out["backbone_embeddings"] | |
| mhr["token"] = mhr.get("pose_token") | |
| if self.kp2d_head is not None and mhr.get("kp_tokens") is not None and affine is not None: | |
| d_crop = self.kp2d_head(mhr["kp_tokens"].float()).float() * self.size # (B, 70, 2) crop pixels | |
| lin = affine[:, :, :2].float() # full image -> crop (similarity, possibly rolled) | |
| d_full = torch.linalg.solve(lin[:, None].expand(-1, d_crop.shape[1], -1, -1), d_crop[..., None])[..., 0] | |
| mhr["pred_keypoints_2d_ind"] = mhr["pred_keypoints_2d"][..., :2].float() + d_full | |
| if self.dense_head is not None and mhr["token"] is not None: | |
| if self.contact_head is not None: | |
| self.dense_head.keep_feats = self.contact_head.points | |
| if self.vis_head is not None: # the visibility points (0-83) include the contact head's (MHR keypoints) | |
| self.dense_head.keep_feats = self.vis_head.points | |
| kw = {} | |
| if self.dense_head.p25d_mix > 0 or self.dense_head.pt_ray is not None: # 2.5D lift / viewing rays: the crop camera | |
| kw.update(cam_int=cam_int, affine=affine) | |
| if self.dense_head.pt_dec is not None: # point transformer memory + the decoder's tokens | |
| kw["dec_tokens"] = torch.cat([mhr["token"].reshape(b, 1, -1), mhr["kp_tokens"]], 1) | |
| if (self.dense_head.mask_extent or self.dense_head.mask_corners) and self.dense_head.mask_dconv is not None: | |
| kw["mask_box"] = mask_box | |
| uv, xyz, parts, *mask = self.dense_head(out["image_embeddings"], mhr["token"].reshape(b, -1), img, | |
| point_count=self.dense_point_count, **kw) | |
| mhr["dense_uv"], mhr["dense_xyz"] = uv, xyz | |
| if self.dense_head.p25d_mix > 0: | |
| mhr["dense_xyz3d"] = self.dense_head.last_xyz3d # the regressed 3D before the 2.5D lift | |
| mhr["dense_abs_z"] = self.dense_head.last_abs_z # the lift's absolute point depths (m) | |
| if parts is not None: | |
| mhr["part_logits"] = parts | |
| if mask: | |
| mhr["mask_logits"] = mask[0] # (B, 1, S/2, S/2) person mask | |
| if self.map_outputs and (parts is not None or mask): # inference: the crop maps inside the (graphed) forward | |
| from .seg_maps import crop_maps_gpu | |
| pm, mm = crop_maps_gpu(parts, mask[0] if mask else None) | |
| if pm is not None: | |
| mhr["part_map"] = pm | |
| if mm is not None: | |
| mhr["mask_map"] = mm | |
| if self.contact_head is not None: | |
| ix = self.contact_head.points | |
| pts = xyz[:, ix].float() | |
| if self.contact_residual: # offsets on the MHR keypoints (the residual head's anchors) | |
| pts = pts + mhr["pred_keypoints_3d"][:, ix].float().detach() | |
| cf = self.dense_head.last_feats if self.vis_head is None else self.dense_head.last_feats[:, ix] | |
| logits, hid = self.contact_head(mhr["token"].reshape(b, -1).float(), cf.float(), | |
| pts, uv[:, ix].float(), mhr["pred_cam_t"].float(), | |
| out["image_embeddings"] if self.contact_head.attn_dim else None) | |
| mhr["contact_logits"], mhr["contact_feat"], mhr["contact_pts"] = logits, hid, pts | |
| if self.vis_head is not None: | |
| vp = xyz[:, :84].float() | |
| if self.vis_residual: # MHR keypoint offsets on the mesh's keypoints (the joints' offsets ~ 0) | |
| vp = torch.cat([vp[:, :70] + mhr["pred_keypoints_3d"].float().detach(), vp[:, 70:]], 1) | |
| vp = vp - vp[:, 72:74].mean(1, keepdim=True) # root-relative at the J14 hips | |
| mhr["vis_logits"] = self.vis_head(mhr["token"].reshape(b, -1).float(), self.dense_head.last_feats.float(), | |
| vp, uv[:, :84].float(), | |
| out["image_embeddings"] if self.vis_head.attn_dim else None) | |
| if isinstance(mhr["vis_logits"], tuple): # human-semantics head: the primary output (deployed / | |
| # benchmarked as vis_logits); the rendered-label classifier stays as vis_logits_rendered | |
| mhr["vis_logits_rendered"], mhr["vis_logits"] = mhr["vis_logits"] | |
| if self.vis_head is not None and self.dense_head.p25d_mix > 0 and self.dense_head.p25d_vis_gate: | |
| # vis-gated 2.5D lift: each keypoint's lift weighted by P(visible) (detached), so occluded / out-of-frame | |
| # keypoints keep the regressed 3D; points without a visibility output (the surface) keep the lift | |
| P = self.dense_head.last_xyz3d.float() | |
| pv = mhr["vis_logits"].float().softmax(-1)[..., 0].detach() | |
| g = torch.cat([pv, torch.ones_like(P[:, pv.shape[1]:, 0])], 1)[..., None] | |
| mhr["dense_xyz"] = P + g * (xyz.float() - P) | |
| if self.g1_head is not None and mhr["token"] is not None: | |
| x = mhr["token"].float().reshape(b, -1) | |
| if getattr(self, "g1_joints_input", False): | |
| kp = mhr["pred_keypoints_3d"].float().detach() | |
| x = torch.cat([x, (kp - kp[:, [9, 10]].mean(1, keepdim=True)).reshape(b, -1) * 5.0], -1) | |
| g = self.g1_head(x) | |
| mhr["g1_q"], mhr["g1_rot6d"] = g[:, :29], g[:, 29:] | |
| return mhr | |
| def head_pose(self): | |
| return self.body.head_pose | |
| def head_camera(self): | |
| return self.body.head_camera | |
| def build_fab_decoder(teacher: str | Path, backbone: str = "dinov3_vitb16", size: int = 512, | |
| device: str = "cuda", pretrained: bool = True, dec_layers: int = 0, teacher_weights: bool = False): | |
| """-> (student FABDecoder, frozen teacher backbone), built from SAM 3D Body's config and the MHR model in the folder | |
| `teacher` (model_config.yaml, assets/mhr_model.pt). Inference loads every network weight from the FAB checkpoint, so | |
| SAM 3D Body's network weights are not needed; teacher_weights=True also loads <teacher>/model.ckpt (training).""" | |
| from fab import load_sam_3d_body | |
| teacher = Path(teacher) | |
| teacher = teacher.parent if teacher.suffix == ".ckpt" else teacher | |
| sam3db, _ = load_sam_3d_body(checkpoint_path=str(teacher / "model.ckpt") if teacher_weights else "", device=device, | |
| mhr_path=str(teacher / "assets" / "mhr_model.pt"), config_dir=str(teacher)) | |
| teacher_backbone = copy.deepcopy(sam3db.backbone).eval() | |
| for q in teacher_backbone.parameters(): | |
| q.requires_grad_(False) | |
| model = FABDecoder(sam3db, backbone, size, pretrained, dec_layers).to(device) | |
| return model, teacher_backbone | |