File size: 15,641 Bytes
83c6c2d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
"""FAB-L / FAB-S: SAM 3D Body's promptable decoder and heads on a small student backbone.

The student (a DINOv3 ViT at the teacher's 512 px input, 16 px patches -> the same 32 x 32 token grid) is
projected to the teacher's embedding width and replaces the 840M backbone; everything after the backbone (ray
conditioning, prompt / keypoint tokens, iterative decoder, MHR and camera heads) is SAM 3D Body's own, initialised
from its checkpoint. Trained with token-level feature distillation against the frozen teacher backbone plus the
FAB output / ground-truth losses (train/train_fab.py --arch fabd).
"""

from __future__ import annotations

import copy
from pathlib import Path

import torch
import torch.nn as nn

from .model import BACKBONES


class StudentBackbone(nn.Module):
    """timm DINOv3 ViT -> (B, out_dim, H/16, W/16), the teacher backbone's output format."""

    def __init__(self, backbone: str = "dinov3_vitb16", size: int = 512, out_dim: int = 1280, pretrained: bool = True):
        super().__init__()
        import timm

        timm_name, dim = BACKBONES[backbone]
        self.vit = timm.create_model(timm_name, pretrained=pretrained, img_size=size, num_classes=0,
                                     dynamic_img_size=False)
        self.num_prefix = self.vit.num_prefix_tokens
        self.proj = nn.Sequential(nn.LayerNorm(dim), nn.Linear(dim, out_dim))

    def forward(self, x, extra_embed=None):
        t = self.proj(self.vit.forward_features(x)[:, self.num_prefix:])
        b, n, c = t.shape
        h = int(round(n ** 0.5))
        return t.transpose(1, 2).reshape(b, c, h, n // h)


class FABDecoder(nn.Module):
    """forward(img (B, 3, 512, 512) in [0, 1], bbox_center, bbox_scale, ori_size, cam_int, affine) -> SAM 3D Body's
    `mhr` output dict (+ unconditioned "backbone_embeddings" for the feature loss)."""

    def __init__(self, sam3db, backbone: str = "dinov3_vitb16", size: int = 512, pretrained: bool = True,
                 dec_layers: int = 0):
        super().__init__()
        self.body = sam3db
        if dec_layers:  # SAM 3D Body's 3-layer estimate is within 0.2 mm of its 6-layer one (3DPW PA-MPJPE)
            self.body.decoder.layers = self.body.decoder.layers[:dec_layers]
        out_dim = self._teacher_dim(sam3db)
        self.body.backbone = StudentBackbone(backbone, size, out_dim, pretrained)
        self.body.backbone_dtype = torch.float32  # the student runs under autocast
        self.size = size
        self.g1_head = None
        self.kp2d_head = None
        self.dense_head = None
        self.contact_head = None
        self.vis_head = None
        self.dense_point_count = None  # inference-only prefix; None retains every training query
        self.map_outputs = False  # inference: + part_map / mask_map (B, 256, 256) uint8 (fab/seg_maps.py)

    def add_dense_head(self, surface_idx, residual=False, parts=False, xyz_layers=0, part_depth=0, hr=0, mask_depth=0,
                       mask_query="box", xyz_dim=256, xyz_mem="img", p25d=0.0, p25d_border=0.02, p25d_vis=False,
                       ray_in=False, uv_off=0.0):
        """Direct dense keypoints (fab/dense_head.py): 70 MHR keypoints, 14 evaluation joints and the SMPL
        template vertices `surface_idx`, each with heatmap 2D and regressed root-relative 3D; parts: + body part
        segmentation (fab/body_parts.py; part_depth > 0: the d6-style head); xyz_layers: transformer layers
        before the 3D regression; hr: the first hr points' 2D from a 2x finer heatmap; xyz_dim / xyz_mem / p25d: the
        points decoder (saved heads: dense_head.dense_head_kwargs)."""
        from .body_parts import N_PARTS
        from .dense_head import BodyDenseHead

        self.dense_head = BodyDenseHead(surface_idx, img_dim=self._teacher_dim(self.body),
                                        tok_dim=self.body.head_pose.proj.embed_dims, residual=residual,
                                        parts=N_PARTS if parts else 0, xyz_layers=xyz_layers, part_depth=part_depth,
                                        hr=hr, mask_depth=mask_depth, mask_query=mask_query, xyz_dim=xyz_dim,
                                        xyz_mem=xyz_mem, p25d=p25d, p25d_border=p25d_border, p25d_vis=p25d_vis,
                                        ray_in=ray_in, uv_off=uv_off)
        return self.dense_head

    def add_kp2d_head(self):
        """Independent 2D keypoints: a zero-initialised residual (crop pixels) on the mesh's projected keypoints from
        the decoder's final keypoint tokens, supervised by 2D labels; the mesh keeps its own losses (weak agreement)."""
        dim = self.body.head_pose.proj.embed_dims
        self.kp2d_head = nn.Sequential(nn.LayerNorm(dim), nn.Linear(dim, 256), nn.GELU(), nn.Linear(256, 2))
        nn.init.zeros_(self.kp2d_head[-1].weight)
        nn.init.zeros_(self.kp2d_head[-1].bias)
        return self.kp2d_head

    def add_g1_head(self, hidden: int = 512, joints_input: bool = False):
        """In-model retargeting: pose token -> Unitree G1 joint angles (29, URDF order) + root rotation (6D, camera
        frame), trained on G1 retargeting targets (the g1_body sidecars, docs/training.md). joints_input: also feed the (detached, pelvis-relative) 3D MHR
        keypoints of the mesh path, so the head can learn the deterministic skeleton -> robot mapping."""
        dim = self.body.head_pose.proj.embed_dims + (70 * 3 if joints_input else 0)
        self.g1_joints_input = joints_input
        self.g1_head = nn.Sequential(nn.LayerNorm(dim), nn.Linear(dim, hidden), nn.GELU(), nn.Linear(hidden, hidden),
                                     nn.GELU(), nn.Linear(hidden, 29 + 6))
        nn.init.zeros_(self.g1_head[-1].weight)
        with torch.no_grad():
            self.g1_head[-1].bias.copy_(torch.tensor([0.0] * 29 + [1.0, 0, 0, 0, 1.0, 0]))
        return self.g1_head

    def add_contact_head(self, attn_dim=0, n_out=8):
        """Explicit contact probabilities (fab/contact_head.py) from the pose token and the dense head's features
        at the foot / knee / wrist keypoints; needs the dense head."""
        from .contact_head import ContactHead

        self.contact_head = ContactHead(self.body.head_pose.proj.embed_dims, 2 * self.dense_head.dim, attn_dim=attn_dim,
                                        img_dim=self._teacher_dim(self.body), n_out=n_out)
        self.contact_residual = bool(int(self.dense_head.residual))  # host flag: no device read in the forward (graphs)
        return self.contact_head

    def add_vis_head(self, attn_dim=128, human=False):
        """Per-keypoint visibility (fab/vis_head.py): visible / occluded / outside the crop for the 70 MHR
        keypoints and the 14 joints, from the dense head's features at those points; needs the dense head.
        human: + the human-annotation classifier (then vis_logits = human, vis_logits_rendered = rendered labels)."""
        from .vis_head import VisHead

        self.vis_head = VisHead(self.body.head_pose.proj.embed_dims, 2 * self.dense_head.dim, attn_dim=attn_dim,
                                img_dim=self._teacher_dim(self.body), human=human)
        self.vis_residual = bool(int(self.dense_head.residual))
        return self.vis_head

    @staticmethod
    def _teacher_dim(sam3db):
        enc = getattr(sam3db.backbone, "encoder", None)
        return int(getattr(enc, "embed_dim", 1280))

    def forward(self, img, bbox_center, bbox_scale, ori_size, cam_int, affine=None, return_vertices=True, mask=None,
                mask_score=None, mask_box=None):
        """mask (B, 1, S, S) / mask_score (B,): optional person-mask prompt in crop coordinates (default: none).
        mask_box (B, 4): the prompt box x0, y0, x1, y1 in crop units, read only by a "+extent" person-mask head."""
        b = img.shape[0]
        dev = img.device
        batch = {
            "img": img[:, None], "bbox_center": bbox_center[:, None], "bbox_scale": bbox_scale[:, None],
            "ori_img_size": ori_size[:, None].float(), "cam_int": cam_int, "affine_trans": affine[:, None],
            "person_valid": torch.ones(b, 1, device=dev),
            "mask": torch.zeros(b, 1, 1, self.size, self.size, device=dev) if mask is None else mask[:, None].float(),
            "mask_score": torch.zeros(b, 1, device=dev) if mask_score is None else mask_score.reshape(b, 1).float(),
            "img_size": torch.full((b, 1, 2), float(self.size), device=dev),
        }
        self.body._initialize_batch(batch)
        self.body.final_vertices = return_vertices
        out = self.body.forward_step(batch, decoder_type="body")
        mhr = dict(out["mhr"])
        mhr["image_embeddings"] = out["image_embeddings"]
        mhr["backbone_embeddings"] = out["backbone_embeddings"]
        mhr["token"] = mhr.get("pose_token")
        if self.kp2d_head is not None and mhr.get("kp_tokens") is not None and affine is not None:
            d_crop = self.kp2d_head(mhr["kp_tokens"].float()).float() * self.size  # (B, 70, 2) crop pixels
            lin = affine[:, :, :2].float()  # full image -> crop (similarity, possibly rolled)
            d_full = torch.linalg.solve(lin[:, None].expand(-1, d_crop.shape[1], -1, -1), d_crop[..., None])[..., 0]
            mhr["pred_keypoints_2d_ind"] = mhr["pred_keypoints_2d"][..., :2].float() + d_full
        if self.dense_head is not None and mhr["token"] is not None:
            if self.contact_head is not None:
                self.dense_head.keep_feats = self.contact_head.points
            if self.vis_head is not None:  # the visibility points (0-83) include the contact head's (MHR keypoints)
                self.dense_head.keep_feats = self.vis_head.points
            kw = {}
            if self.dense_head.p25d_mix > 0 or self.dense_head.pt_ray is not None:  # 2.5D lift / viewing rays: the crop camera
                kw.update(cam_int=cam_int, affine=affine)
            if self.dense_head.pt_dec is not None:  # point transformer memory + the decoder's tokens
                kw["dec_tokens"] = torch.cat([mhr["token"].reshape(b, 1, -1), mhr["kp_tokens"]], 1)
            if (self.dense_head.mask_extent or self.dense_head.mask_corners) and self.dense_head.mask_dconv is not None:
                kw["mask_box"] = mask_box
            uv, xyz, parts, *mask = self.dense_head(out["image_embeddings"], mhr["token"].reshape(b, -1), img,
                                                    point_count=self.dense_point_count, **kw)
            mhr["dense_uv"], mhr["dense_xyz"] = uv, xyz
            if self.dense_head.p25d_mix > 0:
                mhr["dense_xyz3d"] = self.dense_head.last_xyz3d  # the regressed 3D before the 2.5D lift
                mhr["dense_abs_z"] = self.dense_head.last_abs_z  # the lift's absolute point depths (m)
            if parts is not None:
                mhr["part_logits"] = parts
            if mask:
                mhr["mask_logits"] = mask[0]  # (B, 1, S/2, S/2) person mask
            if self.map_outputs and (parts is not None or mask):  # inference: the crop maps inside the (graphed) forward
                from .seg_maps import crop_maps_gpu

                pm, mm = crop_maps_gpu(parts, mask[0] if mask else None)
                if pm is not None:
                    mhr["part_map"] = pm
                if mm is not None:
                    mhr["mask_map"] = mm
            if self.contact_head is not None:
                ix = self.contact_head.points
                pts = xyz[:, ix].float()
                if self.contact_residual:  # offsets on the MHR keypoints (the residual head's anchors)
                    pts = pts + mhr["pred_keypoints_3d"][:, ix].float().detach()
                cf = self.dense_head.last_feats if self.vis_head is None else self.dense_head.last_feats[:, ix]
                logits, hid = self.contact_head(mhr["token"].reshape(b, -1).float(), cf.float(),
                                                pts, uv[:, ix].float(), mhr["pred_cam_t"].float(),
                                                out["image_embeddings"] if self.contact_head.attn_dim else None)
                mhr["contact_logits"], mhr["contact_feat"], mhr["contact_pts"] = logits, hid, pts
            if self.vis_head is not None:
                vp = xyz[:, :84].float()
                if self.vis_residual:  # MHR keypoint offsets on the mesh's keypoints (the joints' offsets ~ 0)
                    vp = torch.cat([vp[:, :70] + mhr["pred_keypoints_3d"].float().detach(), vp[:, 70:]], 1)
                vp = vp - vp[:, 72:74].mean(1, keepdim=True)  # root-relative at the J14 hips
                mhr["vis_logits"] = self.vis_head(mhr["token"].reshape(b, -1).float(), self.dense_head.last_feats.float(),
                                                  vp, uv[:, :84].float(),
                                                  out["image_embeddings"] if self.vis_head.attn_dim else None)
                if isinstance(mhr["vis_logits"], tuple):  # human-semantics head: the primary output (deployed /
                    # benchmarked as vis_logits); the rendered-label classifier stays as vis_logits_rendered
                    mhr["vis_logits_rendered"], mhr["vis_logits"] = mhr["vis_logits"]
            if self.vis_head is not None and self.dense_head.p25d_mix > 0 and self.dense_head.p25d_vis_gate:
                # vis-gated 2.5D lift: each keypoint's lift weighted by P(visible) (detached), so occluded / out-of-frame
                # keypoints keep the regressed 3D; points without a visibility output (the surface) keep the lift
                P = self.dense_head.last_xyz3d.float()
                pv = mhr["vis_logits"].float().softmax(-1)[..., 0].detach()
                g = torch.cat([pv, torch.ones_like(P[:, pv.shape[1]:, 0])], 1)[..., None]
                mhr["dense_xyz"] = P + g * (xyz.float() - P)
        if self.g1_head is not None and mhr["token"] is not None:
            x = mhr["token"].float().reshape(b, -1)
            if getattr(self, "g1_joints_input", False):
                kp = mhr["pred_keypoints_3d"].float().detach()
                x = torch.cat([x, (kp - kp[:, [9, 10]].mean(1, keepdim=True)).reshape(b, -1) * 5.0], -1)
            g = self.g1_head(x)
            mhr["g1_q"], mhr["g1_rot6d"] = g[:, :29], g[:, 29:]
        return mhr

    @property
    def head_pose(self):
        return self.body.head_pose

    @property
    def head_camera(self):
        return self.body.head_camera


def build_fab_decoder(teacher: str | Path, backbone: str = "dinov3_vitb16", size: int = 512,
                      device: str = "cuda", pretrained: bool = True, dec_layers: int = 0, teacher_weights: bool = False):
    """-> (student FABDecoder, frozen teacher backbone), built from SAM 3D Body's config and the MHR model in the folder
    `teacher` (model_config.yaml, assets/mhr_model.pt). Inference loads every network weight from the FAB checkpoint, so
    SAM 3D Body's network weights are not needed; teacher_weights=True also loads <teacher>/model.ckpt (training)."""
    from fab import load_sam_3d_body

    teacher = Path(teacher)
    teacher = teacher.parent if teacher.suffix == ".ckpt" else teacher
    sam3db, _ = load_sam_3d_body(checkpoint_path=str(teacher / "model.ckpt") if teacher_weights else "", device=device,
                                 mhr_path=str(teacher / "assets" / "mhr_model.pt"), config_dir=str(teacher))
    teacher_backbone = copy.deepcopy(sam3db.backbone).eval()
    for q in teacher_backbone.parameters():
        q.requires_grad_(False)
    model = FABDecoder(sam3db, backbone, size, pretrained, dec_layers).to(device)
    return model, teacher_backbone