Add files using upload-large-folder tool

migrator/checkpoints/hub/facebookresearch_dinov2_main/dinov2/hub/cell_dino/backbones.py

@@ -0,0 +1,182 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the CC-by-NC licence,
+# found in the LICENSE_CELL_DINO_CODE file in the root directory of this source tree.
+
+from enum import Enum
+from typing import Optional, Union
+
+import torch
+
+
+class Weights(Enum):
+    CELL_DINO = "CELL-DINO"
+
+
+def _make_cell_dino_model(
+    *,
+    arch_name: str = "vit_large",
+    img_size: int = 518,
+    patch_size: int = 14,
+    init_values: float = 1.0,
+    ffn_layer: str = "mlp",
+    block_chunks: int = 0,
+    num_register_tokens: int = 0,
+    interpolate_antialias: bool = False,
+    interpolate_offset: float = 0.1,
+    pretrained: bool = True,
+    channel_adaptive: bool = False,
+    weights: Union[Weights, str] = Weights.CELL_DINO,
+    pretrained_url: Optional[str] = None,
+    pretrained_path: Optional[str] = None,
+    **kwargs,
+):
+    from ...models import vision_transformer as vits
+
+    if isinstance(weights, str):
+        try:
+            weights = Weights[weights]
+        except KeyError:
+            raise AssertionError(f"Unsupported weights: {weights}")
+
+    vit_kwargs = dict(
+        img_size=img_size,
+        patch_size=patch_size,
+        init_values=init_values,
+        ffn_layer=ffn_layer,
+        block_chunks=block_chunks,
+        num_register_tokens=num_register_tokens,
+        interpolate_antialias=interpolate_antialias,
+        interpolate_offset=interpolate_offset,
+        channel_adaptive=channel_adaptive,
+    )
+    vit_kwargs.update(**kwargs)
+    model = vits.__dict__[arch_name](**vit_kwargs)
+
+    if pretrained:
+        if pretrained_path is not None:
+            state_dict = torch.load(pretrained_path, map_location="cpu")
+        else:
+            pretrained_url is not None
+            state_dict = torch.hub.load_state_dict_from_url(pretrained_url, map_location="cpu")
+        model.load_state_dict(state_dict, strict=True)
+
+    return model
+
+
+def cell_dino_hpa_vitl16(
+    *,
+    pretrained_url: Optional[str] = None,
+    pretrained_path: Optional[str] = None,
+    pretrained: bool = True,
+    weights: Union[Weights, str] = Weights.CELL_DINO,
+    in_channels: int = 4,
+    **kwargs,
+):
+    """
+    Cell-DINO ViT-L/16 model dataset pretrained on HPA dataset.
+    """
+    return _make_cell_dino_model(
+        arch_name="vit_large",
+        patch_size=16,
+        img_size=224,
+        num_register_tokens=0,
+        interpolate_antialias=False,
+        interpolate_offset=0.1,
+        block_chunks=4,
+        pretrained_url=pretrained_url,
+        pretrained_path=pretrained_path,
+        pretrained=pretrained,
+        weights=weights,
+        in_chans=in_channels,
+        **kwargs,
+    )
+
+
+def cell_dino_hpa_vitl14(
+    *,
+    pretrained_url: Optional[str] = None,
+    pretrained_path: Optional[str] = None,
+    pretrained: bool = True,
+    weights: Union[Weights, str] = Weights.CELL_DINO,
+    in_channels: int = 4,
+    **kwargs,
+):
+    """
+    Cell-DINO ViT-L/14 model dataset pretrained on LVD, then on HPA dataset.
+    """
+    return _make_cell_dino_model(
+        arch_name="vit_large",
+        patch_size=14,
+        img_size=518,
+        num_register_tokens=0,
+        interpolate_antialias=False,
+        interpolate_offset=0.1,
+        block_chunks=4,
+        pretrained_url=pretrained_url,
+        pretrained_path=pretrained_path,
+        pretrained=pretrained,
+        weights=weights,
+        in_chans=in_channels,
+        **kwargs,
+    )
+
+
+def cell_dino_cp_vits8(
+    *,
+    pretrained_url: Optional[str] = None,
+    pretrained_path: Optional[str] = None,
+    pretrained: bool = True,
+    weights: Union[Weights, str] = Weights.CELL_DINO,
+    in_channels: int = 5,
+    **kwargs,
+):
+    """
+    Cell-DINO ViT-S/8 model dataset pretrained on the combined cell painting dataset.
+    """
+    return _make_cell_dino_model(
+        arch_name="vit_small",
+        patch_size=8,
+        img_size=128,
+        num_register_tokens=0,
+        interpolate_antialias=False,
+        interpolate_offset=0.1,
+        block_chunks=4,
+        pretrained_url=pretrained_url,
+        pretrained_path=pretrained_path,
+        pretrained=pretrained,
+        weights=weights,
+        in_chans=in_channels,
+        **kwargs,
+    )
+
+
+def channel_adaptive_dino_vitl16(
+    *,
+    pretrained_url: Optional[str] = None,
+    pretrained_path: Optional[str] = None,
+    pretrained: bool = True,
+    weights: Union[Weights, str] = Weights.CELL_DINO,
+    in_channels: int = 1,
+    channel_adaptive: bool = True,
+    **kwargs,
+):
+    """
+    Cell-DINO ViT-L/16 model dataset pretrained on HPA dataset.
+    """
+    return _make_cell_dino_model(
+        arch_name="vit_large",
+        patch_size=16,
+        img_size=224,
+        num_register_tokens=0,
+        interpolate_antialias=False,
+        interpolate_offset=0.1,
+        block_chunks=4,
+        pretrained_url=pretrained_url,
+        pretrained_path=pretrained_path,
+        pretrained=pretrained,
+        weights=weights,
+        in_chans=in_channels,
+        channel_adaptive=channel_adaptive,
+        **kwargs,
+    )

migrator/checkpoints/hub/facebookresearch_dinov2_main/dinov2/hub/depth/__init__.py

@@ -0,0 +1,7 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+from .decode_heads import BNHead, DPTHead
+from .encoder_decoder import DepthEncoderDecoder

migrator/checkpoints/hub/facebookresearch_dinov2_main/dinov2/hub/depth/decode_heads.py

@@ -0,0 +1,747 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+import copy
+from functools import partial
+import math
+import warnings
+
+import torch
+import torch.nn as nn
+
+from .ops import resize
+
+
+# XXX: (Untested) replacement for mmcv.imdenormalize()
+def _imdenormalize(img, mean, std, to_bgr=True):
+    import numpy as np
+
+    mean = mean.reshape(1, -1).astype(np.float64)
+    std = std.reshape(1, -1).astype(np.float64)
+    img = (img * std) + mean
+    if to_bgr:
+        img = img[::-1]
+    return img
+
+
+class DepthBaseDecodeHead(nn.Module):
+    """Base class for BaseDecodeHead.
+
+    Args:
+        in_channels (List): Input channels.
+        channels (int): Channels after modules, before conv_depth.
+        conv_layer (nn.Module): Conv layers. Default: None.
+        act_layer (nn.Module): Activation layers. Default: nn.ReLU.
+        loss_decode (dict): Config of decode loss.
+            Default: ().
+        sampler (dict|None): The config of depth map sampler.
+            Default: None.
+        align_corners (bool): align_corners argument of F.interpolate.
+            Default: False.
+        min_depth (int): Min depth in dataset setting.
+            Default: 1e-3.
+        max_depth (int): Max depth in dataset setting.
+            Default: None.
+        norm_layer (dict|None): Norm layers.
+            Default: None.
+        classify (bool): Whether predict depth in a cls.-reg. manner.
+            Default: False.
+        n_bins (int): The number of bins used in cls. step.
+            Default: 256.
+        bins_strategy (str): The discrete strategy used in cls. step.
+            Default: 'UD'.
+        norm_strategy (str): The norm strategy on cls. probability
+            distribution. Default: 'linear'
+        scale_up (str): Whether predict depth in a scale-up manner.
+            Default: False.
+    """
+
+    def __init__(
+        self,
+        in_channels,
+        conv_layer=None,
+        act_layer=nn.ReLU,
+        channels=96,
+        loss_decode=(),
+        sampler=None,
+        align_corners=False,
+        min_depth=1e-3,
+        max_depth=None,
+        norm_layer=None,
+        classify=False,
+        n_bins=256,
+        bins_strategy="UD",
+        norm_strategy="linear",
+        scale_up=False,
+    ):
+        super(DepthBaseDecodeHead, self).__init__()
+
+        self.in_channels = in_channels
+        self.channels = channels
+        self.conf_layer = conv_layer
+        self.act_layer = act_layer
+        self.loss_decode = loss_decode
+        self.align_corners = align_corners
+        self.min_depth = min_depth
+        self.max_depth = max_depth
+        self.norm_layer = norm_layer
+        self.classify = classify
+        self.n_bins = n_bins
+        self.scale_up = scale_up
+
+        if self.classify:
+            assert bins_strategy in ["UD", "SID"], "Support bins_strategy: UD, SID"
+            assert norm_strategy in ["linear", "softmax", "sigmoid"], "Support norm_strategy: linear, softmax, sigmoid"
+
+            self.bins_strategy = bins_strategy
+            self.norm_strategy = norm_strategy
+            self.softmax = nn.Softmax(dim=1)
+            self.conv_depth = nn.Conv2d(channels, n_bins, kernel_size=3, padding=1, stride=1)
+        else:
+            self.conv_depth = nn.Conv2d(channels, 1, kernel_size=3, padding=1, stride=1)
+
+        self.relu = nn.ReLU()
+        self.sigmoid = nn.Sigmoid()
+
+    def forward(self, inputs, img_metas):
+        """Placeholder of forward function."""
+        pass
+
+    def forward_train(self, img, inputs, img_metas, depth_gt):
+        """Forward function for training.
+        Args:
+            inputs (list[Tensor]): List of multi-level img features.
+            img_metas (list[dict]): List of image info dict where each dict
+                has: 'img_shape', 'scale_factor', 'flip', and may also contain
+                'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'.
+                For details on the values of these keys see
+                `depth/datasets/pipelines/formatting.py:Collect`.
+            depth_gt (Tensor): GT depth
+
+        Returns:
+            dict[str, Tensor]: a dictionary of loss components
+        """
+        depth_pred = self.forward(inputs, img_metas)
+        losses = self.losses(depth_pred, depth_gt)
+
+        log_imgs = self.log_images(img[0], depth_pred[0], depth_gt[0], img_metas[0])
+        losses.update(**log_imgs)
+
+        return losses
+
+    def forward_test(self, inputs, img_metas):
+        """Forward function for testing.
+        Args:
+            inputs (list[Tensor]): List of multi-level img features.
+            img_metas (list[dict]): List of image info dict where each dict
+                has: 'img_shape', 'scale_factor', 'flip', and may also contain
+                'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'.
+                For details on the values of these keys see
+                `depth/datasets/pipelines/formatting.py:Collect`.
+
+        Returns:
+            Tensor: Output depth map.
+        """
+        return self.forward(inputs, img_metas)
+
+    def depth_pred(self, feat):
+        """Prediction each pixel."""
+        if self.classify:
+            logit = self.conv_depth(feat)
+
+            if self.bins_strategy == "UD":
+                bins = torch.linspace(self.min_depth, self.max_depth, self.n_bins, device=feat.device)
+            elif self.bins_strategy == "SID":
+                bins = torch.logspace(self.min_depth, self.max_depth, self.n_bins, device=feat.device)
+
+            # following Adabins, default linear
+            if self.norm_strategy == "linear":
+                logit = torch.relu(logit)
+                eps = 0.1
+                logit = logit + eps
+                logit = logit / logit.sum(dim=1, keepdim=True)
+            elif self.norm_strategy == "softmax":
+                logit = torch.softmax(logit, dim=1)
+            elif self.norm_strategy == "sigmoid":
+                logit = torch.sigmoid(logit)
+                logit = logit / logit.sum(dim=1, keepdim=True)
+
+            output = torch.einsum("ikmn,k->imn", [logit, bins]).unsqueeze(dim=1)
+
+        else:
+            if self.scale_up:
+                output = self.sigmoid(self.conv_depth(feat)) * self.max_depth
+            else:
+                output = self.relu(self.conv_depth(feat)) + self.min_depth
+        return output
+
+    def losses(self, depth_pred, depth_gt):
+        """Compute depth loss."""
+        loss = dict()
+        depth_pred = resize(
+            input=depth_pred, size=depth_gt.shape[2:], mode="bilinear", align_corners=self.align_corners, warning=False
+        )
+        if not isinstance(self.loss_decode, nn.ModuleList):
+            losses_decode = [self.loss_decode]
+        else:
+            losses_decode = self.loss_decode
+        for loss_decode in losses_decode:
+            if loss_decode.loss_name not in loss:
+                loss[loss_decode.loss_name] = loss_decode(depth_pred, depth_gt)
+            else:
+                loss[loss_decode.loss_name] += loss_decode(depth_pred, depth_gt)
+        return loss
+
+    def log_images(self, img_path, depth_pred, depth_gt, img_meta):
+        import numpy as np
+
+        show_img = copy.deepcopy(img_path.detach().cpu().permute(1, 2, 0))
+        show_img = show_img.numpy().astype(np.float32)
+        show_img = _imdenormalize(
+            show_img,
+            img_meta["img_norm_cfg"]["mean"],
+            img_meta["img_norm_cfg"]["std"],
+            img_meta["img_norm_cfg"]["to_rgb"],
+        )
+        show_img = np.clip(show_img, 0, 255)
+        show_img = show_img.astype(np.uint8)
+        show_img = show_img[:, :, ::-1]
+        show_img = show_img.transpose(0, 2, 1)
+        show_img = show_img.transpose(1, 0, 2)
+
+        depth_pred = depth_pred / torch.max(depth_pred)
+        depth_gt = depth_gt / torch.max(depth_gt)
+
+        depth_pred_color = copy.deepcopy(depth_pred.detach().cpu())
+        depth_gt_color = copy.deepcopy(depth_gt.detach().cpu())
+
+        return {"img_rgb": show_img, "img_depth_pred": depth_pred_color, "img_depth_gt": depth_gt_color}
+
+
+class BNHead(DepthBaseDecodeHead):
+    """Just a batchnorm."""
+
+    def __init__(self, input_transform="resize_concat", in_index=(0, 1, 2, 3), upsample=1, **kwargs):
+        super().__init__(**kwargs)
+        self.input_transform = input_transform
+        self.in_index = in_index
+        self.upsample = upsample
+        # self.bn = nn.SyncBatchNorm(self.in_channels)
+        if self.classify:
+            self.conv_depth = nn.Conv2d(self.channels, self.n_bins, kernel_size=1, padding=0, stride=1)
+        else:
+            self.conv_depth = nn.Conv2d(self.channels, 1, kernel_size=1, padding=0, stride=1)
+
+    def _transform_inputs(self, inputs):
+        """Transform inputs for decoder.
+        Args:
+            inputs (list[Tensor]): List of multi-level img features.
+        Returns:
+            Tensor: The transformed inputs
+        """
+
+        if "concat" in self.input_transform:
+            inputs = [inputs[i] for i in self.in_index]
+            if "resize" in self.input_transform:
+                inputs = [
+                    resize(
+                        input=x,
+                        size=[s * self.upsample for s in inputs[0].shape[2:]],
+                        mode="bilinear",
+                        align_corners=self.align_corners,
+                    )
+                    for x in inputs
+                ]
+            inputs = torch.cat(inputs, dim=1)
+        elif self.input_transform == "multiple_select":
+            inputs = [inputs[i] for i in self.in_index]
+        else:
+            inputs = inputs[self.in_index]
+
+        return inputs
+
+    def _forward_feature(self, inputs, img_metas=None, **kwargs):
+        """Forward function for feature maps before classifying each pixel with
+        ``self.cls_seg`` fc.
+        Args:
+            inputs (list[Tensor]): List of multi-level img features.
+        Returns:
+            feats (Tensor): A tensor of shape (batch_size, self.channels,
+                H, W) which is feature map for last layer of decoder head.
+        """
+        # accept lists (for cls token)
+        inputs = list(inputs)
+        for i, x in enumerate(inputs):
+            if len(x) == 2:
+                x, cls_token = x[0], x[1]
+                if len(x.shape) == 2:
+                    x = x[:, :, None, None]
+                cls_token = cls_token[:, :, None, None].expand_as(x)
+                inputs[i] = torch.cat((x, cls_token), 1)
+            else:
+                x = x[0]
+                if len(x.shape) == 2:
+                    x = x[:, :, None, None]
+                inputs[i] = x
+        x = self._transform_inputs(inputs)
+        # feats = self.bn(x)
+        return x
+
+    def forward(self, inputs, img_metas=None, **kwargs):
+        """Forward function."""
+        output = self._forward_feature(inputs, img_metas=img_metas, **kwargs)
+        output = self.depth_pred(output)
+        return output
+
+
+class ConvModule(nn.Module):
+    """A conv block that bundles conv/norm/activation layers.
+
+    This block simplifies the usage of convolution layers, which are commonly
+    used with a norm layer (e.g., BatchNorm) and activation layer (e.g., ReLU).
+    It is based upon three build methods: `build_conv_layer()`,
+    `build_norm_layer()` and `build_activation_layer()`.
+
+    Besides, we add some additional features in this module.
+    1. Automatically set `bias` of the conv layer.
+    2. Spectral norm is supported.
+    3. More padding modes are supported. Before PyTorch 1.5, nn.Conv2d only
+    supports zero and circular padding, and we add "reflect" padding mode.
+
+    Args:
+        in_channels (int): Number of channels in the input feature map.
+            Same as that in ``nn._ConvNd``.
+        out_channels (int): Number of channels produced by the convolution.
+            Same as that in ``nn._ConvNd``.
+        kernel_size (int | tuple[int]): Size of the convolving kernel.
+            Same as that in ``nn._ConvNd``.
+        stride (int | tuple[int]): Stride of the convolution.
+            Same as that in ``nn._ConvNd``.
+        padding (int | tuple[int]): Zero-padding added to both sides of
+            the input. Same as that in ``nn._ConvNd``.
+        dilation (int | tuple[int]): Spacing between kernel elements.
+            Same as that in ``nn._ConvNd``.
+        groups (int): Number of blocked connections from input channels to
+            output channels. Same as that in ``nn._ConvNd``.
+        bias (bool | str): If specified as `auto`, it will be decided by the
+            norm_layer. Bias will be set as True if `norm_layer` is None, otherwise
+            False. Default: "auto".
+        conv_layer (nn.Module): Convolution layer. Default: None,
+            which means using conv2d.
+        norm_layer (nn.Module): Normalization layer. Default: None.
+        act_layer (nn.Module): Activation layer. Default: nn.ReLU.
+        inplace (bool): Whether to use inplace mode for activation.
+            Default: True.
+        with_spectral_norm (bool): Whether use spectral norm in conv module.
+            Default: False.
+        padding_mode (str): If the `padding_mode` has not been supported by
+            current `Conv2d` in PyTorch, we will use our own padding layer
+            instead. Currently, we support ['zeros', 'circular'] with official
+            implementation and ['reflect'] with our own implementation.
+            Default: 'zeros'.
+        order (tuple[str]): The order of conv/norm/activation layers. It is a
+            sequence of "conv", "norm" and "act". Common examples are
+            ("conv", "norm", "act") and ("act", "conv", "norm").
+            Default: ('conv', 'norm', 'act').
+    """
+
+    _abbr_ = "conv_block"
+
+    def __init__(
+        self,
+        in_channels,
+        out_channels,
+        kernel_size,
+        stride=1,
+        padding=0,
+        dilation=1,
+        groups=1,
+        bias="auto",
+        conv_layer=nn.Conv2d,
+        norm_layer=None,
+        act_layer=nn.ReLU,
+        inplace=True,
+        with_spectral_norm=False,
+        padding_mode="zeros",
+        order=("conv", "norm", "act"),
+    ):
+        super(ConvModule, self).__init__()
+        official_padding_mode = ["zeros", "circular"]
+        self.conv_layer = conv_layer
+        self.norm_layer = norm_layer
+        self.act_layer = act_layer
+        self.inplace = inplace
+        self.with_spectral_norm = with_spectral_norm
+        self.with_explicit_padding = padding_mode not in official_padding_mode
+        self.order = order
+        assert isinstance(self.order, tuple) and len(self.order) == 3
+        assert set(order) == set(["conv", "norm", "act"])
+
+        self.with_norm = norm_layer is not None
+        self.with_activation = act_layer is not None
+        # if the conv layer is before a norm layer, bias is unnecessary.
+        if bias == "auto":
+            bias = not self.with_norm
+        self.with_bias = bias
+
+        if self.with_explicit_padding:
+            if padding_mode == "zeros":
+                padding_layer = nn.ZeroPad2d
+            else:
+                raise AssertionError(f"Unsupported padding mode: {padding_mode}")
+            self.pad = padding_layer(padding)
+
+        # reset padding to 0 for conv module
+        conv_padding = 0 if self.with_explicit_padding else padding
+        # build convolution layer
+        self.conv = self.conv_layer(
+            in_channels,
+            out_channels,
+            kernel_size,
+            stride=stride,
+            padding=conv_padding,
+            dilation=dilation,
+            groups=groups,
+            bias=bias,
+        )
+        # export the attributes of self.conv to a higher level for convenience
+        self.in_channels = self.conv.in_channels
+        self.out_channels = self.conv.out_channels
+        self.kernel_size = self.conv.kernel_size
+        self.stride = self.conv.stride
+        self.padding = padding
+        self.dilation = self.conv.dilation
+        self.transposed = self.conv.transposed
+        self.output_padding = self.conv.output_padding
+        self.groups = self.conv.groups
+
+        if self.with_spectral_norm:
+            self.conv = nn.utils.spectral_norm(self.conv)
+
+        # build normalization layers
+        if self.with_norm:
+            # norm layer is after conv layer
+            if order.index("norm") > order.index("conv"):
+                norm_channels = out_channels
+            else:
+                norm_channels = in_channels
+            norm = partial(norm_layer, num_features=norm_channels)
+            self.add_module("norm", norm)
+            if self.with_bias:
+                from torch.nnModules.batchnorm import _BatchNorm
+                from torch.nnModules.instancenorm import _InstanceNorm
+
+                if isinstance(norm, (_BatchNorm, _InstanceNorm)):
+                    warnings.warn("Unnecessary conv bias before batch/instance norm")
+        else:
+            self.norm_name = None
+
+        # build activation layer
+        if self.with_activation:
+            # nn.Tanh has no 'inplace' argument
+            # (nn.Tanh, nn.PReLU, nn.Sigmoid, nn.HSigmoid, nn.Swish, nn.GELU)
+            if not isinstance(act_layer, (nn.Tanh, nn.PReLU, nn.Sigmoid, nn.GELU)):
+                act_layer = partial(act_layer, inplace=inplace)
+            self.activate = act_layer()
+
+        # Use msra init by default
+        self.init_weights()
+
+    @property
+    def norm(self):
+        if self.norm_name:
+            return getattr(self, self.norm_name)
+        else:
+            return None
+
+    def init_weights(self):
+        # 1. It is mainly for customized conv layers with their own
+        #    initialization manners by calling their own ``init_weights()``,
+        #    and we do not want ConvModule to override the initialization.
+        # 2. For customized conv layers without their own initialization
+        #    manners (that is, they don't have their own ``init_weights()``)
+        #    and PyTorch's conv layers, they will be initialized by
+        #    this method with default ``kaiming_init``.
+        # Note: For PyTorch's conv layers, they will be overwritten by our
+        #    initialization implementation using default ``kaiming_init``.
+        if not hasattr(self.conv, "init_weights"):
+            if self.with_activation and isinstance(self.act_layer, nn.LeakyReLU):
+                nonlinearity = "leaky_relu"
+                a = 0.01  # XXX: default negative_slope
+            else:
+                nonlinearity = "relu"
+                a = 0
+            if hasattr(self.conv, "weight") and self.conv.weight is not None:
+                nn.init.kaiming_normal_(self.conv.weight, a=a, mode="fan_out", nonlinearity=nonlinearity)
+            if hasattr(self.conv, "bias") and self.conv.bias is not None:
+                nn.init.constant_(self.conv.bias, 0)
+        if self.with_norm:
+            if hasattr(self.norm, "weight") and self.norm.weight is not None:
+                nn.init.constant_(self.norm.weight, 1)
+            if hasattr(self.norm, "bias") and self.norm.bias is not None:
+                nn.init.constant_(self.norm.bias, 0)
+
+    def forward(self, x, activate=True, norm=True):
+        for layer in self.order:
+            if layer == "conv":
+                if self.with_explicit_padding:
+                    x = self.pad(x)
+                x = self.conv(x)
+            elif layer == "norm" and norm and self.with_norm:
+                x = self.norm(x)
+            elif layer == "act" and activate and self.with_activation:
+                x = self.activate(x)
+        return x
+
+
+class Interpolate(nn.Module):
+    def __init__(self, scale_factor, mode, align_corners=False):
+        super(Interpolate, self).__init__()
+        self.interp = nn.functional.interpolate
+        self.scale_factor = scale_factor
+        self.mode = mode
+        self.align_corners = align_corners
+
+    def forward(self, x):
+        x = self.interp(x, scale_factor=self.scale_factor, mode=self.mode, align_corners=self.align_corners)
+        return x
+
+
+class HeadDepth(nn.Module):
+    def __init__(self, features):
+        super(HeadDepth, self).__init__()
+        self.head = nn.Sequential(
+            nn.Conv2d(features, features // 2, kernel_size=3, stride=1, padding=1),
+            Interpolate(scale_factor=2, mode="bilinear", align_corners=True),
+            nn.Conv2d(features // 2, 32, kernel_size=3, stride=1, padding=1),
+            nn.ReLU(),
+            nn.Conv2d(32, 1, kernel_size=1, stride=1, padding=0),
+        )
+
+    def forward(self, x):
+        x = self.head(x)
+        return x
+
+
+class ReassembleBlocks(nn.Module):
+    """ViTPostProcessBlock, process cls_token in ViT backbone output and
+    rearrange the feature vector to feature map.
+    Args:
+        in_channels (int): ViT feature channels. Default: 768.
+        out_channels (List): output channels of each stage.
+            Default: [96, 192, 384, 768].
+        readout_type (str): Type of readout operation. Default: 'ignore'.
+        patch_size (int): The patch size. Default: 16.
+    """
+
+    def __init__(self, in_channels=768, out_channels=[96, 192, 384, 768], readout_type="ignore", patch_size=16):
+        super(ReassembleBlocks, self).__init__()
+
+        assert readout_type in ["ignore", "add", "project"]
+        self.readout_type = readout_type
+        self.patch_size = patch_size
+
+        self.projects = nn.ModuleList(
+            [
+                ConvModule(
+                    in_channels=in_channels,
+                    out_channels=out_channel,
+                    kernel_size=1,
+                    act_layer=None,
+                )
+                for out_channel in out_channels
+            ]
+        )
+
+        self.resize_layers = nn.ModuleList(
+            [
+                nn.ConvTranspose2d(
+                    in_channels=out_channels[0], out_channels=out_channels[0], kernel_size=4, stride=4, padding=0
+                ),
+                nn.ConvTranspose2d(
+                    in_channels=out_channels[1], out_channels=out_channels[1], kernel_size=2, stride=2, padding=0
+                ),
+                nn.Identity(),
+                nn.Conv2d(
+                    in_channels=out_channels[3], out_channels=out_channels[3], kernel_size=3, stride=2, padding=1
+                ),
+            ]
+        )
+        if self.readout_type == "project":
+            self.readout_projects = nn.ModuleList()
+            for _ in range(len(self.projects)):
+                self.readout_projects.append(nn.Sequential(nn.Linear(2 * in_channels, in_channels), nn.GELU()))
+
+    def forward(self, inputs):
+        assert isinstance(inputs, list)
+        out = []
+        for i, x in enumerate(inputs):
+            assert len(x) == 2
+            x, cls_token = x[0], x[1]
+            feature_shape = x.shape
+            if self.readout_type == "project":
+                x = x.flatten(2).permute((0, 2, 1))
+                readout = cls_token.unsqueeze(1).expand_as(x)
+                x = self.readout_projects[i](torch.cat((x, readout), -1))
+                x = x.permute(0, 2, 1).reshape(feature_shape)
+            elif self.readout_type == "add":
+                x = x.flatten(2) + cls_token.unsqueeze(-1)
+                x = x.reshape(feature_shape)
+            else:
+                pass
+            x = self.projects[i](x)
+            x = self.resize_layers[i](x)
+            out.append(x)
+        return out
+
+
+class PreActResidualConvUnit(nn.Module):
+    """ResidualConvUnit, pre-activate residual unit.
+    Args:
+        in_channels (int): number of channels in the input feature map.
+        act_layer (nn.Module): activation layer.
+        norm_layer (nn.Module): norm layer.
+        stride (int): stride of the first block. Default: 1
+        dilation (int): dilation rate for convs layers. Default: 1.
+    """
+
+    def __init__(self, in_channels, act_layer, norm_layer, stride=1, dilation=1):
+        super(PreActResidualConvUnit, self).__init__()
+
+        self.conv1 = ConvModule(
+            in_channels,
+            in_channels,
+            3,
+            stride=stride,
+            padding=dilation,
+            dilation=dilation,
+            norm_layer=norm_layer,
+            act_layer=act_layer,
+            bias=False,
+            order=("act", "conv", "norm"),
+        )
+
+        self.conv2 = ConvModule(
+            in_channels,
+            in_channels,
+            3,
+            padding=1,
+            norm_layer=norm_layer,
+            act_layer=act_layer,
+            bias=False,
+            order=("act", "conv", "norm"),
+        )
+
+    def forward(self, inputs):
+        inputs_ = inputs.clone()
+        x = self.conv1(inputs)
+        x = self.conv2(x)
+        return x + inputs_
+
+
+class FeatureFusionBlock(nn.Module):
+    """FeatureFusionBlock, merge feature map from different stages.
+    Args:
+        in_channels (int): Input channels.
+        act_layer (nn.Module): activation layer for ResidualConvUnit.
+        norm_layer (nn.Module): normalization layer.
+        expand (bool): Whether expand the channels in post process block.
+            Default: False.
+        align_corners (bool): align_corner setting for bilinear upsample.
+            Default: True.
+    """
+
+    def __init__(self, in_channels, act_layer, norm_layer, expand=False, align_corners=True):
+        super(FeatureFusionBlock, self).__init__()
+
+        self.in_channels = in_channels
+        self.expand = expand
+        self.align_corners = align_corners
+
+        self.out_channels = in_channels
+        if self.expand:
+            self.out_channels = in_channels // 2
+
+        self.project = ConvModule(self.in_channels, self.out_channels, kernel_size=1, act_layer=None, bias=True)
+
+        self.res_conv_unit1 = PreActResidualConvUnit(
+            in_channels=self.in_channels, act_layer=act_layer, norm_layer=norm_layer
+        )
+        self.res_conv_unit2 = PreActResidualConvUnit(
+            in_channels=self.in_channels, act_layer=act_layer, norm_layer=norm_layer
+        )
+
+    def forward(self, *inputs):
+        x = inputs[0]
+        if len(inputs) == 2:
+            if x.shape != inputs[1].shape:
+                res = resize(inputs[1], size=(x.shape[2], x.shape[3]), mode="bilinear", align_corners=False)
+            else:
+                res = inputs[1]
+            x = x + self.res_conv_unit1(res)
+        x = self.res_conv_unit2(x)
+        x = resize(x, scale_factor=2, mode="bilinear", align_corners=self.align_corners)
+        x = self.project(x)
+        return x
+
+
+class DPTHead(DepthBaseDecodeHead):
+    """Vision Transformers for Dense Prediction.
+    This head is implemented of `DPT <https://arxiv.org/abs/2103.13413>`_.
+    Args:
+        embed_dims (int): The embed dimension of the ViT backbone.
+            Default: 768.
+        post_process_channels (List): Out channels of post process conv
+            layers. Default: [96, 192, 384, 768].
+        readout_type (str): Type of readout operation. Default: 'ignore'.
+        patch_size (int): The patch size. Default: 16.
+        expand_channels (bool): Whether expand the channels in post process
+            block. Default: False.
+    """
+
+    def __init__(
+        self,
+        embed_dims=768,
+        post_process_channels=[96, 192, 384, 768],
+        readout_type="ignore",
+        patch_size=16,
+        expand_channels=False,
+        **kwargs,
+    ):
+        super(DPTHead, self).__init__(**kwargs)
+
+        self.in_channels = self.in_channels
+        self.expand_channels = expand_channels
+        self.reassemble_blocks = ReassembleBlocks(embed_dims, post_process_channels, readout_type, patch_size)
+
+        self.post_process_channels = [
+            channel * math.pow(2, i) if expand_channels else channel for i, channel in enumerate(post_process_channels)
+        ]
+        self.convs = nn.ModuleList()
+        for channel in self.post_process_channels:
+            self.convs.append(ConvModule(channel, self.channels, kernel_size=3, padding=1, act_layer=None, bias=False))
+        self.fusion_blocks = nn.ModuleList()
+        for _ in range(len(self.convs)):
+            self.fusion_blocks.append(FeatureFusionBlock(self.channels, self.act_layer, self.norm_layer))
+        self.fusion_blocks[0].res_conv_unit1 = None
+        self.project = ConvModule(self.channels, self.channels, kernel_size=3, padding=1, norm_layer=self.norm_layer)
+        self.num_fusion_blocks = len(self.fusion_blocks)
+        self.num_reassemble_blocks = len(self.reassemble_blocks.resize_layers)
+        self.num_post_process_channels = len(self.post_process_channels)
+        assert self.num_fusion_blocks == self.num_reassemble_blocks
+        assert self.num_reassemble_blocks == self.num_post_process_channels
+        self.conv_depth = HeadDepth(self.channels)
+
+    def forward(self, inputs, img_metas):
+        assert len(inputs) == self.num_reassemble_blocks
+        x = [inp for inp in inputs]
+        x = self.reassemble_blocks(x)
+        x = [self.convs[i](feature) for i, feature in enumerate(x)]
+        out = self.fusion_blocks[0](x[-1])
+        for i in range(1, len(self.fusion_blocks)):
+            out = self.fusion_blocks[i](out, x[-(i + 1)])
+        out = self.project(out)
+        out = self.depth_pred(out)
+        return out

migrator/checkpoints/hub/facebookresearch_dinov2_main/dinov2/hub/depth/encoder_decoder.py

@@ -0,0 +1,351 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+from collections import OrderedDict
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from .ops import resize
+
+
+def add_prefix(inputs, prefix):
+    """Add prefix for dict.
+
+    Args:
+        inputs (dict): The input dict with str keys.
+        prefix (str): The prefix to add.
+
+    Returns:
+
+        dict: The dict with keys updated with ``prefix``.
+    """
+
+    outputs = dict()
+    for name, value in inputs.items():
+        outputs[f"{prefix}.{name}"] = value
+
+    return outputs
+
+
+class DepthEncoderDecoder(nn.Module):
+    """Encoder Decoder depther.
+
+    EncoderDecoder typically consists of backbone and decode_head.
+    """
+
+    def __init__(self, backbone, decode_head):
+        super(DepthEncoderDecoder, self).__init__()
+
+        self.backbone = backbone
+        self.decode_head = decode_head
+        self.align_corners = self.decode_head.align_corners
+
+    def extract_feat(self, img):
+        """Extract features from images."""
+        return self.backbone(img)
+
+    def encode_decode(self, img, img_metas, rescale=True, size=None):
+        """Encode images with backbone and decode into a depth estimation
+        map of the same size as input."""
+        x = self.extract_feat(img)
+        out = self._decode_head_forward_test(x, img_metas)
+        # crop the pred depth to the certain range.
+        out = torch.clamp(out, min=self.decode_head.min_depth, max=self.decode_head.max_depth)
+        if rescale:
+            if size is None:
+                if img_metas is not None:
+                    size = img_metas[0]["ori_shape"][:2]
+                else:
+                    size = img.shape[2:]
+            out = resize(input=out, size=size, mode="bilinear", align_corners=self.align_corners)
+        return out
+
+    def _decode_head_forward_train(self, img, x, img_metas, depth_gt, **kwargs):
+        """Run forward function and calculate loss for decode head in
+        training."""
+        losses = dict()
+        loss_decode = self.decode_head.forward_train(img, x, img_metas, depth_gt, **kwargs)
+        losses.update(add_prefix(loss_decode, "decode"))
+        return losses
+
+    def _decode_head_forward_test(self, x, img_metas):
+        """Run forward function and calculate loss for decode head in
+        inference."""
+        depth_pred = self.decode_head.forward_test(x, img_metas)
+        return depth_pred
+
+    def forward_dummy(self, img):
+        """Dummy forward function."""
+        depth = self.encode_decode(img, None)
+
+        return depth
+
+    def forward_train(self, img, img_metas, depth_gt, **kwargs):
+        """Forward function for training.
+
+        Args:
+            img (Tensor): Input images.
+            img_metas (list[dict]): List of image info dict where each dict
+                has: 'img_shape', 'scale_factor', 'flip', and may also contain
+                'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'.
+                For details on the values of these keys see
+                `depth/datasets/pipelines/formatting.py:Collect`.
+            depth_gt (Tensor): Depth gt
+                used if the architecture supports depth estimation task.
+
+        Returns:
+            dict[str, Tensor]: a dictionary of loss components
+        """
+
+        x = self.extract_feat(img)
+
+        losses = dict()
+
+        # the last of x saves the info from neck
+        loss_decode = self._decode_head_forward_train(img, x, img_metas, depth_gt, **kwargs)
+
+        losses.update(loss_decode)
+
+        return losses
+
+    def whole_inference(self, img, img_meta, rescale, size=None):
+        """Inference with full image."""
+        return self.encode_decode(img, img_meta, rescale, size=size)
+
+    def slide_inference(self, img, img_meta, rescale, stride, crop_size):
+        """Inference by sliding-window with overlap.
+
+        If h_crop > h_img or w_crop > w_img, the small patch will be used to
+        decode without padding.
+        """
+
+        h_stride, w_stride = stride
+        h_crop, w_crop = crop_size
+        batch_size, _, h_img, w_img = img.size()
+        h_grids = max(h_img - h_crop + h_stride - 1, 0) // h_stride + 1
+        w_grids = max(w_img - w_crop + w_stride - 1, 0) // w_stride + 1
+        preds = img.new_zeros((batch_size, 1, h_img, w_img))
+        count_mat = img.new_zeros((batch_size, 1, h_img, w_img))
+        for h_idx in range(h_grids):
+            for w_idx in range(w_grids):
+                y1 = h_idx * h_stride
+                x1 = w_idx * w_stride
+                y2 = min(y1 + h_crop, h_img)
+                x2 = min(x1 + w_crop, w_img)
+                y1 = max(y2 - h_crop, 0)
+                x1 = max(x2 - w_crop, 0)
+                crop_img = img[:, :, y1:y2, x1:x2]
+                depth_pred = self.encode_decode(crop_img, img_meta, rescale)
+                preds += F.pad(depth_pred, (int(x1), int(preds.shape[3] - x2), int(y1), int(preds.shape[2] - y2)))
+
+                count_mat[:, :, y1:y2, x1:x2] += 1
+        assert (count_mat == 0).sum() == 0
+        if torch.onnx.is_in_onnx_export():
+            # cast count_mat to constant while exporting to ONNX
+            count_mat = torch.from_numpy(count_mat.cpu().detach().numpy()).to(device=img.device)
+        preds = preds / count_mat
+        return preds
+
+    def inference(self, img, img_meta, rescale, size=None, mode="whole"):
+        """Inference with slide/whole style.
+
+        Args:
+            img (Tensor): The input image of shape (N, 3, H, W).
+            img_meta (dict): Image info dict where each dict has: 'img_shape',
+                'scale_factor', 'flip', and may also contain
+                'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'.
+                For details on the values of these keys see
+                `depth/datasets/pipelines/formatting.py:Collect`.
+            rescale (bool): Whether rescale back to original shape.
+
+        Returns:
+            Tensor: The output depth map.
+        """
+
+        assert mode in ["slide", "whole"]
+        ori_shape = img_meta[0]["ori_shape"]
+        assert all(_["ori_shape"] == ori_shape for _ in img_meta)
+        if mode == "slide":
+            depth_pred = self.slide_inference(img, img_meta, rescale)
+        else:
+            depth_pred = self.whole_inference(img, img_meta, rescale, size=size)
+        output = depth_pred
+        flip = img_meta[0]["flip"]
+        if flip:
+            flip_direction = img_meta[0]["flip_direction"]
+            assert flip_direction in ["horizontal", "vertical"]
+            if flip_direction == "horizontal":
+                output = output.flip(dims=(3,))
+            elif flip_direction == "vertical":
+                output = output.flip(dims=(2,))
+
+        return output
+
+    def simple_test(self, img, img_meta, rescale=True):
+        """Simple test with single image."""
+        depth_pred = self.inference(img, img_meta, rescale)
+        if torch.onnx.is_in_onnx_export():
+            # our inference backend only support 4D output
+            depth_pred = depth_pred.unsqueeze(0)
+            return depth_pred
+        depth_pred = depth_pred.cpu().numpy()
+        # unravel batch dim
+        depth_pred = list(depth_pred)
+        return depth_pred
+
+    def aug_test(self, imgs, img_metas, rescale=True):
+        """Test with augmentations.
+
+        Only rescale=True is supported.
+        """
+        # aug_test rescale all imgs back to ori_shape for now
+        assert rescale
+        # to save memory, we get augmented depth logit inplace
+        depth_pred = self.inference(imgs[0], img_metas[0], rescale)
+        for i in range(1, len(imgs)):
+            cur_depth_pred = self.inference(imgs[i], img_metas[i], rescale, size=depth_pred.shape[-2:])
+            depth_pred += cur_depth_pred
+        depth_pred /= len(imgs)
+        depth_pred = depth_pred.cpu().numpy()
+        # unravel batch dim
+        depth_pred = list(depth_pred)
+        return depth_pred
+
+    def forward_test(self, imgs, img_metas, **kwargs):
+        """
+        Args:
+            imgs (List[Tensor]): the outer list indicates test-time
+                augmentations and inner Tensor should have a shape NxCxHxW,
+                which contains all images in the batch.
+            img_metas (List[List[dict]]): the outer list indicates test-time
+                augs (multiscale, flip, etc.) and the inner list indicates
+                images in a batch.
+        """
+        for var, name in [(imgs, "imgs"), (img_metas, "img_metas")]:
+            if not isinstance(var, list):
+                raise TypeError(f"{name} must be a list, but got " f"{type(var)}")
+        num_augs = len(imgs)
+        if num_augs != len(img_metas):
+            raise ValueError(f"num of augmentations ({len(imgs)}) != " f"num of image meta ({len(img_metas)})")
+        # all images in the same aug batch all of the same ori_shape and pad
+        # shape
+        for img_meta in img_metas:
+            ori_shapes = [_["ori_shape"] for _ in img_meta]
+            assert all(shape == ori_shapes[0] for shape in ori_shapes)
+            img_shapes = [_["img_shape"] for _ in img_meta]
+            assert all(shape == img_shapes[0] for shape in img_shapes)
+            pad_shapes = [_["pad_shape"] for _ in img_meta]
+            assert all(shape == pad_shapes[0] for shape in pad_shapes)
+
+        if num_augs == 1:
+            return self.simple_test(imgs[0], img_metas[0], **kwargs)
+        else:
+            return self.aug_test(imgs, img_metas, **kwargs)
+
+    def forward(self, img, img_metas, return_loss=True, **kwargs):
+        """Calls either :func:`forward_train` or :func:`forward_test` depending
+        on whether ``return_loss`` is ``True``.
+
+        Note this setting will change the expected inputs. When
+        ``return_loss=True``, img and img_meta are single-nested (i.e. Tensor
+        and List[dict]), and when ``resturn_loss=False``, img and img_meta
+        should be double nested (i.e.  List[Tensor], List[List[dict]]), with
+        the outer list indicating test time augmentations.
+        """
+        if return_loss:
+            return self.forward_train(img, img_metas, **kwargs)
+        else:
+            return self.forward_test(img, img_metas, **kwargs)
+
+    def train_step(self, data_batch, optimizer, **kwargs):
+        """The iteration step during training.
+
+        This method defines an iteration step during training, except for the
+        back propagation and optimizer updating, which are done in an optimizer
+        hook. Note that in some complicated cases or models, the whole process
+        including back propagation and optimizer updating is also defined in
+        this method, such as GAN.
+
+        Args:
+            data (dict): The output of dataloader.
+            optimizer (:obj:`torch.optim.Optimizer` | dict): The optimizer of
+                runner is passed to ``train_step()``. This argument is unused
+                and reserved.
+
+        Returns:
+            dict: It should contain at least 3 keys: ``loss``, ``log_vars``,
+                ``num_samples``.
+                ``loss`` is a tensor for back propagation, which can be a
+                weighted sum of multiple losses.
+                ``log_vars`` contains all the variables to be sent to the
+                logger.
+                ``num_samples`` indicates the batch size (when the model is
+                DDP, it means the batch size on each GPU), which is used for
+                averaging the logs.
+        """
+        losses = self(**data_batch)
+
+        # split losses and images
+        real_losses = {}
+        log_imgs = {}
+        for k, v in losses.items():
+            if "img" in k:
+                log_imgs[k] = v
+            else:
+                real_losses[k] = v
+
+        loss, log_vars = self._parse_losses(real_losses)
+
+        outputs = dict(loss=loss, log_vars=log_vars, num_samples=len(data_batch["img_metas"]), log_imgs=log_imgs)
+
+        return outputs
+
+    def val_step(self, data_batch, **kwargs):
+        """The iteration step during validation.
+
+        This method shares the same signature as :func:`train_step`, but used
+        during val epochs. Note that the evaluation after training epochs is
+        not implemented with this method, but an evaluation hook.
+        """
+        output = self(**data_batch, **kwargs)
+        return output
+
+    @staticmethod
+    def _parse_losses(losses):
+        import torch.distributed as dist
+
+        """Parse the raw outputs (losses) of the network.
+
+        Args:
+            losses (dict): Raw output of the network, which usually contain
+                losses and other necessary information.
+
+        Returns:
+            tuple[Tensor, dict]: (loss, log_vars), loss is the loss tensor
+                which may be a weighted sum of all losses, log_vars contains
+                all the variables to be sent to the logger.
+        """
+        log_vars = OrderedDict()
+        for loss_name, loss_value in losses.items():
+            if isinstance(loss_value, torch.Tensor):
+                log_vars[loss_name] = loss_value.mean()
+            elif isinstance(loss_value, list):
+                log_vars[loss_name] = sum(_loss.mean() for _loss in loss_value)
+            else:
+                raise TypeError(f"{loss_name} is not a tensor or list of tensors")
+
+        loss = sum(_value for _key, _value in log_vars.items() if "loss" in _key)
+
+        log_vars["loss"] = loss
+        for loss_name, loss_value in log_vars.items():
+            # reduce loss when distributed training
+            if dist.is_available() and dist.is_initialized():
+                loss_value = loss_value.data.clone()
+                dist.all_reduce(loss_value.div_(dist.get_world_size()))
+            log_vars[loss_name] = loss_value.item()
+
+        return loss, log_vars

migrator/checkpoints/hub/facebookresearch_dinov2_main/dinov2/hub/depth/ops.py

@@ -0,0 +1,28 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+import warnings
+
+import torch.nn.functional as F
+
+
+def resize(input, size=None, scale_factor=None, mode="nearest", align_corners=None, warning=False):
+    if warning:
+        if size is not None and align_corners:
+            input_h, input_w = tuple(int(x) for x in input.shape[2:])
+            output_h, output_w = tuple(int(x) for x in size)
+            if output_h > input_h or output_w > output_h:
+                if (
+                    (output_h > 1 and output_w > 1 and input_h > 1 and input_w > 1)
+                    and (output_h - 1) % (input_h - 1)
+                    and (output_w - 1) % (input_w - 1)
+                ):
+                    warnings.warn(
+                        f"When align_corners={align_corners}, "
+                        "the output would more aligned if "
+                        f"input size {(input_h, input_w)} is `x+1` and "
+                        f"out size {(output_h, output_w)} is `nx+1`"
+                    )
+    return F.interpolate(input, size, scale_factor, mode, align_corners)

migrator/checkpoints/hub/facebookresearch_dinov2_main/dinov2/hub/text/dinotxt_model.py

@@ -0,0 +1,135 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+import math
+from dataclasses import dataclass
+from typing import Optional, Tuple
+
+import torch
+import torch.nn.functional as F
+from torch import nn, Tensor
+
+from .vision_tower import VisionTower
+from .text_tower import TextTower
+
+
+@dataclass
+class DinoTxtConfig:
+    embed_dim: int
+    vision_model_freeze_backbone: bool = True
+    vision_model_train_img_size: int = 224
+    vision_model_use_class_token: bool = True
+    vision_model_use_patch_tokens: bool = False
+    vision_model_num_head_blocks: int = 0
+    vision_model_head_blocks_drop_path: float = 0.3
+    vision_model_use_linear_projection: bool = False
+    vision_model_patch_tokens_pooler_type: str = "mean"
+    vision_model_patch_token_layer: int = 1  # which layer to take patch tokens from
+    # 1 - last layer, 2 - second last layer, etc.
+    text_model_freeze_backbone: bool = False
+    text_model_num_head_blocks: int = 0
+    text_model_head_blocks_is_causal: bool = False
+    text_model_head_blocks_drop_prob: float = 0.0
+    text_model_tokens_pooler_type: str = "first"
+    text_model_use_linear_projection: bool = False
+    init_logit_scale: float = math.log(1 / 0.07)
+    init_logit_bias: Optional[float] = None
+    freeze_logit_scale: bool = False
+
+
+class DinoTxt(nn.Module):
+    def __init__(
+        self,
+        model_config: DinoTxtConfig,
+        vision_backbone: nn.Module,
+        text_backbone: nn.Module,
+    ):
+        super().__init__()
+        self.model_config = model_config
+        self.visual_model = VisionTower(
+            vision_backbone,
+            model_config.vision_model_freeze_backbone,
+            model_config.embed_dim,
+            model_config.vision_model_num_head_blocks,
+            model_config.vision_model_head_blocks_drop_path,
+            model_config.vision_model_use_class_token,
+            model_config.vision_model_use_patch_tokens,
+            model_config.vision_model_patch_token_layer,
+            model_config.vision_model_patch_tokens_pooler_type,
+            model_config.vision_model_use_linear_projection,
+        )
+        self.text_model = TextTower(
+            text_backbone,
+            model_config.text_model_freeze_backbone,
+            model_config.embed_dim,
+            model_config.text_model_num_head_blocks,
+            model_config.text_model_head_blocks_is_causal,
+            model_config.text_model_head_blocks_drop_prob,
+            model_config.text_model_tokens_pooler_type,
+            model_config.text_model_use_linear_projection,
+        )
+        self.logit_scale = nn.Parameter(torch.ones(1) * model_config.init_logit_scale)
+        if model_config.freeze_logit_scale:
+            self.logit_scale.requires_grad = False
+
+    def init_weights(self):
+        self.visual_model.init_weights()
+        self.text_model.init_weights()
+
+    def get_visual_class_and_patch_tokens(self, image: Tensor) -> Tuple[Tensor, Tensor]:
+        return self.visual_model.get_class_and_patch_tokens(image)
+
+    def encode_image(
+        self,
+        image: Tensor,
+        normalize: bool = False,
+    ) -> Tensor:
+        """
+        Encode an image into a vector descriptor containing both global and local features.
+
+        Args:
+            image (Tensor): Tensor of shape `(batch_size, rgb, height, width)`, normalized using ImageNet mean and std.
+            normalize (bool, optional): Whether to normalize the output vectors. Default is False.
+                Image features should always be normalized before comparing them with text features:
+        Returns:
+            Tensor: Tensor of shape `(batch_size, embed_dim)` containing the image features.
+                The first half of the vector corresponds to the global features (class token),
+                and the second half corresponds to the pooled patch features.
+        """
+        features = self.visual_model(image)
+        return F.normalize(features, dim=-1) if normalize else features
+
+    def encode_text(self, text: Tensor, normalize: bool = False) -> Tensor:
+        """
+        Encode a text input into a vector descriptor.
+
+        Args:
+            text (Tensor): Tensor of shape `(batch_size, seq_len)` containing token indices.
+            normalize (bool, optional): Whether to normalize the output vectors. Default is False.
+                Text features should be normalized before comparing them with image features:
+        Returns:
+            Tensor: Tensor of shape `(batch_size, embed_dim)` containing the text features.
+                As a consequence of the training procedure, assume that the first half of the tensor corresponds
+                to global image features and the second half to pooled patch features.
+        """
+        features = self.text_model(text)
+        return F.normalize(features, dim=-1) if normalize else features
+
+    def get_logits(self, image: Tensor, text: Tensor) -> Tuple[Tensor, Tensor]:
+        text_features = self.encode_text(text, normalize=True)
+        image_features = self.encode_image(image, normalize=True)
+        image_logits = self.logit_scale.exp() * image_features @ text_features.T
+        text_logits = image_logits.T
+        return image_logits, text_logits
+
+    def forward(
+        self,
+        image: Tensor,
+        text: Tensor,
+    ) -> Tuple[Tensor, Tensor, Tensor]:
+
+        text_features = self.encode_text(text, normalize=True)
+        image_features = self.encode_image(image, normalize=True)
+        return image_features, text_features, self.logit_scale.exp()

migrator/checkpoints/hub/facebookresearch_dinov2_main/dinov2/hub/text/dinov2_wrapper.py

@@ -0,0 +1,64 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+from typing import Sequence
+
+import torch
+
+
+class DINOv2Wrapper(torch.nn.Module):
+    def __init__(self, model):
+        super().__init__()
+        self.model = model
+        self.embed_dim = model.embed_dim
+        self.num_heads = model.num_heads
+        self.num_register_tokens = model.num_register_tokens
+
+    # Same as the original forward, but assert is_training and rename x_norm_regtokens -> x_storage_tokens
+    def forward(self, img, is_training: bool):
+        assert is_training
+        H, W = img.shape[-2:]
+        P = self.model.patch_size
+        x_dict = self.model(img, is_training=True)
+        x_dict["h"] = h = H // P
+        x_dict["w"] = w = W // P
+        assert x_dict["x_norm_patchtokens"].shape[-2] == h * w
+        return x_dict
+
+    # Same as the original get_intermediate_layers, but allow returining extra tokens (registers)
+    def get_intermediate_layers(
+        self,
+        x: torch.Tensor,
+        n: int | Sequence[int] = 1,  # Layers or n last layers to take
+        reshape: bool = False,
+        return_class_token: bool = False,
+        return_register_tokens: bool = False,
+        norm=True,
+    ) -> tuple[torch.Tensor] | tuple[tuple[torch.Tensor, ...], ...]:
+        if self.model.chunked_blocks:
+            outputs = self.model._get_intermediate_layers_chunked(x, n)
+        else:
+            outputs = self.model._get_intermediate_layers_not_chunked(x, n)
+        if norm:
+            outputs = [self.model.norm(out) for out in outputs]
+        class_tokens = [out[:, 0] for out in outputs]
+        register_tokens = [out[:, 1 : 1 + self.model.num_register_tokens] for out in outputs]
+        outputs = [out[:, 1 + self.model.num_register_tokens :] for out in outputs]
+        if reshape:
+            B, _, h, w = x.shape
+            outputs = [
+                out.reshape(B, h // self.model.patch_size, w // self.model.patch_size, -1)
+                .permute(0, 3, 1, 2)
+                .contiguous()
+                for out in outputs
+            ]
+
+        if not return_class_token and not return_register_tokens:
+            return tuple(outputs)
+        if return_class_token and not return_register_tokens:
+            return tuple(zip(outputs, class_tokens))
+        if not return_class_token and return_register_tokens:
+            return tuple(zip(outputs, register_tokens))
+        return tuple(zip(outputs, class_tokens, register_tokens))

migrator/checkpoints/hub/facebookresearch_dinov2_main/dinov2/hub/text/text_tower.py

@@ -0,0 +1,104 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+import torch
+from torch import nn, Tensor
+
+from dinov2.layers import (
+    CausalAttentionBlock,
+)
+
+
+class TextHead(nn.Module):
+    def __init__(
+        self,
+        input_dim: int,
+        embed_dim: int,
+        num_heads: int,
+        num_blocks: int,
+        block_drop_prob: float,
+        is_causal: bool,
+        use_linear_projection: bool,
+    ):
+        super().__init__()
+        block_list = [nn.Identity()]
+        self.ln_final = nn.Identity()
+        if num_blocks > 0:
+            block_list = [
+                CausalAttentionBlock(
+                    dim=input_dim,
+                    num_heads=num_heads,
+                    is_causal=is_causal,
+                    dropout_prob=block_drop_prob,
+                )
+                for _ in range(num_blocks)
+            ]
+            self.ln_final = nn.LayerNorm(input_dim)
+        self.block_list = nn.ModuleList(block_list)
+        self.num_blocks = num_blocks
+        self.linear_projection = nn.Identity()
+        if input_dim != embed_dim or use_linear_projection:
+            self.linear_projection = nn.Linear(input_dim, embed_dim, bias=False)
+
+    def init_weights(self):
+        if self.num_blocks > 0:
+            for i in range(self.num_blocks):
+                self.block_list[i].init_weights()
+            self.ln_final.reset_parameters()
+        if isinstance(self.linear_projection, nn.Linear):
+            nn.init.normal_(self.linear_projection.weight, std=self.linear_projection.in_features**-0.5)
+
+    def forward(self, text_tokens: Tensor) -> Tensor:
+        for block in self.block_list:
+            text_tokens = block(text_tokens)
+        text_tokens = self.ln_final(text_tokens)
+        return self.linear_projection(text_tokens)
+
+
+class TextTower(nn.Module):
+    def __init__(
+        self,
+        backbone: nn.Module,
+        freeze_backbone: bool,
+        embed_dim: int,
+        num_head_blocks: int,
+        head_blocks_is_causal: bool,
+        head_blocks_block_drop_prob: float,
+        tokens_pooler_type: str,
+        use_linear_projection: bool,
+    ):
+        super().__init__()
+        self.backbone = backbone
+        self.freeze_backbone = freeze_backbone
+        backbone_out_dim = backbone.embed_dim
+        self.backbone = backbone
+        self.head = TextHead(
+            backbone_out_dim,
+            embed_dim,
+            self.backbone.num_heads,
+            num_head_blocks,
+            head_blocks_block_drop_prob,
+            head_blocks_is_causal,
+            use_linear_projection,
+        )
+        self.tokens_pooler_type = tokens_pooler_type
+
+    def init_weights(self):
+        self.backbone.init_weights()
+        self.head.init_weights()
+
+    def forward(self, token_indices: Tensor) -> Tensor:
+        text_tokens = self.backbone(token_indices)
+        text_tokens = self.head(text_tokens)
+        if self.tokens_pooler_type == "first":
+            features = text_tokens[:, 0]
+        elif self.tokens_pooler_type == "last":
+            features = text_tokens[:, -1]
+        elif self.tokens_pooler_type == "argmax":
+            assert token_indices is not None
+            features = text_tokens[torch.arange(text_tokens.shape[0]), token_indices.argmax(dim=-1)]
+        else:
+            raise ValueError(f"Unknown text tokens pooler type: {self.pooler_type}")
+        return features

migrator/checkpoints/hub/facebookresearch_dinov2_main/dinov2/hub/text/text_transformer.py

@@ -0,0 +1,72 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+from typing import Callable, Optional, Tuple
+
+import torch
+from torch import nn, Tensor
+
+
+from dinov2.layers import CausalAttentionBlock
+
+
+class TextTransformer(nn.Module):
+    def __init__(
+        self,
+        context_length: int,
+        vocab_size: int,
+        dim: int,
+        num_heads: int,
+        num_layers: int,
+        ffn_ratio: float,
+        is_causal: bool,
+        ls_init_value: Optional[float] = None,
+        act_layer: Callable = nn.GELU,
+        norm_layer: Callable = nn.LayerNorm,
+        dropout_prob: float = 0.0,
+    ):
+        super().__init__()
+        self.vocab_size = vocab_size
+        self.embed_dim = dim
+        self.num_heads = num_heads
+
+        self.token_embedding = nn.Embedding(vocab_size, dim)
+        self.positional_embedding = nn.Parameter(torch.empty(context_length, dim))
+        self.dropout = nn.Dropout(dropout_prob)
+        self.num_layers = num_layers
+        block_list = [
+            CausalAttentionBlock(
+                dim=dim,
+                num_heads=num_heads,
+                ffn_ratio=ffn_ratio,
+                ls_init_value=ls_init_value,
+                is_causal=is_causal,
+                act_layer=act_layer,
+                norm_layer=norm_layer,
+                dropout_prob=dropout_prob,
+            )
+            for _ in range(num_layers)
+        ]
+        self.blocks = nn.ModuleList(block_list)
+        self.ln_final = norm_layer(dim)
+
+    def init_weights(self):
+        nn.init.normal_(self.token_embedding.weight, std=0.02)
+        nn.init.normal_(self.positional_embedding, std=0.01)
+        init_attn_std = self.embed_dim**-0.5
+        init_proj_std = (self.embed_dim**-0.5) * ((2 * self.num_layers) ** -0.5)
+        init_fc_std = (2 * self.embed_dim) ** -0.5
+        for block in self.blocks:
+            block.init_weights(init_attn_std, init_proj_std, init_fc_std)
+        self.ln_final.reset_parameters()
+
+    def forward(self, token_indices: Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+        _, N = token_indices.size()
+        x = self.token_embedding(token_indices) + self.positional_embedding[:N]
+        x = self.dropout(x)
+        for block in self.blocks:
+            x = block(x)
+        x = self.ln_final(x)
+        return x

migrator/checkpoints/hub/facebookresearch_dinov2_main/dinov2/hub/text/tokenizer.py

@@ -0,0 +1,45 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+import torch
+from typing import List, Union
+
+
+from dinov2.thirdparty.CLIP.clip.simple_tokenizer import SimpleTokenizer
+
+
+class Tokenizer(SimpleTokenizer):
+    def __init__(self, vocab_path: str):
+        SimpleTokenizer.__init__(self, bpe_path=vocab_path)
+
+    def tokenize(self, texts: Union[str, List[str]], context_length: int = 77) -> torch.LongTensor:
+        """
+        Returns the tokenized representation of given input string(s)
+
+        Parameters
+        ----------
+        texts : Union[str, List[str]]
+            An input string or a list of input strings to tokenize
+        context_length : int
+            The context length to use; all CLIP models use 77 as the context length
+
+        Returns
+        -------
+        A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length]
+        """
+        if isinstance(texts, str):
+            texts = [texts]
+        sot_token = self.encoder["<|startoftext|>"]
+        eot_token = self.encoder["<|endoftext|>"]
+        all_tokens = [[sot_token] + self.encode(text) + [eot_token] for text in texts]
+        result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)
+
+        for i, tokens in enumerate(all_tokens):
+            if len(tokens) > context_length:
+                tokens = tokens[:context_length]  # Truncate
+                tokens[-1] = eot_token
+            result[i, : len(tokens)] = torch.tensor(tokens)
+
+        return result

migrator/checkpoints/hub/facebookresearch_dinov2_main/dinov2/hub/text/vision_tower.py

@@ -0,0 +1,179 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+from functools import partial
+from typing import Callable, Tuple
+
+import torch
+from torch import nn, Tensor
+
+from dinov2.layers import (
+    LayerScale,
+    NestedTensorBlock as AttentionBlock,
+    SwiGLUFFNAligned as SwiGLUFFN,
+)
+
+
+def init_weights_vit_timm(module: nn.Module, name: str = ""):
+    """ViT weight initialization, original timm impl (for reproducibility)"""
+    if isinstance(module, nn.Linear):
+        nn.init.trunc_normal_(module.weight, std=0.02)
+        if module.bias is not None:
+            nn.init.zeros_(module.bias)
+    if isinstance(module, nn.LayerNorm):
+        module.reset_parameters()
+    if isinstance(module, LayerScale):
+        module.reset_parameters()
+    if isinstance(module, nn.Conv2d):
+        module.reset_parameters()
+
+
+def named_apply(fn: Callable, module: nn.Module, name="", depth_first=True, include_root=False) -> nn.Module:
+    if not depth_first and include_root:
+        fn(module=module, name=name)
+    for child_name, child_module in module.named_children():
+        child_name = ".".join((name, child_name)) if name else child_name
+        named_apply(
+            fn=fn,
+            module=child_module,
+            name=child_name,
+            depth_first=depth_first,
+            include_root=True,
+        )
+    if depth_first and include_root:
+        fn(module=module, name=name)
+    return module
+
+
+class VisionHead(nn.Module):
+    def __init__(
+        self,
+        input_dim: int,
+        embed_dim: int,
+        num_heads: int,
+        num_blocks: int,
+        blocks_drop_path: float,
+        use_class_token: bool,
+        use_patch_tokens: bool,
+        use_linear_projection: bool,
+    ):
+        super().__init__()
+        block_list = [nn.Identity()]
+        self.ln_final = nn.Identity()
+        if num_blocks > 0:
+            block_list = [
+                AttentionBlock(
+                    input_dim,
+                    num_heads,
+                    ffn_layer=partial(SwiGLUFFN, align_to=64),
+                    init_values=1e-5,
+                    drop_path=blocks_drop_path,
+                )
+                for _ in range(num_blocks)
+            ]
+            self.ln_final = nn.LayerNorm(input_dim)
+        self.block_list = nn.ModuleList(block_list)
+        self.num_blocks = num_blocks
+        multiplier = 2 if use_class_token and use_patch_tokens else 1
+        self.linear_projection = nn.Identity()
+        if multiplier * input_dim != embed_dim or use_linear_projection:
+            assert embed_dim % multiplier == 0, f"Expects {embed_dim} to be divisible by {multiplier}"
+            self.linear_projection = nn.Linear(input_dim, embed_dim // multiplier, bias=False)
+
+    def init_weights(self):
+        if self.num_blocks > 0:
+            for i in range(self.num_blocks):
+                block = self.block_list[i]
+                named_apply(init_weights_vit_timm, block)
+            self.ln_final.reset_parameters()
+        if isinstance(self.linear_projection, nn.Linear):
+            nn.init.normal_(self.linear_projection.weight, std=self.linear_projection.in_features**-0.5)
+
+    def forward(self, image_tokens: Tensor) -> Tensor:
+        for block in self.block_list:
+            image_tokens = block(image_tokens)
+        image_tokens = self.ln_final(image_tokens)
+        return self.linear_projection(image_tokens)
+
+
+class VisionTower(nn.Module):
+    def __init__(
+        self,
+        backbone: nn.Module,
+        freeze_backbone: bool,
+        embed_dim: int,
+        num_head_blocks: int,
+        head_blocks_block_drop_path: float,
+        use_class_token: bool,
+        use_patch_tokens: bool,
+        patch_token_layer: int,
+        patch_tokens_pooler_type: str,
+        use_linear_projection: bool,
+    ):
+        super().__init__()
+        self.backbone = backbone
+        self.freeze_backbone = freeze_backbone
+        self.use_class_token = use_class_token
+        self.use_patch_tokens = use_patch_tokens
+        self.patch_token_layer = patch_token_layer
+        self.patch_tokens_pooler_type = patch_tokens_pooler_type
+        self.num_register_tokens = 0
+        if hasattr(self.backbone, "num_register_tokens"):
+            self.num_register_tokens = self.backbone.num_register_tokens
+        elif hasattr(self.backbone, "n_storage_tokens"):
+            self.num_register_tokens = self.backbone.n_storage_tokens
+        backbone_out_dim = self.backbone.embed_dim
+        self.head = VisionHead(
+            backbone_out_dim,
+            embed_dim,
+            self.backbone.num_heads,
+            num_head_blocks,
+            head_blocks_block_drop_path,
+            use_class_token,
+            use_patch_tokens,
+            use_linear_projection,
+        )
+
+    def init_weights(self):
+        if not self.freeze_backbone:
+            self.backbone.init_weights()
+        self.head.init_weights()
+
+    def get_backbone_features(self, images: Tensor) -> Tuple[Tensor, Tensor, Tensor]:
+        tokens = self.backbone.get_intermediate_layers(
+            images,
+            n=self.patch_token_layer,
+            return_class_token=True,
+            return_register_tokens=True,
+        )
+        class_token = tokens[-1][1]
+        patch_tokens = tokens[0][0]
+        register_tokens = tokens[0][2]
+        return class_token, patch_tokens, register_tokens
+
+    def get_class_and_patch_tokens(self, images: Tensor) -> Tuple[Tensor, Tensor]:
+        class_token, patch_tokens, register_tokens = self.get_backbone_features(images)
+        image_tokens = self.head(torch.cat([class_token.unsqueeze(1), register_tokens, patch_tokens], dim=1))
+        class_token, patch_tokens = image_tokens[:, 0], image_tokens[:, self.num_register_tokens + 1 :]
+        return class_token, patch_tokens
+
+    def forward(self, images: Tensor) -> Tensor:
+        class_token, patch_tokens = self.get_class_and_patch_tokens(images)
+        features = []
+        if self.use_class_token:
+            features.append(class_token)
+        if self.use_patch_tokens:
+            if self.patch_tokens_pooler_type == "mean":
+                features.append(torch.mean(patch_tokens, dim=1))
+            elif self.patch_tokens_pooler_type == "max":
+                features.append(torch.max(patch_tokens, dim=1).values)
+            elif self.patch_tokens_pooler_type == "gem":
+                power = 3
+                eps = 1e-6
+                patch_tokens_power = patch_tokens.clamp(min=eps).pow(power)
+                features.append(torch.mean(patch_tokens_power, dim=1).pow(1 / power))
+            else:
+                raise ValueError(f"Unknown patch tokens pooler type: {self.patch_tokens_pooler_type}")
+        return torch.cat(features, dim=-1)

migrator/checkpoints/hub/facebookresearch_dinov2_main/dinov2/hub/xray_dino/backbones.py

@@ -0,0 +1,28 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the licence
+# found in the LICENSE_XRAY_DINO_MODEL file in the root directory of this source tree.
+
+from typing import Union
+
+from ..backbones import Weights, _make_dinov2_model
+
+
+def xray_dino_vitl16(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.XRAY_DINO, **kwargs):
+    """
+    XRay-DINO ViT-L/16 model (optionally) pretrained on the XRay-DINO dataset.
+    """
+    return _make_dinov2_model(
+        arch_name="vit_large",
+        patch_size=16,
+        img_size=512,
+        num_register_tokens=0,
+        interpolate_antialias=False,
+        interpolate_offset=0.1,
+        block_chunks=4,
+        pretrained=pretrained,
+        weights=weights,
+        hash="ad31c2b0",
+        check_hash=True,
+        **kwargs,
+    )

migrator/code/mv-sam3d-for-6d-v2/IMPLEMENTATION.md

@@ -0,0 +1,208 @@
+# mv-sam3d-for-6d-v2 — Implementation
+
+A clean, self-contained rebuild of the multi-view (multi-camera) DexYCB 6D-pose
+flow model.  Conceptually: *take SAM-3D (`sam3d-for-6d`), drop the shape-K/V
+forcing, change the conditioning fed to the flow transformer, for multi-view
+DexYCB 6D pose.*  Everything that already exists in the fork is **vendored
+verbatim** and warm-started from pretrained weights; only the genuinely-new
+pieces (DexYCB multi-view dataset, calibration, training loop, latent-cache
+builder, and the one per-view embedding) are written from scratch.
+
+All paths are absolute.  Python env:
+`/NHNHOME/data/jonghoon/miniconda3/envs/sam3d-objects/bin/python`.
+GPU work is **sbatch-only**; nothing sets `CUDA_VISIBLE_DEVICES`.  The repo is
+self-contained: `PYTHONPATH=<this repo>` only — no dependency on the fork's path,
+no TRELLIS.  Every vendored-package `__init__.py` is empty (the fork's lidra init
+chain is NOT pulled in), so imports are clean.
+
+---
+
+## Model design (locked)
+
+* **Backbone:** `SparseStructureFlowTdfyWrapper` (MoT DiT), `mode="default"`
+  (fully-symmetric joint self-attention), **`is_shortcut_model=False`** = plain
+  rectified flow, v-prediction, `sigma_min=0`.  `model_channels=cond_channels=1024`,
+  `num_blocks=24`, `num_heads=16`, `mlp_ratio=4`, `pe_mode=ape`,
+  `qk_rms_norm=True`, `time_scale=1000`.
+* **Two MoT streams:**
+  * `shape` — SS latent `[B,8,16,16,16]` → `view(B,8,-1).permute(0,2,1)` →
+    `[B,4096,8]` (row-major to match `ShapePositionEmbedder` meshgrid `ij`,
+    res16/patch1).
+  * pose — one shared transformer over `{6drotation_normalized[1,6],
+    translation[1,3], scale[1,1]}` via `latent_share_transformer`.  The shared
+    group is **keyed `6drotation_normalized`** (matches the pretrained block-stream
+    name so all per-block weights warm-start; see warm-start note below).
+  * **NO shape-K/V injection.**
+* **Loss** = weighted v-pred MSE, all stream weights **1.0** (shape /
+  6drotation_normalized / translation / scale) — matches the pretrained backbone's
+  training regime (SAM-3D fork uses rot=trans=scale=1.0).
+
+## Conditioning / multi-view (the genuinely-new part)
+
+One timestamp, multiple synchronized DexYCB cameras sharing one object pose.  The
+**anchor** camera defines the reference frame; others are **aux**.  Everything is
+in the anchor camera's PyTorch3D frame.  The SAME `EmbedderFuser` instance runs
+per view, and a per-view **camera embedding** `mlp_cam(extrinsics_v)` is added to
+that view's tokens (replacing the old content-free `view_emb` slot table).  A
+2-layer MLP maps each view's relative aux→anchor extrinsics — rot6d(6) +
+SSI-normalized camera centre(3), 9-D, anchor = identity — to `model_channels`,
+with the output layer zero-initialized so step-0 conditioning equals the no-tag
+baseline.  This binds the camera frame to every token of the view (including the
+geometry-free RGB/mask tokens) and is camera-consistent across the random aux
+subset.  Anchor
+passes all 6 fuser kwargs (cropped+full of image/mask/pointmap); each aux passes
+the 3 cropped kwargs.  All views' tokens are concatenated into one condition
+sequence the DiT cross-attends to.  Pointmaps KEEP NaN at invalid pixels so the
+pretrained `invalid_xyz_token` pathway fires; pointmaps/masks use nearest-mode
+(NaN-preserving) interpolation.
+
+---
+
+## File map
+
+```
+mvsam3d/
+  data/
+    canonical.py        # CAD -> canonical voxel (vendored voxelize_mesh) + GT layout math
+    dexycb_calib.py     # extrinsics/intrinsics; cross-camera P3D transform (aux->anchor)
+    dexycb_dataset.py   # MultiViewDexYCB + collate_multiview   [from scratch]
+  model/
+    condition.py        # MultiViewConditionEmbedder (real EmbedderFuser + per-view view_emb)
+    flow_model.py        # MVSam3DFlow: MoT joint flow + flow_step + loss + warm-start
+    ss_vae.py           # REAL SAM-3D SS encoder/decoder loaders (ss_encoder.ckpt / ss_decoder.ckpt)
+  train/
+    train.py            # YAML-driven DDP training loop (grad-accum + periodic val)  [from scratch]
+    validate.py         # sampler-based validation: decode->64^3 IoU + pose metrics  [from scratch]
+sam3d_objects/          # VENDORED closure (byte-identical to the fork; empty __init__ chain)
+  data/utils.py
+  model/layers/llama3/ff.py
+  model/backbone/dit/embedder/{dino,embedder_fuser,pointmap,point_remapper}.py
+  model/backbone/tdfy_dit/models/{mm_latent,mot_sparse_structure_flow,timestep_embedder,
+                                  sparse_structure_vae}.py
+  model/backbone/tdfy_dit/modules/{spatial,utils,norm}.py + transformer/ + attention/
+  model/backbone/generator/flow_matching/solver.py  # Euler ODE solver (reused for sampling)
+  training/stage1_flow_lora/distributed.py           # DDP/DeepSpeed primitives (reused)
+  geometry/{geometry,rotation_constants}.py    # column-convention rot6d utils
+  pipeline/inference_utils.py   # voxelize_mesh / normalize_mesh_verts / trimesh2o3d_mesh (verbatim)
+tools/
+  build_canonical_latents.py    # 21 objects -> cache/canonical/*.pt (GPU)
+  validate_model.py             # pretrained-load + fwd/bwd + grad audit (GPU)
+  verify_rotation_roundtrip.py  # rot6d encode/decode inverse check (CPU)
+  verify_gt_reproject.py        # GT modal-reproject IoU vs seg (CPU)
+  amodal_reproject.py           # GT amodal IoU vs independent mesh render (CPU)
+  diagnose_reproject.py         # GT reproject recall vs seg (CPU)
+configs/train.yaml
+sbatch/train.sh                 # training launcher
+cache/canonical/<object>.pt     # {latent[8,16,16,16], occupancy[64^3] uint8, max_extent, center_cad}
+```
+
+## Inherited (vendored, byte-identical to fork) vs from-scratch
+
+| Component | Source | Notes |
+|---|---|---|
+| MoT backbone, `Latent`, position embedders | vendored | `mot_sparse_structure_flow.py`, `mm_latent.py` |
+| `EmbedderFuser`, `Dino`, `PointPatchEmbed`, `FeedForward`, `PointRemapper` | vendored | embedder closure |
+| SS VAE `SparseStructureEncoder/Decoder` | vendored | `sparse_structure_vae.py` |
+| rot6d utils (column convention) + constants | vendored | `geometry.py`, `rotation_constants.py` |
+| `voxelize_mesh` / `normalize_mesh_verts` / `trimesh2o3d_mesh` | vendored | verbatim functions only |
+| `Euler` ODE solver (`flow_matching/solver.py`) | vendored | reused by `MVSam3DFlow.sample` |
+| DDP/DeepSpeed primitives (`distributed.py`) | vendored | reused by `train.py` |
+| `MultiViewConditionEmbedder` | from scratch | thin wrapper; only `mlp_cam` (per-view camera embedding from extrinsics) is a new module |
+| `MVSam3DFlow` (config/flow_step/loss/warm-start/sample/predict) | from scratch | plain flow, no shape-K/V |
+| DexYCB dataset / calibration / canonical GT | from scratch | no fork equivalent |
+| latent-cache builder, DDP training loop, validation, config | from scratch | |
+
+## Pretrained weights
+
+* **SS encoder/decoder = the REAL SAM-3D ones** (`checkpoints/hf/ss_encoder.ckpt`,
+  `ss_decoder.ckpt`), loaded with **0 missing / 0 unexpected** keys through the
+  vendored `SparseStructureEncoder`/`SparseStructureDecoder`.  GT latents use the
+  posterior MEAN (`sample_posterior=False`).  GT latents are validated by
+  roundtrip through the **standalone** `ss_decoder.ckpt` (expect IoU≈1.0).  The
+  decoder embedded in `ss_generator.ckpt` is TRELLIS-identical and gives IoU≈0 on
+  these latents — reported for contrast, never used to validate.
+* **Warm-start from `ss_generator.ckpt`:** the condition embedder
+  (`_base_models.condition_embedder.*`, 722 tensors) loads fully into the frozen
+  `EmbedderFuser`; the torso (`_base_models.generator.reverse_fn.backbone.*`)
+  loads **936 / 936** tensors — `t_embedder`, both `latent_mapping` streams, and
+  **all** per-block transformer weights for both MoT streams (`shape` and the
+  shared pose stream).  **The shared pose group is named `6drotation_normalized`
+  (NOT `pose`)** so its per-block keys match the pretrained block-stream naming
+  (the fork names a shared MoT group after one of its member latents); naming it
+  `pose` previously left all 432 pose-stream block tensors at random init.  Only
+  the shortcut `d_embedder` (4 tensors) and the unused `translation_scale` stream
+  have no home here and are skipped cleanly (never forced).  After this, **the
+  ONLY from-scratch module in the whole model is `mlp_cam`** (the per-view camera
+  embedding; no pretrained source, output layer zero-init), which injects the
+  relative camera extrinsics that tag the new multi-view axis.
+
+## Data / GT
+
+* `dexycb_root=/NHNHOME/data/jonghoon/pose6daug/dataset/dexycb`.  `ycb_ids`/`seg`
+  are 1-based class indices; object mask = `(seg==ycb_id)`.  Depth uint16 mm /1000,
+  clip to [0.1,3.0] m, invalid→NaN.  Pointmap = unproject(aligned_depth) in OpenCV
+  → PyTorch3D via `*[-1,-1,1]`.  Aux pointmaps transformed into the anchor frame
+  via DexYCB extrinsics BEFORE one shared SSI on the anchor.
+* **SSI** = base/default isotropic: `shift=[0,0,median_z]`,
+  `scale=mean(|p-shift|)` over finite points (matches the fork's BASE
+  `ScaleShiftInvariant.get_scale_and_shift`).  Isotropic is required because the
+  GT pose is a rigid similarity; a per-dim anisotropic SSI would shear the
+  rotation cue.  One SSI computed on the anchor is applied to all fused views.
+* **GT layout (anchor cam, P3D):** with `M=[[1,0,0],[0,0,-1],[0,1,0]]`,
+  `D=diag(-1,-1,1)`: `R_p3d=D·R_cv`, `t_p3d=D·t_cv`; then `R=R_dex·Mᵀ`,
+  `t=R_dex·center_cad+t_dex`, `s=max_extent`.  `center_cad`/`max_extent` are
+  derived from the vendored `voxelize_mesh` output (`max_extent=1/scale`,
+  `center_cad=Mᵀ·center_rot`).
+* **Rotation 6D:** encode `normalize_rot6d(matrix_to_rot6d(R))`; decode
+  `rot6d_to_matrix(denormalize_rot6d(x))` — the vendored COLUMN-convention
+  functions used consistently in BOTH dataset and model.  (The prior attempt's
+  interleaved `R[:,:2].reshape(6)` encode was inconsistent with the
+  column-convention decode — a real bug, fixed here.)
+
+## Distributed + validation
+
+* **DDP.** `sbatch/train.sh` launches `torch.distributed.run --nproc_per_node=$NPROC`
+  (`NPROC` defaults to `SLURM_GPUS_ON_NODE`).  The generic DDP/DeepSpeed primitives
+  are the VENDORED `distributed.py` (`init_distributed`/`setup_runtime`/`barrier`/
+  `reduce_scalar_dict`/`cleanup_distributed`); `train.py` is the only from-scratch
+  glue.  Device selection is by `LOCAL_RANK` (never `CUDA_VISIBLE_DEVICES`).
+  `WORLD_SIZE=1` transparently skips the process group + DDP wrap.  Training routes
+  through `model(batch)` (== `flow_step`) so DDP all-reduces gradients; supports
+  `train.grad_accum_steps` (micro-step `no_sync` until the boundary).
+* **Sampling.** `MVSam3DFlow.sample` integrates the rectified-flow ODE from noise
+  (t=0) to data (t=1) with the VENDORED `Euler` solver, exact inverse of `flow_step`
+  (same `time_scale`, `mode="default"`).  `predict` decodes the streams to metric
+  `{shape_latent, rotation[3,3], translation[3], scale}`.
+* **Validation** (`validate.py`, every `train.val_every` steps on a disjoint,
+  deterministic held-out split): runs `predict` then reports `shape_iou` (decode
+  pred latent through the standalone `ss_decoder.ckpt` vs GT canonical occupancy),
+  `rot_deg` (geodesic), `trans_cm`, `scale_rel` — all summed-with-count and
+  all-reduced across ranks for an exact mean.  Val uses a fixed `max_aux` aux views
+  so `view_emb` is never indexed past its trained range.
+* **Verified (job 4331, 2× B200, world=2):** DDP process group + gradient reduce +
+  barrier, 4 steps finite loss, validation at steps 2 & 4 (sampler→decode→metrics,
+  reduced n=2), checkpoint saved, clean exit (COMPLETED).
+
+## How to run
+
+```sh
+# multi-GPU: request N GPUs; one process per GPU is launched automatically
+sbatch --gres=gpu:b200:4 sbatch/train.sh --config configs/train.yaml
+# single GPU (default sbatch header) or quick smoke:
+sbatch sbatch/train.sh --config configs/train.yaml --set train.steps=3 --set data.subjects=[subject-01]
+```
+
+View selection lives under `data:`: `random_views: true` with
+`min_aux_views`/`max_aux_views` (total = 1 + k aux), or `random_views: false`
+for the first `num_aux_views` cameras.  `model.max_views` is derived in `train.py`
+(now API-only; `mlp_cam` has no per-slot table, so the view count is unbounded).
+Camera embedding: `condition.cam_hidden_dim` sets the `mlp_cam` hidden width.
+Validation knobs: `data.val_fraction`/`split_seed`, `train.val_every`/
+`val_num_steps`/`val_max_batches`.  Distributed knobs: `train.backend`
+(`ddp`|`deepspeed`), `grad_accum_steps`, `amp`/`amp_dtype`.  LR schedule:
+`train.scheduler` (`cosine`|`constant`), `warmup_steps`, `min_lr` — REUSES the
+vendored `train_utils.build_scheduler` (warmup→cosine→min_lr).  Resume:
+`train.resume_from=<ckpt>` restores trainables + optimizer + scheduler + scaler +
+step via the vendored `checkpointing.{save,load}_checkpoint` (frozen DINO/fuser are
+reconstructed from `pretrained_ckpt`, not stored).  Verified: jobs 4385 (0→8) +
+4386 (resume@3) — LR curve continuous across resume, optimizer state restored.

migrator/code/mv-sam3d-for-6d-v2/configs/heldout.yaml

@@ -0,0 +1,70 @@
+# Held-out manifest for integrated training. Two kinds (distinction matters):
+#  ood_datasets           : whole-dataset OOD eval; objects MAY overlap training
+#                           (HO-3D is YCB) -> NO object-disjointness enforced.
+#  unseen_object_datasets : whole dataset held out AND its object identities
+#                           reserved (removed from every train dataset). Objects
+#                           are non-YCB so this is safe.
+ood_datasets: [ho3d, ycbm, hope]
+unseen_object_datasets: [housecat6d]
+# GraspNet test_novel object holdout (M1 resolved). GraspNet stays a `role:train`
+# dataset built on the full `graspnet` root, but the 42 objects that appear in the
+# official test_novel scenes (160-189) are reserved here so they are removed from
+# EVERY train dataset (leak-proof unseen-object holdout for GraspNet-novel eval).
+# Derivation (authoritative, matches graspnetAPI): test_novel = sceneIds 160-189
+# (graspnetAPI/graspnet.py); getObjIds() = union of each scene's object_id_list.txt.
+# We took that union over scenes 160-189 (42 distinct 0-based object ids). The
+# adapter key is `graspnet/{cls_value-1:03d}` and cls_value (meta.mat cls_indexes)
+# == object_id+1 (verified on scenes 160/165/170/180/189), so cls_value-1 ==
+# object_id_list value; keys below are `graspnet/<id:03d>` for the 42 novel ids.
+# NOTE: 9 of these (017,020,026,027,030,051,062,063,066) also appear in
+# train/seen/similar scenes — the global object-disjointness filter removes them
+# from training everywhere, which is exactly why this reservation is required.
+unseen_object_keys_extra:
+- graspnet/003
+- graspnet/004
+- graspnet/006
+- graspnet/017
+- graspnet/019
+- graspnet/020
+- graspnet/024
+- graspnet/026
+- graspnet/027
+- graspnet/028
+- graspnet/030
+- graspnet/031
+- graspnet/032
+- graspnet/033
+- graspnet/035
+- graspnet/045
+- graspnet/047
+- graspnet/049
+- graspnet/051
+- graspnet/055
+- graspnet/062
+- graspnet/063
+- graspnet/065
+- graspnet/066
+- graspnet/067
+- graspnet/071
+- graspnet/072
+- graspnet/073
+- graspnet/074
+- graspnet/075
+- graspnet/076
+- graspnet/077
+- graspnet/078
+- graspnet/079
+- graspnet/080
+- graspnet/081
+- graspnet/082
+- graspnet/083
+- graspnet/084
+- graspnet/085
+- graspnet/086
+- graspnet/087
+never_train: [ho3d, housecat6d]      # adapters refused for split:train
+
+roots:
+  housecat6d: ${MIGRATOR_DATA_ROOT}/housecat6d/extracted
+adapter_kwargs:
+  housecat6d: {split: test}

migrator/code/mv-sam3d-for-6d-v2/configs/train.yaml

@@ -0,0 +1,141 @@
+# Training config for the multi-view 6D-pose flow model (v2).
+# ALL training-related hyperparameters live here; train.py reads them and nothing
+# training-related is hardcoded in Python. Launch via:
+#   sbatch sbatch/train.sh --config configs/train.yaml
+# Per-key overrides: --set key.subkey=value (repeatable), e.g. --set train.steps=3
+
+# --------------------------------------------------------------------------- #
+# data -> MultiViewDexYCB
+# --------------------------------------------------------------------------- #
+data:
+  dexycb_root: ${MIGRATOR_DATA_ROOT}/dexycb
+  canonical_cache: ${MIGRATOR_CACHE}/canonical
+  ssi_mode: scene            # scene | object_centric — SSI convention for pointmaps
+                             # + t/s targets + cam features. "scene" = base SSI
+                             # (shift=[0,0,median z], all pre-2026-07-17 ckpts).
+                             # "object_centric" = vendored ObjectCentricSSI (object-
+                             # median shift, scene scale; release-pipeline pairing).
+                             # A ckpt MUST be evaluated with its training mode.
+  frame_stride: 5            # sample every Nth frame in each sequence
+  anchor_sweep: true         # every camera serves as anchor (else only the first)
+  subjects: null             # null = all subjects; or a list of name substrings
+  image_size: 518            # DINOv2 input resolution (= IMG_SIZE)
+  # --- view selection ---
+  num_aux_views: 7           # legacy fixed count, used only when random_views=false
+  random_views: true         # draw a random count + random subset of aux cameras per sample
+  min_aux_views: 1           # inclusive lower bound on aux count when random_views=true
+  max_aux_views: 3           # inclusive upper bound -> total views = 1 + [1..3] = 2..4
+  # --- depth preprocessing constants ---
+  depth_scale: 1000.0        # DexYCB depth PNG is uint16 millimetres
+  depth_near: 0.1            # metres; clip implausibly-near depth
+  depth_far: 3.0             # metres; clip implausibly-far depth
+  # --- train/val split (deterministic; disjoint) ---
+  val_fraction: 0.02         # fraction of samples held out for validation (0 = no val)
+  split_seed: 0              # fixed seed for the train/val partition
+
+# --------------------------------------------------------------------------- #
+# model -> MVSam3DFlow
+# NOTE: max_views is DERIVED in train.py as
+#   (max_aux_views if random_views else num_aux_views) + 1
+# SHAPE_RES (16) and SHAPE_CHANNELS (8) are locked by the SAM-3D SS VAE and
+# remain code constants in flow_model.py.
+# --------------------------------------------------------------------------- #
+model:
+  model_channels: 1024
+  cond_channels: 1024
+  num_blocks: 24
+  num_heads: 16
+  sigma_min: 0.0             # rectified flow; 0 = straight OT path
+  time_scale: 1000.0         # flow time scaling fed to the backbone
+  # Timestep sampling for the flow loss. Default follows the ORIGINAL SAM-3D
+  # generator pretraining schedule (ss_generator.yaml training_time_sampler_fn =
+  # lognorm_sampler mean=-1, std=1). v2's t convention (t=0 noise -> t=1 data) is
+  # identical to the FlowMatching module's, so mean maps directly (no sign flip).
+  # Set timestep_sampler=uniform to reproduce the stage1_flow_lora harness.
+  timestep_sampler: logit_normal   # logit_normal | uniform
+  timestep_lognorm_mean: -1.0
+  timestep_lognorm_std: 1.0
+  # Warm-start from SAM-3D's pretrained ss_generator. Loads BOTH the condition
+  # embedder (722 tensors -> frozen EmbedderFuser) AND the DiT torso (shape /
+  # 6drotation_normalized / translation streams + blocks + t_embedder). Set to
+  # null to train from scratch.
+  pretrained_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/ss_generator.ckpt
+  # Standalone SAM-3D SS decoder (authoritative latent space) — used ONLY by
+  # validation to decode predicted shape latents to 64^3 occupancy for IoU.
+  ss_decoder_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/ss_decoder.ckpt
+
+# --------------------------------------------------------------------------- #
+# condition -> MultiViewConditionEmbedder (thin multi-view wrapper around
+# SAM-3D's real EmbedderFuser). Mirrors ss_generator.yaml's
+# module.condition_embedder.backbone exactly: 3 embedders (2x Dino +
+# PointPatchEmbed), learned pos-embedding, projection mult 4.0, dropout 0.1
+# with the pointmap/rgb_pointmap drop weight, freeze: true.
+# --------------------------------------------------------------------------- #
+condition:
+  dino_model: dinov2_vitl14_reg
+  dino_input_size: 518
+  pointmap_input_size: 256
+  pointmap_patch_size: 8
+  pointmap_embed_dim: 512
+  projection_hidden_mult: 4.0     # = projection_net_hidden_dim_multiplier
+  dropout_prob: 0.1               # fuser modality dropout (pointmap/rgb_pointmap)
+  # Per-view camera embedding (mlp_cam): maps each view's relative aux->anchor
+  # extrinsics (rot6d + SSI-normalized camera centre, 9-D; anchor = identity) to
+  # model_channels and adds it to that view's tokens. Replaces the old
+  # content-free view_emb index table. Output layer is zero-init (no perturbation
+  # at step 0). This is the ONLY from-scratch module in the model.
+  cam_hidden_dim: 256             # hidden width of the 2-layer mlp_cam
+  # SAM-3D freezes its ENTIRE condition embedder (freeze: true). We mirror that:
+  # the whole EmbedderFuser (both DINOs, PointPatchEmbed, all projection_nets,
+  # idx_emb) is frozen after warm-start. The per-view camera embedding (mlp_cam,
+  # the extrinsics-derived module that replaced view_emb) is always trainable
+  # (no pretrained source). freeze: false would fine-tune the entire condition
+  # pathway.
+  freeze: true
+  freeze_dino: true               # DINO backbone freeze at construction time
+
+# --------------------------------------------------------------------------- #
+# loss_weights -> MVSam3DFlow.loss_weights (per-stream v-prediction MSE weights)
+# --------------------------------------------------------------------------- #
+loss_weights:
+  # All 1.0 — matches the pretrained backbone's training regime (SAM-3D fork uses
+  # rot=trans=scale=1.0) so warm-started weights are fine-tuned under the same
+  # objective balance. Rotation has the largest raw loss (hardest stream), so 1.0
+  # gives it the gradient share it needs rather than down-weighting it.
+  shape: 1.0
+  6drotation_normalized: 1.0
+  translation: 1.0
+  scale: 1.0
+
+# --------------------------------------------------------------------------- #
+# optim + train loop
+# --------------------------------------------------------------------------- #
+optim:
+  lr: 1.0e-4
+  weight_decay: 0.0          # AdamW
+  grad_clip: 1.0             # max grad norm
+
+train:
+  batch_size: 1              # per-GPU micro-batch
+  steps: 100000
+  num_workers: 8
+  # --- LR schedule (reused fork build_scheduler: warmup -> cosine -> min_lr) ---
+  scheduler: cosine          # cosine | constant
+  warmup_steps: 1000         # linear warmup steps before cosine decay
+  min_lr: 1.0e-6             # cosine decay floor
+  # --- resume ---
+  resume_from: ""            # path to a checkpoint to resume (empty = fresh start)
+  log_every: 20
+  ckpt_every: 2000
+  ckpt_dir: ${MIGRATOR_CACHE}/ckpt
+  amp: true                  # autocast on cuda (dtype below)
+  amp_dtype: bf16            # bf16 | fp16 (fp16 enables GradScaler)
+  seed: 0                    # seeds torch/numpy/random at startup (offset by rank)
+  # --- distributed (DDP via torchrun; world_size=1 transparently skips it) ---
+  backend: ddp               # ddp | deepspeed
+  deepspeed_config: ""       # path to a DeepSpeed JSON (only used when backend=deepspeed)
+  grad_accum_steps: 1        # micro-steps per optimizer step
+  # --- validation (model's own Euler sampler + decode) ---
+  val_every: 2000            # run validation every N optimizer steps (0 = never)
+  val_num_steps: 50          # Euler ODE steps for sampling during validation
+  val_max_batches: 16        # val batches per rank per validation pass

migrator/code/mv-sam3d-for-6d-v2/configs/train_integrated.yaml

@@ -0,0 +1,127 @@
+data:
+  dexycb_root: ${MIGRATOR_DATA_ROOT}/dexycb
+  canonical_cache: ${MIGRATOR_CACHE}/canonical
+  frame_stride: 5
+  anchor_sweep: true
+  subjects: null
+  image_size: 518
+  num_aux_views: 7
+  random_views: true
+  min_aux_views: 1
+  max_aux_views: 3
+  depth_scale: 1000.0
+  depth_near: 0.1
+  depth_far: 3.0
+  val_fraction: 0.02
+  split_seed: 0
+  heldout_manifest: ${MIGRATOR_MV_REPO}/configs/heldout.yaml
+  datasets:
+  - name: dexycb
+    adapter: dexycb
+    root: ${MIGRATOR_DATA_ROOT}/dexycb
+    role: train
+    weight: 1.0
+  - name: pace
+    adapter: pace
+    root: ${MIGRATOR_DATA_ROOT}/pace
+    role: train
+    weight: 3.0
+    data_split: test
+  - name: graspnet
+    adapter: graspnet
+    root: ${MIGRATOR_DATA_ROOT}/graspnet
+    role: train
+    weight: 2.0
+  - name: ycbv
+    adapter: ycbv
+    root: ${MIGRATOR_DATA_ROOT}/ycbv
+    role: train
+    weight: 1.0
+    data_split: test
+  # Adapters implemented + latent-complete + reprojection-gate >=0.8 (2026-07-17).
+  - name: hocap
+    adapter: hocap
+    root: ${MIGRATOR_DATA_ROOT}/hocap
+    role: train
+    weight: 1.0
+    pseudo_gt: true            # FoundationPose+SDF pseudo-GT (train-only; never eval)
+  - name: h2o
+    adapter: h2o
+    root: ${MIGRATOR_DATA_ROOT}/h2o
+    role: train
+    weight: 0.5               # only 8 objects
+  - name: hograspnet
+    adapter: hograspnet
+    root: ${MIGRATOR_DATA_ROOT}/hograspnet
+    role: train
+    weight: 1.0
+    frame_stride: 15          # grasp-hold is quasi-static; keep enumeration tractable
+  - name: dexh2r
+    adapter: dexh2r
+    root: ${MIGRATOR_DATA_ROOT}/dexh2r
+    role: train
+    weight: 1.0
+    pseudo_gt: true            # global-registration+ICP pseudo-GT (train-only); enumerates extracted subjects
+  - name: contactpose
+    adapter: contactpose
+    root: ${MIGRATOR_DATA_ROOT}/contactpose
+    role: train
+    weight: 0.5               # static grasps; only bowl present on disk so far
+    pseudo_gt: true
+  - name: housecat6d
+    adapter: housecat6d
+    root: ${MIGRATOR_DATA_ROOT}/housecat6d/extracted
+    role: heldout
+    data_split: test
+model:
+  model_channels: 1024
+  cond_channels: 1024
+  num_blocks: 24
+  num_heads: 16
+  sigma_min: 0.0
+  time_scale: 1000.0
+  timestep_sampler: logit_normal
+  timestep_lognorm_mean: -1.0
+  timestep_lognorm_std: 1.0
+  pretrained_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/ss_generator.ckpt
+  ss_decoder_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/ss_decoder.ckpt
+condition:
+  dino_model: dinov2_vitl14_reg
+  dino_input_size: 518
+  pointmap_input_size: 256
+  pointmap_patch_size: 8
+  pointmap_embed_dim: 512
+  projection_hidden_mult: 4.0
+  dropout_prob: 0.1
+  cam_hidden_dim: 256
+  freeze: true
+  freeze_dino: true
+loss_weights:
+  shape: 1.0
+  6drotation_normalized: 1.0
+  translation: 1.0
+  scale: 1.0
+optim:
+  lr: 0.0001
+  weight_decay: 0.0
+  grad_clip: 1.0
+train:
+  batch_size: 1
+  steps: 100000
+  num_workers: 8
+  scheduler: cosine
+  warmup_steps: 1000
+  min_lr: 1.0e-06
+  resume_from: ''
+  log_every: 20
+  ckpt_every: 2000
+  ckpt_dir: ${MIGRATOR_CACHE}/ckpt
+  amp: true
+  amp_dtype: bf16
+  seed: 0
+  backend: ddp
+  deepspeed_config: ''
+  grad_accum_steps: 1
+  val_every: 2000
+  val_num_steps: 50
+  val_max_batches: 16

migrator/code/mv-sam3d-for-6d-v2/configs/train_integrated_object_ssi.yaml

@@ -0,0 +1,128 @@
+data:
+  ssi_mode: object_centric            # SSI ablation
+  dexycb_root: ${MIGRATOR_DATA_ROOT}/dexycb
+  canonical_cache: ${MIGRATOR_CACHE}/canonical
+  frame_stride: 5
+  anchor_sweep: true
+  subjects: null
+  image_size: 518
+  num_aux_views: 7
+  random_views: true
+  min_aux_views: 1
+  max_aux_views: 3
+  depth_scale: 1000.0
+  depth_near: 0.1
+  depth_far: 3.0
+  val_fraction: 0.02
+  split_seed: 0
+  heldout_manifest: ${MIGRATOR_MV_REPO}/configs/heldout.yaml
+  datasets:
+  - name: dexycb
+    adapter: dexycb
+    root: ${MIGRATOR_DATA_ROOT}/dexycb
+    role: train
+    weight: 1.0
+  - name: pace
+    adapter: pace
+    root: ${MIGRATOR_DATA_ROOT}/pace
+    role: train
+    weight: 3.0
+    data_split: test
+  - name: graspnet
+    adapter: graspnet
+    root: ${MIGRATOR_DATA_ROOT}/graspnet
+    role: train
+    weight: 2.0
+  - name: ycbv
+    adapter: ycbv
+    root: ${MIGRATOR_DATA_ROOT}/ycbv
+    role: train
+    weight: 1.0
+    data_split: test
+  # Adapters implemented + latent-complete + reprojection-gate >=0.8 (2026-07-17).
+  - name: hocap
+    adapter: hocap
+    root: ${MIGRATOR_DATA_ROOT}/hocap
+    role: train
+    weight: 1.0
+    pseudo_gt: true            # FoundationPose+SDF pseudo-GT (train-only; never eval)
+  - name: h2o
+    adapter: h2o
+    root: ${MIGRATOR_DATA_ROOT}/h2o
+    role: train
+    weight: 0.5               # only 8 objects
+  - name: hograspnet
+    adapter: hograspnet
+    root: ${MIGRATOR_DATA_ROOT}/hograspnet
+    role: train
+    weight: 1.0
+    frame_stride: 15          # grasp-hold is quasi-static; keep enumeration tractable
+  - name: dexh2r
+    adapter: dexh2r
+    root: ${MIGRATOR_DATA_ROOT}/dexh2r
+    role: train
+    weight: 1.0
+    pseudo_gt: true            # global-registration+ICP pseudo-GT (train-only); enumerates extracted subjects
+  - name: contactpose
+    adapter: contactpose
+    root: ${MIGRATOR_DATA_ROOT}/contactpose
+    role: train
+    weight: 0.5               # static grasps; only bowl present on disk so far
+    pseudo_gt: true
+  - name: housecat6d
+    adapter: housecat6d
+    root: ${MIGRATOR_DATA_ROOT}/housecat6d/extracted
+    role: heldout
+    data_split: test
+model:
+  model_channels: 1024
+  cond_channels: 1024
+  num_blocks: 24
+  num_heads: 16
+  sigma_min: 0.0
+  time_scale: 1000.0
+  timestep_sampler: logit_normal
+  timestep_lognorm_mean: -1.0
+  timestep_lognorm_std: 1.0
+  pretrained_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/ss_generator.ckpt
+  ss_decoder_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/ss_decoder.ckpt
+condition:
+  dino_model: dinov2_vitl14_reg
+  dino_input_size: 518
+  pointmap_input_size: 256
+  pointmap_patch_size: 8
+  pointmap_embed_dim: 512
+  projection_hidden_mult: 4.0
+  dropout_prob: 0.1
+  cam_hidden_dim: 256
+  freeze: true
+  freeze_dino: true
+loss_weights:
+  shape: 1.0
+  6drotation_normalized: 1.0
+  translation: 1.0
+  scale: 1.0
+optim:
+  lr: 0.0001
+  weight_decay: 0.0
+  grad_clip: 1.0
+train:
+  batch_size: 1
+  steps: 100000
+  num_workers: 8
+  scheduler: cosine
+  warmup_steps: 1000
+  min_lr: 1.0e-06
+  resume_from: ''
+  log_every: 20
+  ckpt_every: 2000
+  ckpt_dir: ${MIGRATOR_CACHE}/ckpt_object_ssi
+  amp: true
+  amp_dtype: bf16
+  seed: 0
+  backend: ddp
+  deepspeed_config: ''
+  grad_accum_steps: 1
+  val_every: 2000
+  val_num_steps: 50
+  val_max_batches: 16

migrator/code/mv-sam3d-for-6d-v2/configs/train_integrated_scene_ssi.yaml

@@ -0,0 +1,128 @@
+data:
+  ssi_mode: scene            # SSI ablation
+  dexycb_root: ${MIGRATOR_DATA_ROOT}/dexycb
+  canonical_cache: ${MIGRATOR_CACHE}/canonical
+  frame_stride: 5
+  anchor_sweep: true
+  subjects: null
+  image_size: 518
+  num_aux_views: 7
+  random_views: true
+  min_aux_views: 1
+  max_aux_views: 3
+  depth_scale: 1000.0
+  depth_near: 0.1
+  depth_far: 3.0
+  val_fraction: 0.02
+  split_seed: 0
+  heldout_manifest: ${MIGRATOR_MV_REPO}/configs/heldout.yaml
+  datasets:
+  - name: dexycb
+    adapter: dexycb
+    root: ${MIGRATOR_DATA_ROOT}/dexycb
+    role: train
+    weight: 1.0
+  - name: pace
+    adapter: pace
+    root: ${MIGRATOR_DATA_ROOT}/pace
+    role: train
+    weight: 3.0
+    data_split: test
+  - name: graspnet
+    adapter: graspnet
+    root: ${MIGRATOR_DATA_ROOT}/graspnet
+    role: train
+    weight: 2.0
+  - name: ycbv
+    adapter: ycbv
+    root: ${MIGRATOR_DATA_ROOT}/ycbv
+    role: train
+    weight: 1.0
+    data_split: test
+  # Adapters implemented + latent-complete + reprojection-gate >=0.8 (2026-07-17).
+  - name: hocap
+    adapter: hocap
+    root: ${MIGRATOR_DATA_ROOT}/hocap
+    role: train
+    weight: 1.0
+    pseudo_gt: true            # FoundationPose+SDF pseudo-GT (train-only; never eval)
+  - name: h2o
+    adapter: h2o
+    root: ${MIGRATOR_DATA_ROOT}/h2o
+    role: train
+    weight: 0.5               # only 8 objects
+  - name: hograspnet
+    adapter: hograspnet
+    root: ${MIGRATOR_DATA_ROOT}/hograspnet
+    role: train
+    weight: 1.0
+    frame_stride: 15          # grasp-hold is quasi-static; keep enumeration tractable
+  - name: dexh2r
+    adapter: dexh2r
+    root: ${MIGRATOR_DATA_ROOT}/dexh2r
+    role: train
+    weight: 1.0
+    pseudo_gt: true            # global-registration+ICP pseudo-GT (train-only); enumerates extracted subjects
+  - name: contactpose
+    adapter: contactpose
+    root: ${MIGRATOR_DATA_ROOT}/contactpose
+    role: train
+    weight: 0.5               # static grasps; only bowl present on disk so far
+    pseudo_gt: true
+  - name: housecat6d
+    adapter: housecat6d
+    root: ${MIGRATOR_DATA_ROOT}/housecat6d/extracted
+    role: heldout
+    data_split: test
+model:
+  model_channels: 1024
+  cond_channels: 1024
+  num_blocks: 24
+  num_heads: 16
+  sigma_min: 0.0
+  time_scale: 1000.0
+  timestep_sampler: logit_normal
+  timestep_lognorm_mean: -1.0
+  timestep_lognorm_std: 1.0
+  pretrained_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/ss_generator.ckpt
+  ss_decoder_ckpt: ${MIGRATOR_CHECKPOINTS}/hf/ss_decoder.ckpt
+condition:
+  dino_model: dinov2_vitl14_reg
+  dino_input_size: 518
+  pointmap_input_size: 256
+  pointmap_patch_size: 8
+  pointmap_embed_dim: 512
+  projection_hidden_mult: 4.0
+  dropout_prob: 0.1
+  cam_hidden_dim: 256
+  freeze: true
+  freeze_dino: true
+loss_weights:
+  shape: 1.0
+  6drotation_normalized: 1.0
+  translation: 1.0
+  scale: 1.0
+optim:
+  lr: 0.0001
+  weight_decay: 0.0
+  grad_clip: 1.0
+train:
+  batch_size: 1
+  steps: 100000
+  num_workers: 8
+  scheduler: cosine
+  warmup_steps: 1000
+  min_lr: 1.0e-06
+  resume_from: ''
+  log_every: 20
+  ckpt_every: 2000
+  ckpt_dir: ${MIGRATOR_CACHE}/ckpt_scene_ssi
+  amp: true
+  amp_dtype: bf16
+  seed: 0
+  backend: ddp
+  deepspeed_config: ''
+  grad_accum_steps: 1
+  val_every: 2000
+  val_num_steps: 50
+  val_max_batches: 16

migrator/code/mv-sam3d-for-6d-v2/docs/TRAINING_FEATURES.md

@@ -0,0 +1,122 @@
+# mv-sam3d-for-6d-v2 — Training Features
+
+> **Scope.** This will become the single doc that explains every training feature
+> of the multi-view 6D-pose flow model. For now it documents **only the camera
+> embedding (`mlp_cam`)** — the other features (joint flow objective, SSI pose
+> targets, DDP/cosine/resume, held-out validation loss, modality dropout, …) will
+> be added as further sections later. Everything below reflects the code as
+> implemented (verified job 4526); paths are absolute.
+
+---
+
+## 1. Camera embedding (`mlp_cam`)
+
+### One-line summary
+Each view's tokens are tagged with a learned embedding **derived from that view's
+camera pose** (relative aux→anchor extrinsics), instead of a content-free
+per-slot index. This injects inter-camera geometry into the conditioning so the
+appearance tokens (RGB/mask) know which camera they came from.
+
+### The gap it closes
+The conditioning is multi-view: one **anchor** camera defines the reference frame,
+the others are **aux** views. Before this feature, the only thing carrying
+relative camera geometry into the conditions was the anchor-frame **pointmap** —
+RGB and mask tokens carried no geometry, and the old per-view tag (`view_emb`) was
+a learned `[max_views, C]` table indexed by **slot**, which was:
+- **content-free** — it encoded only "which slot," not where the camera is;
+- **order-arbitrary** — with random view sampling, "aux slot 1" is a *different
+  physical camera* every sample, so the table could never learn a stable
+  per-camera signal.
+
+`mlp_cam` replaces that table with a per-view bias computed from the actual
+extrinsics, so the camera frame is bound to **every** token of the view —
+including the otherwise geometry-free RGB/mask tokens — and is consistent across
+the random aux subset.
+
+### Why this form (design decision)
+Three options were considered for injecting camera geometry; we chose the first:
+
+1. **Additive per-view embedding (chosen).** `mlp_cam(extrinsics_v)` → one vector
+   added to all of view *v*'s tokens. It **binds** the camera pose to every
+   appearance token automatically (no ambiguity about which tokens belong to which
+   camera). Cheap, no token-count growth, condition-side only.
+2. **Concatenated camera token (rejected).** A dedicated camera token per view in
+   the sequence has full capacity but *floats* — nothing links it to that view's
+   appearance tokens, so the model has to learn the association.
+3. **Per-pixel ray positional encoding (rejected for this setting).** Plücker rays
+   through the embedder give per-token geometry, but it is **largely redundant**
+   with the anchor-frame pointmap (which already carries per-pixel 3D where depth
+   is valid) and cannot cleanly reach the frozen-DINO RGB/mask tokens. It would be
+   the right choice only in a depth-sparse / monocular regime.
+
+> Note: adopting `mlp_cam` deliberately **reverses** the earlier "no new modules"
+> rule. It is now the *only* from-scratch module in the model.
+
+### What exactly is added
+For each view, a **9-D camera feature** is built from the relative transform that
+maps that camera's points into the anchor PyTorch3D frame
+(`X_anchor = Rrel · X_aux + trel`):
+
+```
+cam_feat (9) = [ rot6d(Rrel) | (trel − shift) / scale ]
+                 └── 6 ──┘     └──────── 3 ─────────┘
+```
+
+- **`rot6d(Rrel)` (6)** — column-convention 6D of the aux→anchor rotation
+  (`matrix_to_rot6d`, the same convention as the object-rotation target). Raw 6D
+  columns are already unit-scaled, so **no** mean/std normalization is applied
+  here (that normalization is tuned to the *object* rotation distribution, not the
+  camera one).
+- **`(trel − shift)/scale` (3)** — the aux camera **centre** in the anchor frame
+  (which is exactly `trel`, the image of the aux origin), expressed in the **same
+  SSI space as the conditioning pointmaps**. So the camera position lives in the
+  identical normalized coordinate system as the 3D points it observes.
+- **Anchor view**: passes `Rrel = I`, `trel = 0`, computed by the *same* path — its
+  rot6d is the identity `[1,0,0,0,1,0]` and its centre normalizes to
+  `−shift/scale`. No special-casing.
+
+There is **no scale component in the transform itself** (camera↔camera is rigid
+SE(3)); the SSI `scale`/`shift` appear only to normalize the camera *position*
+into pointmap-consistent units — they are not a camera scale and not the object
+scale `s`. The rotation never sees them.
+
+### Where it is computed and wired
+| Stage | Location | What happens |
+|---|---|---|
+| build feature | `mvsam3d/data/dexycb_dataset.py` → `_cam_feature(Rrel, trel, shift, scale)` (`CAM_FEAT_DIM = 9`) | packs the 9-D vector; attached to every view dict as `view["cam"]` (anchor uses `I, 0`) |
+| batch | `mvsam3d/data/dexycb_dataset.py` → `collate_multiview` | `cam` stacks per view position to `[B, 9]` with no special handling |
+| consume | `mvsam3d/model/condition.py` → `MultiViewConditionEmbedder.forward` | per view: `tok = tok + self.mlp_cam(view["cam"])[:, None, :]` (broadcast over all tokens of the view) |
+
+### The module
+- `self.mlp_cam = Sequential(Linear(9, cam_hidden_dim), SiLU, Linear(cam_hidden_dim, model_channels))`.
+- **Output layer is zero-initialized** (weight & bias). At step 0 the per-view bias
+  is exactly **0**, so the conditioning equals the no-tag baseline and the
+  warm-started fuser/backbone are perturbed only as `mlp_cam` learns — i.e.
+  resuming/continuing from a prior run sees no init shock.
+- **Trainable.** The whole `EmbedderFuser` (both DINOs, PointPatchEmbed,
+  projection nets, `idx_emb`) stays **frozen** after warm-start; `mlp_cam` is the
+  only trainable condition-side module (and the only from-scratch module in the
+  whole model).
+
+### Config
+- `condition.cam_hidden_dim` (default `256`) — hidden width of the 2-layer MLP.
+- `model.max_views` is now **API-only**: `mlp_cam` derives the per-view bias from
+  the extrinsics, so there is no per-slot table to size and **no upper bound on the
+  view count**.
+
+### Properties / invariants
+- **Camera-consistent under random views**: the same physical camera yields the
+  same embedding regardless of which random slot it lands in.
+- **Unbounded view count**: any number of cameras works (no table to overflow).
+- **Warm-start neutral at step 0**: zero-init output ⇒ identical to the no-camera
+  baseline initially.
+- **Intrinsics not included** (extrinsics only). Appending normalized
+  focal/principal-point would extend the feature to 13-D; deferred as minor for a
+  fixed-rig, pointmap-present DexYCB setup.
+
+### Verification
+Job 4526 (B200): warm-start intact (backbone 936/936, fuser 722/722 frozen),
+`condition trainable modules == ['mlp_cam']` (265,728 params), grad audit
+`fuser=0 / mlp_cam=6.5e3 / backbone=1.75e5`, finite loss, train + validation
+smoke clean. CPU smoke confirmed anchor `rot6d=[1,0,0,0,1,0]`,
+`pos_ssi=[0,0,−median_z/scale]`, and zero-init forward = exactly 0.

migrator/code/mv-sam3d-for-6d-v2/docs/dataset_convention_spec.md

@@ -0,0 +1,44 @@
+# Dataset Coordinate-Convention Spec (verified 2026-07-16/17)
+
+TARGET (what every adapter must output, per view, per object):
+- pose (R_oc, t_oc): `x_cam = R_oc @ p_cad + t_oc`, OpenCV axes (x-right, y-down, z-forward), t METERS, p_cad = mesh verts (m) in mesh frame.
+- K OpenCV pinhole (fx,fy,cx,cy); depth meters registered to RGB; extrinsics world<->cam OpenCV meters; mesh meters in pose frame.
+The base dataloader then applies D=diag(-1,-1,1) (OpenCV->PyTorch3D) and canonical M. Adapters MUST NOT pre-apply D or M.
+
+Legend: ✅=empirical reprojection IoU, 🔵=code/cross-check verified.
+
+## YCB-V 🔵 (train; YCB objs) — LOCAL(models); need BOP train_real
+R_oc=cam_R_m2c.reshape(3,3); t_oc=cam_t_m2c*1e-3; mesh=models_eval/obj_{id:06d}.ply /1000. depth=raw*depth_scale/1000. K=cam_K. extr cam_R/t_w2c (mm->m) world->cam. No flip (native OpenCV). Masks mask_visib/.
+
+## HO-3D 🔵 (HELD-OUT, never train; YCB objs) — LOCAL
+R=Rodrigues(meta['objRot']); t=meta['objTrans'](m); then R=diag(1,-1,-1)@R, t=diag(1,-1,-1)@t (OpenGL->OpenCV, applied to BOTH). mesh dexycb textured_simple.obj (m). depth=(R_ch+G_ch*256)*1.2499e-4. K=meta['camMat']. Eval-split masks rendered.
+
+## HOPE-Video 🔵 (HELD-OUT bonus OOD; non-YCB) — LOCAL
+R=P[:3,:3]; t=P[:3,3]*1e-2 (cm->m). mesh meshes/eval/{class}.obj /100. depth raw*1e-3. extrinsics raw-as-meters world->cam (do NOT scale). K=camera.intrinsics. Masks precomputed masks_visib/.
+
+## YCB-M 🔵 (HELD-OUT; YCB objs) — LOCAL
+R=quat_xyzw->R (F=I, no flip); t=location*1e-2 (cm->m), object in CAMERA frame. mesh models_aligned/ycb_models_aligned_cm /100 (NOT dexycb). depth raw/10000. camera_data=world->cam (invert->cam->world). ⚠️ rotation handedness taken as-is, calib validated translation only — render-verify rotation before any training use.
+
+## HouseCat6D ✅ IoU 0.854 (HELD-OUT #1 benchmark; 194 non-YCB) — LOCAL(sqsh)
+R_oc,t_oc = label pkl rotations[i],translations[i] — ALREADY obj->cam OpenCV meters (identity conversion). mesh obj_models_small_size_final/<cat>/<model>.obj (m, origin-centered; cat=token before '-'). depth /1000. K=intrinsics.txt. extr camera_pose/NNNNNN.txt is CAM->WORLD (invert for world->cam); train/val have NO extrinsics (use pkl directly). ⚠️ masks: bg=255, use `mask!=255` (NOT >0). Index by model_list not class_ids. Do NOT rescale by scales/gt_scales.
+
+## GraspNet-1B ✅ IoU 0.981 (train + test_novel HELD-OUT; ~56 non-YCB of 88) — SAMPLE(scene_0017)
+R_oc=meta['poses'][:3,:3,i]; t_oc=meta['poses'][:3,3,i] (m). mesh models/%03d/nontextured.ply (m). ⚠️ OBJECT-ID OFF-BY-ONE: label/cls_indexes value V -> mesh models/(V-1). depth raw/1000. K=camK.npy (Kinect≠RealSense, read per-scene). extr per-view cam->table=cam0_wrt_table@camera_poses[v] (not needed; per-view poses already in cam frame). CC-BY-NC.
+
+## PACE ✅ IoU 0.90-0.97 (train; 238 non-YCB) — SAMPLE(scene 000215)
+BOP: R_oc=cam_R_m2c.reshape(3,3); t_oc=cam_t_m2c/1000; mesh models_eval/obj_XXXXXX.ply /1000. depth raw*depth_scale/1000 (real depth_scale=1.0; PBR /10000). K=cam_K. ⚠️ real split has NO cam_R/t_w2c (3-cam rig = separate scene folders; per-view poses self-contained). ⚠️ FILTER ARTICULATED: drop obj_id 545..692 (AKB-48); keep 0..544 rigid. masks {imgid:06d}_{gtidx:06d}.png (gtidx into scene_gt).
+
+## HO-Cap ✅ IoU 0.77-0.87 (TRAIN-ONLY pseudo-GT; 64 non-YCB) — SAMPLE(subject_1)
+poses_o.npy [n_obj,n_frame,7]=[qx,qy,qz,qw,tx,ty,tz] scipy xyzw, obj->WORLD (world=tag_1). Per-cam: T_oc(s)=rs_RTs_inv[s]@quat_to_mat(poses_o[o,f]); OR read label_<frame>.npz['obj_poses'] (already per-cam obj->cam, verified 1e-7). rs_RTs[s]=inv(tag_1)@extr2master[s]; rs_RTs_inv=inv. mesh models/<id>/textured_mesh.obj (m, matches poses). depth /1000 with COLOR K. K=color intrinsics. ⚠️ seg_mask ids LOCAL 1..4 objects/5 rh/6 lh. poses/labels in separate zips.
+
+## DexH2R 🔵 round-trip 2.4e-16 (TRAIN-ONLY pseudo-GT, 6 of 18 views; 56 non-YCB) — MESHES+CALIB only (frames quota-blocked)
+obj_pose.pt [T,4,4]=T_world<-obj (world=hand_arm base). Kinect i: T_oc=inv(cali0i)@hand_arm_mesh_to_kinect_pcd_0@obj_pose[t]. RealSense i(0,1): time-varying via ShadowHand FK(qpos.pt)@inv(realsense_to_forearm_i). mesh object_model/<name>.obj (m). depth int16->uint16 /1000. K per-cam (Kinect 8-param distort; images undistorted). ⚠️ ONLY kinect0-3+realsense0-1 have depth (drop 12 ZCAM). No 2D masks (project real_obj_pcd or threshold depth). Use per-trial calibration/.
+
+## H2O 🔵 code-verified (train; 8 non-YCB) — NO DATA (registration wall)
+obj_pose_RT/{f}.txt = class_id + 4x4 obj->cam OpenCV meters, USED AS-IS (⚠️ NO inversion — the "cam_to_obj" note refers to a DIFFERENT file, the keypoint obj_pose/). mesh object/{class}.ply (m). depth /1000. cam_pose/ = cam->world (invert for world->cam). K from cam_intrinsics.txt (fx fy cx cy w h); images undistorted. Per-camera RT already in that cam's frame. Masks must be rendered. Mesh folder names ≠ display names.
+
+## HOGraspNet 🔵 code-verified (train; 22 YCB+8, use scanned meshes; mocap GT) — MESHES only (annotations gated)
+object_mat = 4x4 obj->WORLD (world=mas cam). extrinsic = (3,4) world->cam. T_oc=M4(extrinsic)@object_mat. R_oc=T_oc[:3,:3]; t_oc=T_oc[:3,3]/100 (cm->m). mesh obj_scanned_models/<name>.obj: p_cad=(S*V_raw)/100 where S=_OBJECT_SCALE_FIXED[idx-1] (⚠️ MANDATORY per-obj scale; wine_glass .830, small_marker .1035, spatula .671, golf_ball .430, else 1.0). depth /1000. K OpenCV (full-res 1920x1080; cropped 640x480 -> shift cx,cy by bbox). ⚠️ use scanned meshes NOT dexycb CAD (different frame). obj idx = HOGraspNet 1..30 ordering. masks pseudo.
+
+## ContactPose ✅ IoU 0.979 (TRAIN-ONLY static; 25 non-YCB) — SAMPLE(p28/use/bowl)
+cTo=object_pose(cam,frame)=obj->cam OpenCV meters (R_oc=cTo[:3,:3],t_oc=cTo[:3,3]). mesh object_models/<obj>.ply *1e-3 (PLY is mm). depth /1000 (Kinect v2, ~30mm systematic bias). ⚠️ projection needs affine A: pixels = A@K@x_cam (cp.P); left/right cams are PORTRAIT 540x960 while K is 960x540 landscape. extr wTc=cam->world (invert->cTw). ⚠️ static: sample 1-few frames per (subject,obj,intent) — 2306 grasps not 2.9M frames. Masks render from mesh+cTo. quats w-first if parsing raw.

migrator/code/mv-sam3d-for-6d-v2/mvsam3d/__init__.py

@@ -0,0 +1,9 @@
+"""mvsam3d: multi-view 6D-pose flow model (migrator package).
+
+Importing the package registers the migrator path roots (MIGRATOR_ROOT,
+MIGRATOR_DATA_ROOT, MIGRATOR_CHECKPOINTS, MIGRATOR_CACHE, MIGRATOR_MV_REPO) into
+os.environ so that both YAML configs and helper scripts can reference paths via
+``${MIGRATOR_*}`` placeholders regardless of where the package is installed.
+User-set values are never overridden (setdefault)."""
+from . import paths as paths  # noqa: F401
+paths.register_env()

migrator/code/mv-sam3d-for-6d-v2/mvsam3d/data/adapters.py

@@ -0,0 +1,1402 @@
+"""Per-dataset adapters for MultiViewPoseDataset.
+
+Each adapter converts a dataset's native storage into the UNIFIED contract
+(OpenCV obj->cam meters, depth-m registered to RGB, world/tag->cam extrinsics).
+Conventions are the verified ones in docs/dataset_convention_spec.md.
+
+Design note on extrinsics for the two multi-view regimes:
+  (a) synced rig (DexYCB): extrinsic_cv returns the constant per-serial rig
+      calibration (tag->cam).
+  (b) static scene + moving cam (HouseCat6D, GraspNet, YCB-V, HOPE): the SAMPLE
+      fixes one object; we take the OBJECT'S canonical frame as the shared
+      "world", so extrinsic_cv returns that object's obj->cam pose at the given
+      frame. The object is static, so relative_p3d_from_ext(anchor,aux) yields the
+      exact relative camera transform with NO dependency on a camera-pose file
+      (works for train/val/test uniformly).
+"""
+from __future__ import annotations
+
+import glob
+import os
+from typing import Any, Dict, List, Tuple
+
+import numpy as np
+import torch
+from PIL import Image
+
+from .base_dataset import PoseDatasetAdapter, ViewSample
+from . import dexycb_calib
+from .dexycb_dataset import YCB_ID_TO_NAME, DEPTH_SCALE
+
+
+# --------------------------------------------------------------------------- #
+# canonical-latent cache access (shared)
+# --------------------------------------------------------------------------- #
+class _CanonCache:
+    def __init__(self, cache_dir: str):
+        self.cache_dir = cache_dir
+        self._c: Dict[str, dict] = {}
+
+    def get(self, key: str) -> dict:
+        if key not in self._c:
+            # key may be namespaced "<dataset>/<obj>"; file is <cache>/<key>.pt
+            path = os.path.join(self.cache_dir, f"{key}.pt")
+            if not os.path.exists(path):
+                ds = key.split("/", 1)[0] if "/" in key else "<dataset>"
+                raise FileNotFoundError(
+                    f"canonical latent not built for key '{key}' (expected {path}). "
+                    f"Run: python tools/build_canonical_latents_multi.py --dataset {ds} "
+                    f"--root <dataset_root>  before training on this dataset."
+                )
+            self._c[key] = torch.load(path, map_location="cpu", weights_only=False)
+        return self._c[key]
+
+
+def _bop_gt_index(entries, oid, rank=0):
+    """Positional index of the rank-th entry with obj_id==oid in a BOP scene_gt
+    frame list; None if that instance is absent. Used so aux views resolve the
+    SAME physical object by (obj_id, occurrence-rank) instead of trusting the
+    anchor's positional index across frames (which breaks if scene_gt ordering
+    differs). Mirrors GraspNet's index-by-class approach, rank-aware for scenes
+    with duplicate instances of one obj_id."""
+    cnt = 0
+    for idx, o in enumerate(entries):
+        if int(o["obj_id"]) == oid:
+            if cnt == rank:
+                return idx
+            cnt += 1
+    return None
+
+
+# --------------------------------------------------------------------------- #
+# DexYCB (reference adapter — reproduces MultiViewDexYCB exactly)
+# --------------------------------------------------------------------------- #
+class DexYCBAdapter(PoseDatasetAdapter):
+    name = "dexycb"
+
+    def __init__(self, root: str, canonical_cache: str, *, frame_stride: int = 5,
+                 anchor_sweep: bool = True, subjects=None, depth_scale: float = DEPTH_SCALE):
+        self.root = root
+        self.calib_root = os.path.join(root, "calibration")
+        self.canon = _CanonCache(canonical_cache)
+        self.frame_stride = frame_stride
+        self.anchor_sweep = anchor_sweep
+        self.subjects = subjects
+        self.depth_scale = depth_scale
+        self._ext: Dict[str, dict] = {}
+
+    def _extrinsics(self, tag):
+        if tag not in self._ext:
+            self._ext[tag] = dexycb_calib.load_extrinsics(self.calib_root, tag)
+        return self._ext[tag]
+
+    def list_samples(self) -> List[ViewSample]:
+        seq_dirs = sorted(glob.glob(os.path.join(self.root, "2020*-subject-*", "2020*")))
+        if self.subjects is not None:
+            seq_dirs = [d for d in seq_dirs if any(s in d for s in self.subjects)]
+        out: List[ViewSample] = []
+        for seq_dir in seq_dirs:
+            meta = dexycb_calib._load_yaml(os.path.join(seq_dir, "meta.yml"))
+            serials = [str(s) for s in meta["serials"]]
+            ycb_ids = list(meta["ycb_ids"])
+            tag = str(meta["extrinsics"])
+            n = int(meta["num_frames"])
+            anchors = serials if self.anchor_sweep else serials[:1]
+            for frame in range(0, n, self.frame_stride):
+                for anchor in anchors:
+                    aux = [s for s in serials if s != anchor]
+                    for oi, oid in enumerate(ycb_ids):
+                        if int(oid) not in YCB_ID_TO_NAME:
+                            continue
+                        obj_key = YCB_ID_TO_NAME[int(oid)]
+                        aref = (seq_dir, anchor, frame, int(oid), oi, tag)
+                        pool = [(seq_dir, s, frame, int(oid), oi, tag) for s in aux]
+                        out.append(ViewSample(anchor=aref, aux_pool=pool, obj_key=obj_key))
+        return out
+
+    def load_view(self, ref: Any) -> Dict[str, Any]:
+        seq_dir, serial, frame, obj_id, obj_index, tag = ref
+        rgb = np.asarray(Image.open(os.path.join(seq_dir, serial, f"color_{frame:06d}.jpg"))
+                         .convert("RGB"), dtype=np.float32) / 255.0
+        seg = np.load(os.path.join(seq_dir, serial, f"labels_{frame:06d}.npz"))["seg"]
+        mask = (seg == obj_id).astype(np.float32)
+        depth = np.asarray(Image.open(os.path.join(
+            seq_dir, serial, f"aligned_depth_to_color_{frame:06d}.png")),
+            dtype=np.float32) / self.depth_scale
+        intr = dexycb_calib.load_intrinsics(self.calib_root, serial)
+        return {"rgb": rgb, "mask": mask, "depth_m": depth, "intr": intr}
+
+    def extrinsic_cv(self, ref: Any) -> Dict[str, np.ndarray]:
+        seq_dir, serial, frame, obj_id, obj_index, tag = ref
+        e = self._extrinsics(tag)[serial]
+        return {"R": e["R"], "t": e["t"]}
+
+    def object_pose_cv(self, sample: ViewSample) -> Tuple[np.ndarray, np.ndarray]:
+        seq_dir, serial, frame, obj_id, obj_index, tag = sample.anchor
+        py = np.load(os.path.join(seq_dir, serial, f"labels_{frame:06d}.npz"))["pose_y"][obj_index]
+        return py[:, :3].astype(np.float64), py[:, 3].astype(np.float64)
+
+    def mesh_for(self, sample: ViewSample):
+        return _load_mesh(os.path.join(self.root, "models", sample.obj_key, "textured_simple.obj"))
+
+    def canonical(self, obj_key: str) -> dict:
+        return self.canon.get(obj_key)
+
+
+# --------------------------------------------------------------------------- #
+# shared helpers for the new adapters
+# --------------------------------------------------------------------------- #
+import json as _json
+import trimesh as _trimesh
+
+def _K_to_intr(K):
+    return {"fx": float(K[0, 0]), "fy": float(K[1, 1]),
+            "ppx": float(K[0, 2]), "ppy": float(K[1, 2])}
+
+def _load_mesh(path):
+    m = _trimesh.load(path, process=False, force="mesh")
+    if isinstance(m, _trimesh.Scene):
+        m = _trimesh.util.concatenate([g for g in m.geometry.values()])
+    return m
+
+
+# --------------------------------------------------------------------------- #
+# HouseCat6D  (type b, HELD-OUT; obj->cam OpenCV meters == identity conversion)
+# --------------------------------------------------------------------------- #
+class HouseCat6DAdapter(PoseDatasetAdapter):
+    name = "housecat6d"
+
+    def __init__(self, root, canonical_cache, *, split="test", max_aux_frames=8, frame_stride=10):
+        # root = .../housecat6d/extracted ; split dir under it (test/train/val)
+        self.split_dir = os.path.join(root, split)
+        self.canon = _CanonCache(canonical_cache)
+        self.max_aux_frames = max_aux_frames
+        self.frame_stride = frame_stride
+        self._pkl_cache = {}
+
+    def _pkl(self, scene, frame):
+        key = (scene, frame)
+        if key not in self._pkl_cache:
+            import pickle
+            with open(os.path.join(scene, "labels", f"{frame:06d}_label.pkl"), "rb") as f:
+                self._pkl_cache[key] = pickle.load(f)
+        return self._pkl_cache[key]
+
+    def _frames(self, scene):
+        return sorted(int(os.path.splitext(f)[0]) for f in os.listdir(os.path.join(scene, "rgb")))
+
+    def list_samples(self):
+        out = []
+        scenes = sorted(glob.glob(os.path.join(self.split_dir, "*")))
+        for scene in scenes:
+            if not os.path.isdir(os.path.join(scene, "rgb")):
+                continue
+            frames = self._frames(scene)
+            if len(frames) < 2:
+                continue
+            models = self._pkl(scene, frames[0])["model_list"]
+            for oi, model in enumerate(models):
+                anchors = frames[:: self.frame_stride] or [frames[0]]
+                for af in anchors:
+                    pool = [f for f in frames if f != af]
+                    out.append(ViewSample(
+                        anchor={"scene": scene, "frame": af, "oi": oi, "model": model},
+                        aux_pool=[{"scene": scene, "frame": f, "oi": oi, "model": model} for f in pool],
+                        obj_key=f"housecat6d/{model}",
+                        meta={"scene": scene, "model": model}))
+        return out
+
+    def load_view(self, ref):
+        scene, frame, model = ref["scene"], ref["frame"], ref["model"]
+        rgb = np.asarray(Image.open(os.path.join(scene, "rgb", f"{frame:06d}.png")).convert("RGB"),
+                         np.float32) / 255.0
+        inst = np.asarray(Image.open(os.path.join(scene, "instance", f"{frame:06d}_{model}.png")))
+        mask = (inst != 255).astype(np.float32)              # bg=255
+        depth = np.asarray(Image.open(os.path.join(scene, "depth", f"{frame:06d}.png")), np.float32) / 1000.0
+        K = np.loadtxt(os.path.join(scene, "intrinsics.txt")).reshape(3, 3)
+        return {"rgb": rgb, "mask": mask, "depth_m": depth, "intr": _K_to_intr(K)}
+
+    def extrinsic_cv(self, ref):  # world := object frame -> obj->cam pose
+        d = self._pkl(ref["scene"], ref["frame"])
+        return {"R": np.asarray(d["rotations"][ref["oi"]], np.float64),
+                "t": np.asarray(d["translations"][ref["oi"]], np.float64)}
+
+    def object_pose_cv(self, sample):
+        e = self.extrinsic_cv(sample.anchor)
+        return e["R"], e["t"]
+
+    def mesh_for(self, sample):
+        model = sample.meta["model"]; cat = model.split("-")[0]
+        base = os.path.dirname(os.path.dirname(self.split_dir))  # .../housecat6d
+        return _load_mesh(os.path.join(base, "extracted", "obj_models_small_size_final",
+                                       cat, f"{model}.obj"))
+
+    def canonical(self, obj_key):
+        return self.canon.get(obj_key)
+
+
+# --------------------------------------------------------------------------- #
+# PACE  (BOP; real split; obj->cam OpenCV mm). rigid-only (drop obj_id 545..692)
+# NOTE: real multi-view = the 3 rig cams (separate scene folders per scene/video);
+# a single-cam scene validates convention but not cross-cam grouping.
+# --------------------------------------------------------------------------- #
+class PACEAdapter(PoseDatasetAdapter):
+    name = "pace"
+
+    def __init__(self, root, canonical_cache, *, split="test", depth_scale=1.0, frame_stride=10, max_aux_frames=8):
+        self.split_dir = os.path.join(root, split)
+        self.models_dir = os.path.join(root, "models_eval")
+        self.canon = _CanonCache(canonical_cache)
+        self.depth_scale = depth_scale
+        self.frame_stride = frame_stride
+        self._gt = {}; self._cam = {}
+
+    def _scene_json(self, scene, which):
+        cache = self._gt if which == "gt" else self._cam
+        if scene not in cache:
+            cache[scene] = _json.load(open(os.path.join(scene, f"scene_{'gt' if which=='gt' else 'camera'}.json")))
+        return cache[scene]
+
+    def list_samples(self):
+        out = []
+        for scene in sorted(glob.glob(os.path.join(self.split_dir, "*"))):
+            if not os.path.isdir(os.path.join(scene, "rgb")):
+                continue
+            gt = self._scene_json(scene, "gt")
+            imids = sorted(int(k) for k in gt.keys())
+            for imid in imids[:: self.frame_stride] or imids[:1]:
+                entries = gt[str(imid)]
+                for gi, o in enumerate(entries):
+                    oid = int(o["obj_id"])
+                    if 545 <= oid <= 692:            # articulated AKB-48 -> drop
+                        continue
+                    # occurrence-rank of this instance within its frame
+                    rank = sum(1 for o2 in entries[:gi] if int(o2["obj_id"]) == oid)
+                    # aux = only frames that actually contain this (oid,rank) instance
+                    pool = [{"scene": scene, "imid": j, "oid": oid, "rank": rank}
+                            for j in imids if j != imid
+                            and _bop_gt_index(gt[str(j)], oid, rank) is not None]
+                    ref = {"scene": scene, "imid": imid, "oid": oid, "rank": rank}
+                    out.append(ViewSample(anchor=ref, aux_pool=pool, obj_key=f"pace/{oid:06d}",
+                                          meta={"oid": oid}))
+        return out
+
+    def _gi(self, ref):
+        idx = _bop_gt_index(self._scene_json(ref["scene"], "gt")[str(ref["imid"])],
+                            ref["oid"], ref.get("rank", 0))
+        if idx is None:
+            raise KeyError(f"pace: obj {ref['oid']} rank {ref.get('rank',0)} not in "
+                           f"{ref['scene']} frame {ref['imid']}")
+        return idx
+
+    def load_view(self, ref):
+        scene, imid = ref["scene"], ref["imid"]
+        rgb = np.asarray(Image.open(os.path.join(scene, "rgb", f"{imid:06d}.png")).convert("RGB"), np.float32) / 255.0
+        depth = np.asarray(Image.open(os.path.join(scene, "depth", f"{imid:06d}.png")), np.float32)
+        cam = self._scene_json(scene, "camera")[str(imid)]
+        ds = float(cam.get("depth_scale", self.depth_scale))
+        depth_m = depth * ds / 1000.0
+        gi = self._gi(ref)
+        mp = os.path.join(scene, "mask_visib", f"{imid:06d}_{gi:06d}.png")
+        mask = (np.asarray(Image.open(mp)) > 0).astype(np.float32) if os.path.exists(mp) else np.zeros(depth.shape, np.float32)
+        K = np.asarray(cam["cam_K"], np.float64).reshape(3, 3)
+        return {"rgb": rgb, "mask": mask, "depth_m": depth_m, "intr": _K_to_intr(K)}
+
+    def _pose(self, ref):
+        o = self._scene_json(ref["scene"], "gt")[str(ref["imid"])][self._gi(ref)]
+        R = np.asarray(o["cam_R_m2c"], np.float64).reshape(3, 3)
+        t = np.asarray(o["cam_t_m2c"], np.float64) / 1000.0
+        return R, t
+
+    def extrinsic_cv(self, ref):
+        R, t = self._pose(ref); return {"R": R, "t": t}
+
+    def object_pose_cv(self, sample):
+        return self._pose(sample.anchor)
+
+    def mesh_for(self, sample):
+        m = _load_mesh(os.path.join(self.models_dir, f"obj_{sample.meta['oid']:06d}.ply"))
+        m.apply_scale(1e-3)                # mm -> m
+        return m
+
+    def canonical(self, obj_key):
+        return self.canon.get(obj_key)
+
+
+# --------------------------------------------------------------------------- #
+# GraspNet-1B  (type b; obj->cam OpenCV meters; mesh id = cls_value - 1)
+# --------------------------------------------------------------------------- #
+class GraspNetAdapter(PoseDatasetAdapter):
+    name = "graspnet"
+
+    def __init__(self, root, canonical_cache, *, camera="kinect", frame_stride=20, max_aux_frames=8):
+        self.scenes_dir = os.path.join(root, "scenes")
+        self.models_dir = os.path.join(root, "models")
+        self.canon = _CanonCache(canonical_cache)
+        self.camera = camera
+        self.frame_stride = frame_stride
+        self._meta = {}
+
+    def _load_meta(self, cdir, v):
+        key = (cdir, v)
+        if key not in self._meta:
+            import scipy.io as sio
+            self._meta[key] = sio.loadmat(os.path.join(cdir, "meta", f"{v:04d}.mat"))
+        return self._meta[key]
+
+    def list_samples(self):
+        out = []
+        for scene in sorted(glob.glob(os.path.join(self.scenes_dir, "scene_*"))):
+            cdir = os.path.join(scene, self.camera)
+            if not os.path.isdir(os.path.join(cdir, "rgb")):
+                continue
+            views = sorted(int(os.path.splitext(f)[0]) for f in os.listdir(os.path.join(cdir, "rgb")))
+            m0 = self._load_meta(cdir, views[0])
+            cls = [int(c) for c in m0["cls_indexes"].ravel()]
+            anchors = views[:: self.frame_stride] or [views[0]]
+            for av in anchors:
+                mv = self._load_meta(cdir, av)
+                clsv = [int(c) for c in mv["cls_indexes"].ravel()]
+                for i, V in enumerate(clsv):
+                    pool = [{"cdir": cdir, "v": w, "cls": V} for w in views if w != av]
+                    out.append(ViewSample(
+                        anchor={"cdir": cdir, "v": av, "cls": V, "i": i},
+                        aux_pool=pool, obj_key=f"graspnet/{V-1:03d}", meta={"V": V}))
+        return out
+
+    def load_view(self, ref):
+        cdir, v = ref["cdir"], ref["v"]
+        rgb = np.asarray(Image.open(os.path.join(cdir, "rgb", f"{v:04d}.png")).convert("RGB"), np.float32) / 255.0
+        depth = np.asarray(Image.open(os.path.join(cdir, "depth", f"{v:04d}.png")), np.float32) / 1000.0
+        label = np.asarray(Image.open(os.path.join(cdir, "label", f"{v:04d}.png")))
+        mask = (label == ref["cls"]).astype(np.float32)
+        m = self._load_meta(cdir, v)
+        K = np.asarray(m["intrinsic_matrix"], np.float64).reshape(3, 3)
+        return {"rgb": rgb, "mask": mask, "depth_m": depth, "intr": _K_to_intr(K)}
+
+    def _pose(self, cdir, v, cls):
+        m = self._load_meta(cdir, v)
+        clsv = [int(c) for c in m["cls_indexes"].ravel()]
+        i = clsv.index(cls)
+        P = m["poses"][:, :, i]
+        return P[:, :3].astype(np.float64), P[:, 3].astype(np.float64)
+
+    def extrinsic_cv(self, ref):
+        R, t = self._pose(ref["cdir"], ref["v"], ref["cls"]); return {"R": R, "t": t}
+
+    def object_pose_cv(self, sample):
+        a = sample.anchor
+        return self._pose(a["cdir"], a["v"], a["cls"])
+
+    def mesh_for(self, sample):
+        return _load_mesh(os.path.join(self.models_dir, f"{sample.meta['V']-1:03d}", "nontextured.ply"))
+
+    def canonical(self, obj_key):
+        return self.canon.get(obj_key)
+
+
+# --------------------------------------------------------------------------- #
+# YCB-V  (BOP; obj->cam OpenCV mm; frames-as-views static; YCB objs reuse latents)
+# --------------------------------------------------------------------------- #
+class YCBVAdapter(PoseDatasetAdapter):
+    name = "ycbv"
+    _BOP2NAME = None
+
+    def __init__(self, root, canonical_cache, *, split="test", frame_stride=15, max_aux_frames=8):
+        self.split_dir = os.path.join(root, split)
+        self.models_dir = os.path.join(root, "models_eval")
+        self.canon = _CanonCache(canonical_cache)
+        self.frame_stride = frame_stride
+        self._gt = {}; self._cam = {}
+        if YCBVAdapter._BOP2NAME is None:
+            YCBVAdapter._BOP2NAME = {i + 1: n for i, n in enumerate([
+                "002_master_chef_can","003_cracker_box","004_sugar_box","005_tomato_soup_can",
+                "006_mustard_bottle","007_tuna_fish_can","008_pudding_box","009_gelatin_box",
+                "010_potted_meat_can","011_banana","019_pitcher_base","021_bleach_cleanser",
+                "024_bowl","025_mug","035_power_drill","036_wood_block","037_scissors",
+                "040_large_marker","051_large_clamp","052_extra_large_clamp","061_foam_brick"])}
+
+    def _sj(self, scene, which):
+        c = self._gt if which == "gt" else self._cam
+        if scene not in c:
+            c[scene] = _json.load(open(os.path.join(scene, f"scene_{'gt' if which=='gt' else 'camera'}.json")))
+        return c[scene]
+
+    def list_samples(self):
+        out = []
+        for scene in sorted(glob.glob(os.path.join(self.split_dir, "*"))):
+            if not os.path.isdir(os.path.join(scene, "rgb")):
+                continue
+            gt = self._sj(scene, "gt")
+            imids = sorted(int(k) for k in gt.keys())
+            for imid in imids[:: self.frame_stride] or imids[:1]:
+                entries = gt[str(imid)]
+                for gi, o in enumerate(entries):
+                    oid = int(o["obj_id"])
+                    name = self._BOP2NAME.get(oid)
+                    if name is None:
+                        continue
+                    rank = sum(1 for o2 in entries[:gi] if int(o2["obj_id"]) == oid)
+                    pool = [{"scene": scene, "imid": j, "oid": oid, "rank": rank}
+                            for j in imids if j != imid
+                            and _bop_gt_index(gt[str(j)], oid, rank) is not None]
+                    out.append(ViewSample(
+                        anchor={"scene": scene, "imid": imid, "oid": oid, "rank": rank},
+                        aux_pool=pool, obj_key=name, meta={"oid": oid}))
+        return out
+
+    def _gi(self, ref):
+        idx = _bop_gt_index(self._sj(ref["scene"], "gt")[str(ref["imid"])],
+                            ref["oid"], ref.get("rank", 0))
+        if idx is None:
+            raise KeyError(f"ycbv: obj {ref['oid']} rank {ref.get('rank',0)} not in "
+                           f"{ref['scene']} frame {ref['imid']}")
+        return idx
+
+    def load_view(self, ref):
+        scene, imid = ref["scene"], ref["imid"]
+        rgb = np.asarray(Image.open(os.path.join(scene, "rgb", f"{imid:06d}.png")).convert("RGB"), np.float32) / 255.0
+        cam = self._sj(scene, "camera")[str(imid)]
+        ds = float(cam.get("depth_scale", 1.0))
+        depth = np.asarray(Image.open(os.path.join(scene, "depth", f"{imid:06d}.png")), np.float32) * ds / 1000.0
+        gi = self._gi(ref)
+        mp = os.path.join(scene, "mask_visib", f"{imid:06d}_{gi:06d}.png")
+        mask = (np.asarray(Image.open(mp)) > 0).astype(np.float32) if os.path.exists(mp) else np.zeros(depth.shape, np.float32)
+        K = np.asarray(cam["cam_K"], np.float64).reshape(3, 3)
+        return {"rgb": rgb, "mask": mask, "depth_m": depth, "intr": _K_to_intr(K)}
+
+    def _pose(self, ref):
+        o = self._sj(ref["scene"], "gt")[str(ref["imid"])][self._gi(ref)]
+        return (np.asarray(o["cam_R_m2c"], np.float64).reshape(3, 3),
+                np.asarray(o["cam_t_m2c"], np.float64) / 1000.0)
+
+    def extrinsic_cv(self, ref):
+        R, t = self._pose(ref); return {"R": R, "t": t}
+
+    def object_pose_cv(self, sample):
+        return self._pose(sample.anchor)
+
+    def mesh_for(self, sample):
+        m = _load_mesh(os.path.join(self.models_dir, f"obj_{sample.meta['oid']:06d}.ply"))
+        m.apply_scale(1e-3)
+        return m
+
+    def canonical(self, obj_key):
+        return self.canon.get(obj_key)
+
+
+# --------------------------------------------------------------------------- #
+# STUBS / notes for adapters whose frame data is blocked or needs extra handling
+# --------------------------------------------------------------------------- #
+class _NotReadyAdapter(PoseDatasetAdapter):
+    """Placeholder: convention verified, adapter deferred. See reason."""
+    def __init__(self, *a, reason="", **k): self._reason = reason
+    def list_samples(self): raise NotImplementedError(self._reason)
+    def load_view(self, ref): raise NotImplementedError(self._reason)
+    def extrinsic_cv(self, ref): raise NotImplementedError(self._reason)
+    def object_pose_cv(self, sample): raise NotImplementedError(self._reason)
+    def canonical(self, obj_key): raise NotImplementedError(self._reason)
+
+# --------------------------------------------------------------------------- #
+# HO-Cap  (type a, 8 synced RealSense; TRAIN-ONLY pseudo-GT). world := object
+# frame (same pattern as the type-b adapters): each view's obj->cam comes from
+# that camera's per-frame label npz (verified == official ope_gt to 1e-7), so no
+# calibration-file compose is needed and relative_p3d_from_ext(anchor,aux) yields
+# the exact cross-camera transform. object_ids[oi] <-> obj_poses[oi] <-> seg_mask
+# id (oi+1), confirmed on disk.
+# --------------------------------------------------------------------------- #
+class HOCapAdapter(PoseDatasetAdapter):
+    name = "hocap"
+
+    def __init__(self, root, canonical_cache, *, frame_stride=15, subjects=None):
+        self.root = root                                   # dataset/hocap
+        self.canon = _CanonCache(canonical_cache)
+        self.frame_stride = frame_stride
+        self.subjects = subjects
+        self._lab = {}
+
+    def _seqs(self):
+        out = []
+        for subj in sorted(glob.glob(os.path.join(self.root, "subject_*"))):
+            if os.path.isfile(subj):
+                continue
+            if self.subjects and os.path.basename(subj) not in self.subjects:
+                continue
+            for seq in sorted(glob.glob(os.path.join(subj, "2*"))):
+                if os.path.exists(os.path.join(seq, "meta.yaml")):
+                    out.append(seq)
+        return out
+
+    def _meta(self, seq):
+        import yaml
+        with open(os.path.join(seq, "meta.yaml")) as f:
+            return yaml.safe_load(f)
+
+    def _label(self, seq, serial, frame):
+        k = (seq, serial, frame)
+        if k not in self._lab:
+            self._lab[k] = np.load(os.path.join(seq, serial, f"label_{frame:06d}.npz"))
+        return self._lab[k]
+
+    @staticmethod
+    def _present_frames(d):
+        """Integer frames under serial dir `d` that have color+depth+label ALL
+        present (so every needed file loads). Robust to sparse/non-contiguous
+        frame ids and partially-extracted data."""
+        fr = None
+        try:
+            names = os.listdir(d)
+        except FileNotFoundError:
+            return set()
+        for pre, suf in (("color_", ".jpg"), ("depth_", ".png"), ("label_", ".npz")):
+            s = set()
+            for f in names:
+                if f.startswith(pre) and f.endswith(suf):
+                    try:
+                        s.add(int(f[len(pre):-len(suf)]))
+                    except ValueError:
+                        pass
+            fr = s if fr is None else (fr & s)
+        return fr or set()
+
+    def list_samples(self):
+        out = []
+        for seq in self._seqs():
+            m = self._meta(seq)
+            serials = [str(s) for s in m["realsense"]["serials"]]
+            obj_ids = list(m["object_ids"])
+            # Frames usable by EVERY serial (each view's color/depth/label exist).
+            common = None
+            for s in serials:
+                fs = self._present_frames(os.path.join(seq, s))
+                common = fs if common is None else (common & fs)
+            frames = sorted(common or set())[:: self.frame_stride]
+            for frame in frames:
+                for oi, obj in enumerate(obj_ids):
+                    for anchor in serials:
+                        aux = [s for s in serials if s != anchor]
+                        aref = (seq, anchor, frame, oi, obj)
+                        pool = [(seq, s, frame, oi, obj) for s in aux]
+                        out.append(ViewSample(anchor=aref, aux_pool=pool,
+                                              obj_key=f"hocap/{obj}", meta={"obj": obj, "oi": oi}))
+        return out
+
+    def load_view(self, ref):
+        seq, serial, frame, oi, obj = ref
+        rgb = np.asarray(Image.open(os.path.join(seq, serial, f"color_{frame:06d}.jpg")).convert("RGB"),
+                         np.float32) / 255.0
+        depth = np.asarray(Image.open(os.path.join(seq, serial, f"depth_{frame:06d}.png")), np.float32) / 1000.0
+        lab = self._label(seq, serial, frame)
+        mask = (lab["seg_mask"] == (oi + 1)).astype(np.float32)          # local ids 1..n
+        K = np.asarray(lab["cam_K"], np.float64).reshape(3, 3)           # color K (depth aligned to color)
+        return {"rgb": rgb, "mask": mask, "depth_m": depth, "intr": _K_to_intr(K)}
+
+    def _pose(self, seq, serial, frame, oi):
+        T = np.asarray(self._label(seq, serial, frame)["obj_poses"][oi], np.float64)  # obj->cam
+        return T[:3, :3], T[:3, 3]
+
+    def extrinsic_cv(self, ref):
+        seq, serial, frame, oi, obj = ref
+        R, t = self._pose(seq, serial, frame, oi)
+        return {"R": R, "t": t}
+
+    def object_pose_cv(self, sample):
+        seq, serial, frame, oi, obj = sample.anchor
+        return self._pose(seq, serial, frame, oi)
+
+    def mesh_for(self, sample):
+        return _load_mesh(os.path.join(self.root, "models", sample.meta["obj"], "textured_mesh.obj"))
+
+    def canonical(self, obj_key):
+        return self.canon.get(obj_key)
+
+
+# --------------------------------------------------------------------------- #
+# ContactPose  (type a, 3 synced Kinect v2; TRAIN-ONLY, static grasps). cTo is
+# obj->cam OpenCV meters; mesh PLY is mm (*1e-3). The stored images are in a
+# per-camera orientation (left/right PORTRAIT 540x960, middle rot180) while the
+# intrinsics K are always in the LANDSCAPE 960x540 frame with the affine A
+# mapping landscape->stored (utilities/misc.get_A). We inverse-warp each stored
+# rgb/depth back to the landscape frame so the pipeline's diagonal-K unprojection
+# is exact; masks are rendered from mesh+cTo+K in that same landscape frame.
+# world := object frame (per-view extrinsic = that view's cTo). Static -> sample
+# a few frames per grasp (default), not all ~558.
+# --------------------------------------------------------------------------- #
+class ContactPoseAdapter(PoseDatasetAdapter):
+    name = "contactpose"
+
+    def __init__(self, root, canonical_cache, *, frames_per_grasp=4, frame_stride=None):
+        self.root = root                                       # dataset/contactpose
+        self.data_dir = os.path.join(root, "contactpose_data")
+        self.models_dir = os.path.join(root, "object_models")
+        self.canon = _CanonCache(canonical_cache)
+        self.frames_per_grasp = frames_per_grasp
+        self._ann = {}
+        self._meshc = {}
+
+    def _annot(self, gdir):
+        if gdir not in self._ann:
+            self._ann[gdir] = _json.load(open(os.path.join(gdir, "annotations.json")))
+        return self._ann[gdir]
+
+    @staticmethod
+    def _q2m(q):                                               # transforms3d wxyz -> R
+        from scipy.spatial.transform import Rotation
+        return Rotation.from_quat([q[1], q[2], q[3], q[0]]).as_matrix()
+
+    @classmethod
+    def _pm(cls, p):
+        T = np.eye(4); T[:3, :3] = cls._q2m(p["rotation"]); T[:3, 3] = p["translation"]; return T
+
+    @staticmethod
+    def _get_A(cam):                                           # landscape(960x540) -> stored
+        def flipud(H): return np.array([[1, 0, 0], [0, -1, H], [0, 0, 1]], float)
+        def fliplr(W): return np.array([[-1, 0, W], [0, 1, 0], [0, 0, 1]], float)
+        def transpose(): return np.array([[0, 1, 0], [1, 0, 0], [0, 0, 1]], float)
+        if cam == "kinect2_left":   return fliplr(540) @ transpose()
+        if cam == "kinect2_right":  return flipud(960) @ transpose()
+        if cam == "kinect2_middle": return fliplr(960) @ flipud(540)
+        raise NotImplementedError(cam)
+
+    def list_samples(self):
+        out = []
+        for gdir in sorted(glob.glob(os.path.join(self.data_dir, "*", "*"))):
+            if not os.path.exists(os.path.join(gdir, "annotations.json")):
+                continue
+            ann = self._annot(gdir); obj = ann["object"]
+            cams = [c for c, cv in ann["cameras"].items() if cv.get("valid", True)]
+            if len(cams) < 2:
+                continue
+            nfr = len(ann["frames"])
+            if nfr == 0:
+                continue
+            idxs = np.unique(np.linspace(0, nfr - 1, self.frames_per_grasp).round().astype(int))
+            for fi in idxs:
+                for anchor in cams:
+                    aux = [c for c in cams if c != anchor]
+                    aref = (gdir, anchor, int(fi), obj)
+                    pool = [(gdir, c, int(fi), obj) for c in aux]
+                    out.append(ViewSample(anchor=aref, aux_pool=pool,
+                                          obj_key=f"contactpose/{obj}", meta={"obj": obj}))
+        return out
+
+    def _K(self, gdir, cam):
+        k = self._annot(gdir)["cameras"][cam]["K"]
+        return np.array([[k["fx"], 0, k["cx"]], [0, k["fy"], k["cy"]], [0, 0, 1]], float)
+
+    def _cTo(self, gdir, cam, fi):
+        ann = self._annot(gdir)
+        wTc = self._pm(ann["cameras"][cam]["wTc"])
+        oTw = self._pm(ann["frames"][fi]["oTw"])
+        return np.linalg.inv(oTw @ wTc)                        # obj->cam
+
+    @staticmethod
+    def _unrotate(img, A, Wl=960, Hl=540):
+        """Resample the stored image into the landscape (Wl x Hl) frame:
+        landscape[v_l,u_l] = stored[ round(A @ [u_l,v_l,1]) ]. Correct for any A
+        (rot180 / transpose+flip) by construction; nearest-neighbour (NaN/edge safe)."""
+        us, vs = np.meshgrid(np.arange(Wl), np.arange(Hl))
+        land = np.stack([us.ravel(), vs.ravel(), np.ones(Wl * Hl)], 0)      # 3xN
+        st = A @ land
+        su = np.round(st[0]).astype(int); sv = np.round(st[1]).astype(int)
+        Hs, Ws = img.shape[:2]
+        ok = (su >= 0) & (su < Ws) & (sv >= 0) & (sv < Hs)
+        out = np.zeros((Hl * Wl,) + img.shape[2:], img.dtype)
+        src = img.reshape((Hs * Ws,) + img.shape[2:])
+        out[ok] = src[sv[ok] * Ws + su[ok]]
+        return out.reshape((Hl, Wl) + img.shape[2:])
+
+    def _mesh(self, obj):
+        if obj not in self._meshc:
+            m = _load_mesh(os.path.join(self.models_dir, f"{obj}.ply"))
+            m.apply_scale(1e-3)                                 # mm -> m
+            self._meshc[obj] = m
+        return self._meshc[obj]
+
+    def load_view(self, ref):
+        gdir, cam, fi, obj = ref
+        A = self._get_A(cam)
+        cdir = os.path.join(gdir, "images_full", cam)
+        rgb_s = np.asarray(Image.open(os.path.join(cdir, "color", f"frame{fi:03d}.png")).convert("RGB"),
+                           np.float32) / 255.0
+        dep_s = np.asarray(Image.open(os.path.join(cdir, "depth", f"frame{fi:03d}.png")), np.float32) / 1000.0
+        rgb = self._unrotate(rgb_s, A); depth = self._unrotate(dep_s, A)
+        K = self._K(gdir, cam)
+        H, W = rgb.shape[:2]
+        # render object silhouette (landscape frame) from mesh + cTo + K
+        cTo = self._cTo(gdir, cam, fi)
+        V = np.asarray(self._mesh(obj).vertices)
+        Xc = (cTo[:3, :3] @ V.T + cTo[:3, 3:4]).T
+        z = Xc[:, 2]; f = z > 1e-3
+        uv = (K @ Xc[f].T).T
+        u = np.round(uv[:, 0] / uv[:, 2]).astype(int); v = np.round(uv[:, 1] / uv[:, 2]).astype(int)
+        mask = np.zeros((H, W), np.float32)
+        ok = (u >= 0) & (u < W) & (v >= 0) & (v < H)
+        mask[v[ok], u[ok]] = 1.0
+        try:
+            import scipy.ndimage as _ndi
+            mask = _ndi.binary_fill_holes(_ndi.binary_dilation(mask > 0, iterations=3)).astype(np.float32)
+        except Exception:
+            pass
+        return {"rgb": rgb, "mask": mask, "depth_m": depth, "intr": _K_to_intr(K)}
+
+    def extrinsic_cv(self, ref):
+        gdir, cam, fi, obj = ref
+        cTo = self._cTo(gdir, cam, fi)
+        return {"R": cTo[:3, :3], "t": cTo[:3, 3]}
+
+    def object_pose_cv(self, sample):
+        gdir, cam, fi, obj = sample.anchor
+        cTo = self._cTo(gdir, cam, fi)
+        return cTo[:3, :3], cTo[:3, 3]
+
+    def mesh_for(self, sample):
+        return self._mesh(sample.meta["obj"])
+
+    def canonical(self, obj_key):
+        return self.canon.get(obj_key)
+# --------------------------------------------------------------------------- #
+# H2O  (type a-like: 4 static synced RGB-D cams cam0..cam3 + 1 ego cam4 [skipped];
+# TRAIN, 8 non-YCB objects). Per README+h2oplayer:
+#   camN/obj_pose_RT/{f:06d}.txt = class_id + 4x4 obj->cam OpenCV METERS, USED AS-IS
+#     (NO inversion; the "cam_to_obj" note in README refers to the DIFFERENT
+#      keypoint file obj_pose/, not obj_pose_RT/).
+#   camN/cam_pose/{f:06d}.txt      = 4x4 cam->world  (invert -> world->cam extrinsic)
+#   camN/cam_intrinsics.txt        = fx fy cx cy w h (images undistorted)
+#   camN/depth/{f:06d}.png         = uint16 mm  (/1000 -> meters, registered to rgb)
+#   mesh object/<name>/<*.obj>     = meters (single .obj per folder; 'spray' folder's
+#                                    obj is 'lotion_spray.obj' -> glob, don't assume).
+# Multi-view set = the 4 static synced cams at one frame f (anchor-sweep); each cam
+# supplies its OWN obj->cam pose and its world->cam extrinsic, so the base's
+# relative_p3d_from_ext gives the exact cross-camera transform (static world at
+# instant f). Masks are NOT shipped -> rendered from mesh+pose (CPU vertex splat).
+# canonical_key = h2o/<name>. Mark TRAIN (not held-out).
+# --------------------------------------------------------------------------- #
+_H2O_CLASSES = {1: "book", 2: "espresso", 3: "lotion", 4: "spray",
+                5: "milk", 6: "cocoa", 7: "chips", 8: "cappuccino"}
+
+
+class H2OAdapter(PoseDatasetAdapter):
+    name = "h2o"
+
+    def __init__(self, root, canonical_cache, *, frame_stride=15, num_static_cams=4, subjects=None):
+        self.root = root                                       # dataset/h2o
+        self.obj_root = os.path.join(root, "object_extracted", "object")
+        self.canon = _CanonCache(canonical_cache)
+        self.frame_stride = frame_stride
+        self.ncam = int(num_static_cams)                       # cam0..cam3 (skip cam4 ego)
+        self.subjects = subjects
+        self._intr = {}
+        self._meshc = {}
+
+    # --- layout helpers ------------------------------------------------------ #
+    @staticmethod
+    def _pose_dir(camdir):
+        """H2O ships 'obj_pose_RT' (README); some mirrors use lowercase. Detect."""
+        for nm in ("obj_pose_RT", "obj_pose_rt"):
+            p = os.path.join(camdir, nm)
+            if os.path.isdir(p):
+                return p
+        return os.path.join(camdir, "obj_pose_RT")
+
+    def _seq_dirs(self):
+        """A 'sequence' dir contains cam0..cam{ncam-1}. Native layout is
+        <root>/subject{1..4}/{h1,h2,k1,k2,o1,o2}/<idx>/camN. Discover robustly by
+        finding cam0 dirs that carry an obj_pose_RT folder."""
+        out = []
+        for cam0 in sorted(glob.glob(os.path.join(self.root, "subject*", "*", "*", "cam0"))):
+            seq = os.path.dirname(cam0)
+            if not os.path.isdir(os.path.join(cam0, "rgb")):
+                continue
+            if self.subjects is not None:
+                if not any(f"subject{s}" == seq.split(os.sep)[-3] or str(s) in seq.split(os.sep)[-3]
+                           for s in self.subjects):
+                    continue
+            out.append(seq)
+        return out
+
+    def _frames(self, seq):
+        rgbdir = os.path.join(seq, "cam0", "rgb")
+        return sorted(int(os.path.splitext(f)[0]) for f in os.listdir(rgbdir) if f.endswith(".png"))
+
+    def _common_frames(self, seq):
+        """Integer frames present (rgb + depth + obj_pose_RT) in EVERY of the
+        ncam static cams, so anchor and all aux views load. Robust to in-flight
+        extraction where some cams lag."""
+        common = None
+        for c in range(self.ncam):
+            camdir = os.path.join(seq, f"cam{c}")
+            triples = (("rgb", ".png"), ("depth", ".png"), (self._pose_dir(camdir), ".txt"))
+            perc = None
+            for sub, suf in triples:
+                d = sub if os.path.isabs(sub) else os.path.join(camdir, sub)
+                s = set()
+                try:
+                    for f in os.listdir(d):
+                        if f.endswith(suf):
+                            try:
+                                s.add(int(os.path.splitext(f)[0]))
+                            except ValueError:
+                                pass
+                except FileNotFoundError:
+                    s = set()
+                perc = s if perc is None else (perc & s)
+            common = (perc or set()) if common is None else (common & (perc or set()))
+        return sorted(common or set())
+
+    def _load_RT(self, seq, cam, frame):
+        """camN/obj_pose_RT/{f:06d}.txt -> (class_id, 4x4 obj->cam)."""
+        camdir = os.path.join(seq, f"cam{cam}")
+        arr = np.loadtxt(os.path.join(self._pose_dir(camdir), f"{frame:06d}.txt"))
+        cls = int(round(float(arr[0])))
+        M = np.asarray(arr[1:], np.float64).reshape(4, 4)
+        return cls, M
+
+    def _intrinsics(self, seq, cam):
+        key = (seq, cam)
+        if key not in self._intr:
+            p = os.path.join(seq, f"cam{cam}", "cam_intrinsics.txt")
+            fx, fy, cx, cy, w, h = np.loadtxt(p).reshape(6)
+            self._intr[key] = {"fx": float(fx), "fy": float(fy), "ppx": float(cx), "ppy": float(cy),
+                               "w": int(w), "h": int(h)}
+        return self._intr[key]
+
+    def _mesh(self, name):
+        if name not in self._meshc:
+            cand = sorted(glob.glob(os.path.join(self.obj_root, name, "*.obj")))
+            if not cand:
+                raise FileNotFoundError(f"h2o: no .obj in {os.path.join(self.obj_root, name)}")
+            self._meshc[name] = _load_mesh(cand[0])            # meters, no scaling
+        return self._meshc[name]
+
+    @staticmethod
+    def _zbuffer_cpu(V, faces, R, t, K, H, W):
+        """Exact CPU z-buffer of the mesh at obj->cam pose (R,t). Returns a HxW
+        float depth map (0 where empty) = the FRONT-surface depth. Per-face
+        barycentric fill over each face's 2D bbox, keeping the nearest z. Used to
+        (i) get the true object silhouette and (ii) derive an occlusion-aware
+        VISIBLE mask by comparing to the sensor depth."""
+        Xc = (R @ V.T + t.reshape(3, 1)).T
+        z = Xc[:, 2]
+        uv = (K @ Xc.T).T
+        u = uv[:, 0] / np.maximum(uv[:, 2], 1e-9)
+        v = uv[:, 1] / np.maximum(uv[:, 2], 1e-9)
+        P = np.stack([u, v], 1)
+        zbuf = np.full((H, W), np.inf, np.float64)
+        tri = P[faces]                                          # (m,3,2)
+        triz = z[faces]                                         # (m,3)
+        keep = (triz > 1e-3).all(1)
+        tri = tri[keep]; triz = triz[keep]
+        for (p0, p1, p2), (z0, z1, z2) in zip(tri, triz):
+            x0, y0 = p0; x1, y1 = p1; x2, y2 = p2
+            minx = int(max(np.floor(min(x0, x1, x2)), 0)); maxx = int(min(np.ceil(max(x0, x1, x2)), W - 1))
+            miny = int(max(np.floor(min(y0, y1, y2)), 0)); maxy = int(min(np.ceil(max(y0, y1, y2)), H - 1))
+            if maxx < minx or maxy < miny:
+                continue
+            xs, ys = np.meshgrid(np.arange(minx, maxx + 1), np.arange(miny, maxy + 1))
+            det = (y1 - y2) * (x0 - x2) + (x2 - x1) * (y0 - y2)
+            if abs(det) < 1e-9:
+                continue
+            a = ((y1 - y2) * (xs - x2) + (x2 - x1) * (ys - y2)) / det
+            b = ((y2 - y0) * (xs - x2) + (x0 - x2) * (ys - y2)) / det
+            c = 1.0 - a - b
+            ins = (a >= 0) & (b >= 0) & (c >= 0)
+            if not ins.any():
+                continue
+            zpix = a * z0 + b * z1 + c * z2
+            yy = ys[ins]; xx = xs[ins]; zz = zpix[ins]
+            cur = zbuf[yy, xx]
+            upd = zz < cur
+            zbuf[yy[upd], xx[upd]] = zz[upd]
+        zbuf[~np.isfinite(zbuf)] = 0.0
+        return zbuf
+
+    # --- interface ----------------------------------------------------------- #
+    def list_samples(self):
+        out = []
+        for seq in self._seq_dirs():
+            frames = self._common_frames(seq)      # present in ALL ncam cams
+            if not frames:
+                continue
+            for f in frames[:: self.frame_stride] or frames[:1]:
+                try:
+                    cls, _ = self._load_RT(seq, 0, f)          # scene-global object at frame f
+                except Exception:
+                    continue
+                if cls <= 0 or cls not in _H2O_CLASSES:
+                    continue                                    # background / no object
+                obj = _H2O_CLASSES[cls]
+                cams = list(range(self.ncam))
+                for anchor_ci in cams:
+                    aux = [c for c in cams if c != anchor_ci]
+                    aref = {"seq": seq, "cam": anchor_ci, "frame": f}
+                    pool = [{"seq": seq, "cam": c, "frame": f} for c in aux]
+                    out.append(ViewSample(anchor=aref, aux_pool=pool,
+                                          obj_key=f"h2o/{obj}", meta={"cls": cls, "obj": obj}))
+        return out
+
+    def load_view(self, ref):
+        seq, cam, frame = ref["seq"], ref["cam"], ref["frame"]
+        camdir = os.path.join(seq, f"cam{cam}")
+        rgb = np.asarray(Image.open(os.path.join(camdir, "rgb", f"{frame:06d}.png")).convert("RGB"),
+                         np.float32) / 255.0
+        depth = np.asarray(Image.open(os.path.join(camdir, "depth", f"{frame:06d}.png")),
+                           np.float32) / 1000.0
+        intr = self._intrinsics(seq, cam)
+        K = np.array([[intr["fx"], 0, intr["ppx"]], [0, intr["fy"], intr["ppy"]], [0, 0, 1]], np.float64)
+        H, W = rgb.shape[:2]
+        # masks not shipped -> render the object silhouette (exact CPU z-buffer)
+        # from THIS cam's mesh + obj->cam pose, then derive an OCCLUSION-AWARE
+        # VISIBLE mask (consistent with the other adapters, which all supply
+        # occlusion-aware visible masks): drop pixels where the sensor sees a
+        # surface clearly IN FRONT of the object (a hand occluder). This is a
+        # physical visibility test (nearer surface only); pixels where the sensor
+        # is behind/absent are KEPT so a wrong pose cannot hide there.
+        cls, M = self._load_RT(seq, cam, frame)
+        mask = np.zeros((H, W), np.float32)
+        if cls in _H2O_CLASSES:
+            mesh = self._mesh(_H2O_CLASSES[cls])
+            V = np.asarray(mesh.vertices, np.float64)
+            F = np.asarray(mesh.faces, np.int64)
+            zbuf = self._zbuffer_cpu(V, F, M[:3, :3], M[:3, 3], K, H, W)
+            sil = zbuf > 1e-6
+            occ = sil & (depth > 1e-3) & (depth < zbuf - 0.02)   # nearer surface => occluder
+            mask = (sil & ~occ).astype(np.float32)
+        return {"rgb": rgb, "mask": mask, "depth_m": depth, "intr": _K_to_intr(K)}
+
+    def extrinsic_cv(self, ref):                                # world->cam (invert cam->world)
+        seq, cam, frame = ref["seq"], ref["cam"], ref["frame"]
+        c2w = np.loadtxt(os.path.join(seq, f"cam{cam}", "cam_pose", f"{frame:06d}.txt")).reshape(4, 4)
+        w2c = np.linalg.inv(c2w)
+        return {"R": w2c[:3, :3].astype(np.float64), "t": w2c[:3, 3].astype(np.float64)}
+
+    def object_pose_cv(self, sample):                           # anchor cam's obj->cam, AS-IS
+        a = sample.anchor
+        _, M = self._load_RT(a["seq"], a["cam"], a["frame"])
+        return M[:3, :3].astype(np.float64), M[:3, 3].astype(np.float64)
+
+    def mesh_for(self, sample):
+        return self._mesh(sample.meta["obj"])
+
+    def canonical(self, obj_key):
+        return self.canon.get(obj_key)
+
+
+# --------------------------------------------------------------------------- #
+# HOGraspNet  (type a, 4 synced Kinect cams; TRAIN; mocap GT; grasp = hand
+# occlusion). world := the master ('mas') camera frame -> object_mat is
+# obj->WORLD and each cam's calibration.extrinsic is WORLD->cam (OpenCV), so
+# T_oc = M4(extrinsic) @ object_mat is obj->cam directly and
+# relative_p3d_from_ext(anchor,aux) yields the exact cross-camera transform
+# (same synced-rig pattern as DexYCB/HOCap). Translations are stored in cm
+# (-> /100 for meters). The 4 cams at one frame-instant are the views
+# (anchor-sweep). We use the SCANNED meshes (NOT dexycb CAD; different frame)
+# with the MANDATORY per-object scale cfg._OBJECT_SCALE_FIXED[obj_idx-1] and
+# obj_idx = HOGraspNet's own 1..30 ordering (config.OBJType), read from the
+# sequence name (231023_S01_obj_<idx>_grasp_<g>). We consume the cropped
+# 640x480 source_augmented RGB/depth, shifting cx,cy by the (hand-centred)
+# crop bbox offset. obj_mask (pseudo GT) comes from extra_data.
+# --------------------------------------------------------------------------- #
+# HOGraspNet's own object ordering (config.OBJType, 1..30) -> mandatory per-object
+# scale (config.cfg._OBJECT_SCALE_FIXED); index 0 == obj_idx 1.
+_HOG_OBJECT_SCALE_FIXED = [
+    1.0, 1.0, 1.0, 1.0, 0.8296698468, 1.0, 1.0, 1.0, 1.0, 1.0,
+    0.1035083229, 1.0, 0.6706711338, 1.0, 1.0, 0.43, 1.0, 1.0, 1.0, 1.0,
+    1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
+]
+
+
+class HOGraspNetAdapter(PoseDatasetAdapter):
+    name = "hograspnet"
+    CAMS = ("mas", "sub1", "sub2", "sub3")   # mas == world origin
+    _CROP_W, _CROP_H = 640, 480              # source_augmented crop size
+    _FULL_W, _FULL_H = 1920, 1080            # full-res intrinsics frame
+
+    def __init__(self, root, canonical_cache, *, frame_stride=1, subjects=None):
+        self.root = root                                   # dataset/hograspnet
+        self.anno_root = os.path.join(root, "labeling_data")
+        self.src_root = os.path.join(root, "source_augmented")
+        self.extra_root = os.path.join(root, "extra_data")
+        self.models_root = os.path.join(root, "obj_scanned_models")
+        self.canon = _CanonCache(canonical_cache)
+        self.frame_stride = frame_stride
+        self.subjects = set(subjects) if subjects is not None else None  # e.g. {"S01",...}
+        self._anno = {}
+        self._objdir_map = None
+
+    # -- object index (1..30) -> scanned-model dir name ("01_cracker_box") -- #
+    def _objdir(self):
+        if self._objdir_map is None:
+            m = {}
+            for d in os.listdir(self.models_root):
+                if os.path.isdir(os.path.join(self.models_root, d)):
+                    try:
+                        m[int(d.split("_")[0])] = d
+                    except ValueError:
+                        continue
+            self._objdir_map = m
+        return self._objdir_map
+
+    def _anno_path(self, ref):
+        return os.path.join(self.anno_root, ref["seq"], ref["trial"], "annotation",
+                            ref["cam"], f'{ref["cam"]}_{ref["frame"]}.json')
+
+    def _read_anno(self, ref):
+        p = self._anno_path(ref)
+        if p not in self._anno:
+            with open(p, "r", encoding="UTF-8-SIG") as f:
+                self._anno[p] = _json.load(f)
+        return self._anno[p]
+
+    def _crop_offset(self, hand_2d):
+        """Reproduce HOGraspNet util.utils.extractBbox: a fixed 640x480 window
+        centred on the hand 2D keypoints (full-res coords), clamped to image
+        bounds. Returns the integer (x0, y0) top-left offset used to crop the
+        source_augmented RGB/depth (so cx,cy shift by -x0,-y0)."""
+        x_min, x_max = float(hand_2d[:, 0].min()), float(hand_2d[:, 0].max())
+        y_min, y_max = float(hand_2d[:, 1].min()), float(hand_2d[:, 1].max())
+        x0 = max(0.0, (x_min + x_max) / 2 - self._CROP_W / 2)
+        y0 = max(0.0, (y_min + y_max) / 2 - self._CROP_H / 2)
+        if x0 + self._CROP_W > self._FULL_W:
+            x0 = self._FULL_W - self._CROP_W
+        if y0 + self._CROP_H > self._FULL_H:
+            y0 = self._FULL_H - self._CROP_H
+        return int(x0), int(y0)
+
+    def list_samples(self):
+        out = []
+        if not os.path.isdir(self.anno_root):
+            return out
+        objmap = self._objdir()
+        for seq in sorted(os.listdir(self.anno_root)):
+            seqdir = os.path.join(self.anno_root, seq)
+            if not os.path.isdir(seqdir):
+                continue
+            parts = seq.split("_")
+            if len(parts) < 6:
+                continue
+            subject = parts[1]                       # 'S01'
+            if self.subjects is not None and subject not in self.subjects:
+                continue
+            try:
+                obj_idx = int(parts[3])
+            except ValueError:
+                continue
+            objdir = objmap.get(obj_idx)
+            if objdir is None:
+                continue
+            obj_key = f"hograspnet/{objdir}"
+            for trial in sorted(os.listdir(seqdir)):
+                annob = os.path.join(seqdir, trial, "annotation")
+                srcb = os.path.join(self.src_root, seq, trial, "rgb_crop")
+                if not os.path.isdir(annob) or not os.path.isdir(srcb):
+                    continue                          # need cropped frames to load
+                cams = [c for c in self.CAMS if os.path.isdir(os.path.join(annob, c))
+                        and os.path.isdir(os.path.join(srcb, c))]
+                if not cams:
+                    continue
+                framesets = {}
+                for c in cams:
+                    anno_fs = set()
+                    for fn in os.listdir(os.path.join(annob, c)):
+                        if fn.endswith(".json"):
+                            anno_fs.add(fn[:-5].split("_")[-1])
+
+                    def _crop_frames(cropdir, suf, _c=c):
+                        s = set()
+                        cd = os.path.join(self.src_root, seq, trial, cropdir, _c)
+                        try:
+                            for fn in os.listdir(cd):
+                                if fn.endswith(suf):
+                                    s.add(fn[:-len(suf)].split("_")[-1])
+                        except FileNotFoundError:
+                            pass
+                        return s
+                    # frame usable only if anno json AND rgb_crop AND depth_crop exist
+                    framesets[c] = anno_fs & _crop_frames("rgb_crop", ".jpg") & _crop_frames("depth_crop", ".png")
+                allframes = sorted(set().union(*framesets.values()), key=lambda x: int(x))
+                for fi, frame in enumerate(allframes):
+                    if self.frame_stride > 1 and (fi % self.frame_stride):
+                        continue
+                    present = [c for c in cams if frame in framesets[c]]
+                    if not present:
+                        continue
+                    for anchor in present:
+                        aux = [c for c in present if c != anchor]
+                        aref = {"seq": seq, "trial": trial, "cam": anchor, "frame": frame}
+                        pool = [{"seq": seq, "trial": trial, "cam": c, "frame": frame} for c in aux]
+                        out.append(ViewSample(anchor=aref, aux_pool=pool, obj_key=obj_key,
+                                              meta={"obj_idx": obj_idx, "objdir": objdir}))
+        return out
+
+    def load_view(self, ref):
+        anno = self._read_anno(ref)
+        K = np.squeeze(np.asarray(anno["calibration"]["intrinsic"], np.float64)).reshape(3, 3)
+        hand_2d = np.squeeze(np.asarray(anno["hand"]["projected_2D_pose_per_cam"], np.float64))
+        x0, y0 = self._crop_offset(hand_2d)
+        Kc = K.copy()
+        Kc[0, 2] -= x0                                # cropped-frame principal point
+        Kc[1, 2] -= y0
+        base = f'{ref["cam"]}_{ref["frame"]}'
+        rgb_p = os.path.join(self.src_root, ref["seq"], ref["trial"], "rgb_crop", ref["cam"], base + ".jpg")
+        dep_p = os.path.join(self.src_root, ref["seq"], ref["trial"], "depth_crop", ref["cam"], base + ".png")
+        msk_p = os.path.join(self.extra_root, ref["seq"], ref["trial"], "obj_mask", ref["cam"], base + ".png")
+        rgb = np.asarray(Image.open(rgb_p).convert("RGB"), np.float32) / 255.0
+        depth = np.asarray(Image.open(dep_p), np.float32) / 1000.0     # mm -> m
+        if os.path.exists(msk_p):
+            ma = np.asarray(Image.open(msk_p))
+            if ma.ndim == 3:
+                ma = ma[..., 0]
+            mask = (ma >= 128).astype(np.float32)
+        else:
+            mask = np.zeros(depth.shape, np.float32)
+        return {"rgb": rgb, "mask": mask, "depth_m": depth, "intr": _K_to_intr(Kc)}
+
+    def _extrinsic4(self, ref):
+        M = np.squeeze(np.asarray(self._read_anno(ref)["calibration"]["extrinsic"], np.float64)).reshape(3, 4)
+        M4 = np.eye(4, dtype=np.float64)
+        M4[:3, :] = M
+        return M4
+
+    def extrinsic_cv(self, ref):                        # world(mas)->cam, meters
+        M4 = self._extrinsic4(ref)
+        return {"R": M4[:3, :3], "t": M4[:3, 3] / 100.0}
+
+    def object_pose_cv(self, sample):                   # obj->cam, anchor view, meters
+        a = sample.anchor
+        M4 = self._extrinsic4(a)
+        om = np.squeeze(np.asarray(self._read_anno(a)["Mesh"][0]["object_mat"], np.float64)).reshape(4, 4)
+        T = M4 @ om                                     # obj->cam (translation in cm)
+        return T[:3, :3], T[:3, 3] / 100.0
+
+    def mesh_for(self, sample):
+        objdir = sample.meta["objdir"]
+        S = _HOG_OBJECT_SCALE_FIXED[sample.meta["obj_idx"] - 1]   # mandatory per-object scale
+        m = _load_mesh(os.path.join(self.models_root, objdir, f"{objdir}.obj"))
+        m.apply_scale(S * 1e-2)                         # (S * V_raw) / 100 -> meters
+        return m
+
+    def canonical(self, obj_key):
+        return self.canon.get(obj_key)
+
+
+# --------------------------------------------------------------------------- #
+# DexH2R  (type a, 4 static synced Kinect RGB-D cams; TRAIN-ONLY pseudo-GT;
+# 56 non-YCB objects; human->robot handover => heavy hand/robot occlusion).
+# Per docs/dataset_convention_spec.md (DexH2R) + camera_calibration:
+#   obj_pose.pt = [T,4,4] = T_world<-obj  (world == hand_arm base frame).
+#   Per Kinect i in {0,1,2,3} (STATIC extrinsics):
+#     world->cam_i  W2C_i = inv(cali0i) @ hand_arm_mesh_to_kinect_pcd_0
+#     obj->cam_i    T_oc  = W2C_i @ obj_pose[t]        (OpenCV, meters)
+#   (equivalently T_oc = inv(cali0i) @ hand_arm @ obj_pose[t]; round-trips the
+#    repo's cam->world = inv(hand_arm) @ cali0i, verified in verify_conventions.py.)
+#   Extrinsics come from the PER-TRIAL calibration/ folder (kinect/cali0i.json +
+#   hand_arm_mesh_to_kinect_pcd_0.json). Intrinsics are NOT shipped per-trial ->
+#   read from the reference camera_calibration/ (Kinect 8-param distortion; the
+#   stored RGB/depth are already undistorted so K is used as a plain pinhole).
+#   depth = uint16 PNG in mm (/1000 -> meters), same res (1080x1920) & K as RGB
+#   (registered). Meshes object_model/<name>.obj are meters in the pose frame.
+#   The 4 Kinect cams at one frame t are the multi-view set (anchor-sweep); each
+#   supplies its own W2C_i so the base's relative_p3d_from_ext gives the exact
+#   cross-camera transform (shared static world = hand_arm base at instant t).
+#   No 2D masks are shipped -> render an OCCLUSION-AWARE visible silhouette from
+#   mesh+pose (same CPU z-buffer + sensor-occluder test as the H2O adapter).
+#   RealSense 0/1 also carry depth but their world pose is time-varying via
+#   ShadowHand FK on qpos.pt (needs the shadow_asserts URDF) -> DEFERRED; the 4
+#   Kinect views are sufficient for multi-view. ZCAM (12 RGB-only) dropped.
+#   canonical_key = dexh2r/<name>. TRAIN, pseudo-GT.
+# Layout (after extract, prefix stripped): <root>/extracted/<subj>/<obj>/<seq>/
+#   {obj_pose.pt, calibration/, kinect/<i>/{rgb,depth}/<frame>.{jpg,png}, ...}
+# --------------------------------------------------------------------------- #
+class DexH2RAdapter(PoseDatasetAdapter):
+    name = "dexh2r"
+    KINECTS = (0, 1, 2, 3)                    # the 4 static Kinect RGB-D cams
+
+    def __init__(self, root, canonical_cache, *, frame_stride=15, subjects=None,
+                 split=None, extracted_subdir="extracted"):
+        self.root = root                                          # dataset/dexh2r
+        self.ext_root = os.path.join(root, extracted_subdir)      # extracted subjects live here
+        self.mesh_dir = os.path.join(root, "object_model", "object_model")
+        # reference intrinsics (per-trial calibration ships extrinsics only)
+        self.ref_calib = os.path.join(root, "camera_calibration", "camera_calibration")
+        self.canon = _CanonCache(canonical_cache)
+        self.frame_stride = int(frame_stride)
+        # `subjects`/`split`: restrict to a subset of subject-id strings; split reads
+        # dataset_split.json (<split>_sub_names) when subjects is None.
+        if subjects is None and split is not None:
+            sp = _json.load(open(os.path.join(root, "dataset_split.json")))
+            subjects = [str(s) for s in sp[f"{split}_sub_names"]]
+        self.subjects = set(str(s) for s in subjects) if subjects is not None else None
+        self._intr = {}
+        self._ext = {}
+        self._meshc = {}
+        self._nframe = {}
+
+    # -- calibration ---------------------------------------------------------- #
+    @staticmethod
+    def _seven2mat(js):
+        from scipy.spatial.transform import Rotation
+        v = js["value0"]; r = v["rotation"]; t = v["translation"]
+        M = np.eye(4, dtype=np.float64)
+        M[:3, :3] = Rotation.from_quat([r["x"], r["y"], r["z"], r["w"]]).as_matrix()
+        M[:3, 3] = [t["x"], t["y"], t["z"]]
+        return M
+
+    def _w2c(self, trial_dir, i):
+        """world(hand_arm)->cam_i (OpenCV, meters), from the PER-TRIAL calibration."""
+        key = (trial_dir, i)
+        if key not in self._ext:
+            cdir = os.path.join(trial_dir, "calibration")
+            hand_arm = self._seven2mat(_json.load(open(
+                os.path.join(cdir, "hand_arm_mesh_to_kinect_pcd_0.json"))))
+            cali0i = self._seven2mat(_json.load(open(
+                os.path.join(cdir, "kinect", f"cali0{i}.json"))))
+            self._ext[key] = np.linalg.inv(cali0i) @ hand_arm     # world->cam_i
+        return self._ext[key]
+
+    def _K(self, i):
+        if i not in self._intr:
+            d = _json.load(open(os.path.join(
+                self.ref_calib, "camera_intrinsics", "kinect", f"camera_intrinsics_{i}.json")))
+            self._intr[i] = np.asarray(d["camera_intrinsics"], np.float64).reshape(3, 3)
+        return self._intr[i]
+
+    def _mesh(self, name):
+        if name not in self._meshc:
+            self._meshc[name] = _load_mesh(os.path.join(self.mesh_dir, f"{name}.obj"))
+        return self._meshc[name]
+
+    def _obj_pose(self, trial_dir, frame):
+        """T_world<-obj at frame (4x4)."""
+        op = torch.load(os.path.join(trial_dir, "obj_pose.pt"), map_location="cpu")
+        return np.asarray(op[frame], np.float64)
+
+    def _T_oc(self, trial_dir, i, frame):
+        return self._w2c(trial_dir, i) @ self._obj_pose(trial_dir, frame)   # obj->cam_i
+
+    def _oplen(self, trial_dir):
+        """Number of obj_pose timesteps (upper bound on valid frame index)."""
+        if trial_dir not in self._nframe:
+            try:
+                op = torch.load(os.path.join(trial_dir, "obj_pose.pt"), map_location="cpu")
+                self._nframe[trial_dir] = int(len(op))
+            except Exception:
+                self._nframe[trial_dir] = 0
+        return self._nframe[trial_dir]
+
+    @staticmethod
+    def _cam_frames(trial_dir, i):
+        """Integer frames where kinect cam i has BOTH rgb .jpg and depth .png
+        (files are named '<frame>.jpg'/'<frame>.png'; robust to sparse ids)."""
+        cd = os.path.join(trial_dir, "kinect", str(i))
+        out = None
+        for sub, suf in (("rgb", ".jpg"), ("depth", ".png")):
+            s = set()
+            try:
+                for f in os.listdir(os.path.join(cd, sub)):
+                    if f.endswith(suf):
+                        try:
+                            s.add(int(f[:-len(suf)]))
+                        except ValueError:
+                            pass
+            except FileNotFoundError:
+                return set()
+            out = s if out is None else (out & s)
+        return out or set()
+
+    # -- interface ------------------------------------------------------------ #
+    def list_samples(self):
+        out = []
+        if not os.path.isdir(self.ext_root):
+            return out
+        for subj in sorted(os.listdir(self.ext_root)):
+            subj_dir = os.path.join(self.ext_root, subj)
+            if not os.path.isdir(subj_dir):
+                continue
+            if self.subjects is not None and subj not in self.subjects:
+                continue
+            for name in sorted(os.listdir(subj_dir)):
+                obj_dir = os.path.join(subj_dir, name)
+                if not os.path.isdir(obj_dir) or not os.path.exists(
+                        os.path.join(self.mesh_dir, f"{name}.obj")):
+                    continue
+                for seq in sorted(os.listdir(obj_dir)):
+                    trial_dir = os.path.join(obj_dir, seq)
+                    if not os.path.exists(os.path.join(trial_dir, "obj_pose.pt")):
+                        continue
+                    if not os.path.exists(os.path.join(trial_dir, "calibration",
+                                                        "hand_arm_mesh_to_kinect_pcd_0.json")):
+                        continue
+                    cams = [i for i in self.KINECTS
+                            if os.path.isdir(os.path.join(trial_dir, "kinect", str(i), "depth"))]
+                    if len(cams) < 1:
+                        continue
+                    op_n = self._oplen(trial_dir)
+                    if op_n < 1:
+                        continue
+                    # frames actually present (rgb+depth) per cam, bounded by obj_pose
+                    per = {i: {f for f in self._cam_frames(trial_dir, i) if 0 <= f < op_n}
+                           for i in cams}
+                    allframes = sorted(set().union(*per.values()) if per else set())
+                    for fi, frame in enumerate(allframes):
+                        if self.frame_stride > 1 and (fi % self.frame_stride):
+                            continue
+                        present = [i for i in cams if frame in per[i]]   # cams with this frame
+                        if not present:
+                            continue
+                        for anchor in present:
+                            aux = [c for c in present if c != anchor]
+                            aref = {"trial": trial_dir, "cam": anchor, "frame": frame, "name": name}
+                            pool = [{"trial": trial_dir, "cam": c, "frame": frame, "name": name}
+                                    for c in aux]
+                            out.append(ViewSample(anchor=aref, aux_pool=pool,
+                                                  obj_key=f"dexh2r/{name}",
+                                                  meta={"obj": name, "subject": subj,
+                                                        "pseudo_gt": True, "train": True}))
+        return out
+
+    def load_view(self, ref):
+        trial, i, frame, name = ref["trial"], ref["cam"], ref["frame"], ref["name"]
+        cdir = os.path.join(trial, "kinect", str(i))
+        rgb = np.asarray(Image.open(os.path.join(cdir, "rgb", f"{frame}.jpg")).convert("RGB"),
+                         np.float32) / 255.0
+        depth = np.asarray(Image.open(os.path.join(cdir, "depth", f"{frame}.png")),
+                           np.float32) / 1000.0                    # uint16 mm -> m
+        K = self._K(i)
+        H, W = depth.shape[:2]
+        # No 2D masks shipped -> render the object silhouette (exact CPU z-buffer)
+        # at THIS cam's obj->cam pose, then derive an OCCLUSION-AWARE VISIBLE mask
+        # (identical policy to the H2O adapter): drop pixels where the sensor sees
+        # a surface clearly IN FRONT of the object (hand/robot occluder); KEEP
+        # pixels where the sensor is behind/absent so a wrong pose cannot hide.
+        T = self._T_oc(trial, i, frame)
+        mesh = self._mesh(name)
+        V = np.asarray(mesh.vertices, np.float64)
+        F = np.asarray(mesh.faces, np.int64)
+        zbuf = H2OAdapter._zbuffer_cpu(V, F, T[:3, :3], T[:3, 3], K, H, W)
+        sil = zbuf > 1e-6
+        occ = sil & (depth > 1e-3) & (depth < zbuf - 0.02)         # nearer surface => occluder
+        mask = (sil & ~occ).astype(np.float32)
+        return {"rgb": rgb, "mask": mask, "depth_m": depth, "intr": _K_to_intr(K)}
+
+    def extrinsic_cv(self, ref):                                   # world(hand_arm)->cam_i
+        W2C = self._w2c(ref["trial"], ref["cam"])
+        return {"R": W2C[:3, :3], "t": W2C[:3, 3]}
+
+    def object_pose_cv(self, sample):                              # obj->cam, anchor view
+        a = sample.anchor
+        T = self._T_oc(a["trial"], a["cam"], a["frame"])
+        return T[:3, :3], T[:3, 3]
+
+    def mesh_for(self, sample):
+        return self._mesh(sample.meta["obj"])
+
+    def canonical(self, obj_key):
+        return self.canon.get(obj_key)
+
+
+ADAPTERS = {
+    "dexycb": DexYCBAdapter, "housecat6d": HouseCat6DAdapter, "pace": PACEAdapter,
+    "graspnet": GraspNetAdapter, "ycbv": YCBVAdapter, "contactpose": ContactPoseAdapter,
+    "hocap": HOCapAdapter, "h2o": H2OAdapter, "hograspnet": HOGraspNetAdapter, "dexh2r": DexH2RAdapter,
+}

migrator/code/mv-sam3d-for-6d-v2/mvsam3d/data/base_dataset.py

@@ -0,0 +1,272 @@
+"""Dataset-agnostic multi-view 6D-pose dataset (base + adapter protocol).
+
+This is the generalization of ``MultiViewDexYCB``: ALL geometry / SSI / GT-layout
+/ view-construction math is reused VERBATIM from ``dexycb_dataset`` (the helper
+functions are imported, not re-implemented), and the ONLY dataset-specific logic
+lives behind a thin ``PoseDatasetAdapter``.
+
+One sample = one (object-state, anchor-view) with >=1 aux views.  The two
+multi-view regimes both funnel into the same anchor+aux construction:
+  (a) synced multi-cam rig  -> aux views are OTHER cameras at the same instant;
+      per-view extrinsic = the constant rig calibration (tag->cam, OpenCV).
+  (b) static scene + moving cam -> aux views are OTHER FRAMES of the same static
+      sequence; per-view extrinsic = that frame's world->cam pose (OpenCV).
+In both cases the adapter yields, per view ref, an OpenCV tag/world->cam
+extrinsic ``{R,t}``; the base computes the anchor-relative rigid transform with
+the SAME formula as ``dexycb_calib.relative_p3d`` and re-expresses every aux
+pointmap in the anchor PyTorch3D frame under one shared anchor SSI.
+
+The adapter delivers the object pose as OpenCV **obj->cam, meters** for the
+anchor view (== DexYCB pose_y semantics); the base applies D=diag(-1,-1,1) and
+the canonical M-mapping.  Adapters MUST NOT pre-apply D or M.
+
+Output dict is byte-identical in structure to ``MultiViewDexYCB.__getitem__`` and
+feeds ``collate_multiview`` unchanged.
+"""
+from __future__ import annotations
+
+import random
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Tuple
+
+import numpy as np
+import torch
+
+from sam3d_objects.geometry.geometry import matrix_to_rot6d, normalize_rot6d
+
+from . import canonical
+# Reuse the proven geometry helpers VERBATIM (imported, not re-implemented).
+from .dexycb_dataset import (
+    IMG_SIZE,
+    _apply_ssi,
+    _cam_feature,
+    _ssi_params,
+    _to_p3d,
+    _unproject,
+    _build_view_fn,
+)
+
+# --- Robust data loading: a large multi-dataset loader WILL hit corrupt /
+# truncated / missing files (e.g. a PNG half-written during a disk-full). One
+# bad file must NEVER kill a run. We (1) tolerate mildly-truncated images,
+# (2) force-decode on open so a hard-broken file raises WITH its path (logged
+# once to _bad_files.log), and (3) retry another random sample in __getitem__. ---
+import os as _os
+from PIL import Image as _PILImage, ImageFile as _PILImageFile
+
+_PILImageFile.LOAD_TRUNCATED_IMAGES = True
+from mvsam3d import paths as _mv_paths
+_BAD_FILES_LOG = _mv_paths.BAD_FILES_LOG
+_bad_seen: set = set()
+
+
+def _log_bad_file(path, err):
+    k = str(path)
+    if k in _bad_seen:
+        return
+    _bad_seen.add(k)
+    try:
+        with open(_BAD_FILES_LOG, "a") as _fh:
+            _fh.write(f"{k}\t{type(err).__name__}: {err}\n")
+    except Exception:
+        pass
+
+
+_orig_pil_open = _PILImage.open
+
+
+def _safe_pil_open(fp, *a, **k):
+    """Force-decode on open so a broken/truncated image raises here (with its
+    path) rather than deep inside np.asarray/convert with no context."""
+    try:
+        im = _orig_pil_open(fp, *a, **k)
+        im.load()
+        return im
+    except Exception as _e:
+        _log_bad_file(getattr(fp, "name", fp), _e)
+        raise
+
+
+_PILImage.open = _safe_pil_open  # every adapter uses the shared PIL.Image.open
+
+_D = np.diag([-1.0, -1.0, 1.0]).astype(np.float64)
+MIN_VALID_PM_SCALE = 1e-6
+
+
+@dataclass
+class ViewSample:
+    """One training unit. ``anchor`` and ``aux_pool`` are adapter-defined view
+    refs; ``obj_key`` is the canonical-latent cache key."""
+    anchor: Any
+    aux_pool: List[Any]
+    obj_key: str
+    meta: dict = field(default_factory=dict)
+
+
+class PoseDatasetAdapter(ABC):
+    """Minimal dataset-specific interface. Everything returned is in the UNIFIED
+    contract: OpenCV, meters, depth registered to RGB."""
+
+    name: str = "base"
+
+    @abstractmethod
+    def list_samples(self) -> List[ViewSample]:
+        ...
+
+    @abstractmethod
+    def load_view(self, ref: Any) -> Dict[str, Any]:
+        """Return {'rgb':[H,W,3] float[0,1], 'mask':[H,W] float{0,1},
+        'depth_m':[H,W] float meters (registered to rgb), 'intr':{fx,fy,ppx,ppy}}."""
+        ...
+
+    @abstractmethod
+    def extrinsic_cv(self, ref: Any) -> Dict[str, np.ndarray]:
+        """Return {'R':(3,3),'t':(3,)} tag/world->cam, OpenCV, meters."""
+        ...
+
+    @abstractmethod
+    def object_pose_cv(self, sample: ViewSample) -> Tuple[np.ndarray, np.ndarray]:
+        """Return (R_cv, t_cv): object(CAD)->camera in the ANCHOR view, OpenCV,
+        meters. x_cam = R_cv @ p_cad + t_cv."""
+        ...
+
+    @abstractmethod
+    def canonical(self, obj_key: str) -> dict:
+        """Return the cached canonical dict: {latent[8,16,16,16],
+        occupancy[64,64,64] or [1,...], max_extent float, center_cad (3,)}."""
+        ...
+
+
+def relative_p3d_from_ext(anchor_ext: Dict[str, np.ndarray],
+                          aux_ext: Dict[str, np.ndarray]):
+    """(Rrel, trel) mapping a PyTorch3D point in aux cam -> anchor cam.
+
+    IDENTICAL formula to dexycb_calib.relative_p3d, but taking the two OpenCV
+    tag/world->cam extrinsics directly (works for both synced-rig and
+    static-scene-moving-cam regimes)."""
+    Ra, ta = anchor_ext["R"], anchor_ext["t"]
+    Rb, tb = aux_ext["R"], aux_ext["t"]
+    RaRbT = Ra @ Rb.T
+    Rrel = _D @ RaRbT @ _D
+    trel = _D @ ta - _D @ (RaRbT @ tb)
+    return Rrel.astype(np.float64), trel.astype(np.float64)
+
+
+def _transform_pointmap(pm: np.ndarray, Rrel: np.ndarray, trel: np.ndarray) -> np.ndarray:
+    H, W, _ = pm.shape
+    flat = pm.reshape(-1, 3)
+    valid = np.isfinite(flat).all(axis=1)
+    out = flat.copy()
+    out[valid] = flat[valid] @ Rrel.T + trel
+    return out.reshape(H, W, 3)
+
+
+class MultiViewPoseDataset(torch.utils.data.Dataset):
+    """Adapter-driven multi-view 6D-pose dataset. Behaviourally identical to
+    MultiViewDexYCB when driven by DexYCBAdapter."""
+
+    def __init__(self, adapter: PoseDatasetAdapter, *,
+                 image_size: int = IMG_SIZE,
+                 random_views: bool = False,
+                 num_aux_views: int = 7,
+                 min_aux_views: int = 1,
+                 max_aux_views: int = 3,
+                 depth_near: float = 0.1,
+                 depth_far: float = 3.0,
+                 ssi_mode: str = "scene"):
+        self.adapter = adapter
+        self.image_size = image_size
+        self.random_views = random_views
+        self.num_aux = num_aux_views
+        self.min_aux = min_aux_views
+        self.max_aux = max_aux_views
+        self.depth_near = depth_near
+        self.depth_far = depth_far
+        if ssi_mode not in ("scene", "object_centric"):
+            raise ValueError(f"ssi_mode must be 'scene' or 'object_centric', got {ssi_mode!r}")
+        self.ssi_mode = ssi_mode
+        if random_views and not (1 <= min_aux_views <= max_aux_views):
+            raise ValueError(f"random_views requires 1<=min_aux<=max_aux, got "
+                             f"{min_aux_views}..{max_aux_views}")
+        self.samples: List[ViewSample] = adapter.list_samples()
+
+    def __len__(self):
+        return len(self.samples)
+
+    def _anchor_pm(self, ref):
+        v = self.adapter.load_view(ref)
+        pm = _to_p3d(_unproject(v["depth_m"], v["intr"], self.depth_near, self.depth_far))
+        return v, pm
+
+    def __getitem__(self, idx: int) -> dict:
+        # Robust wrapper: if a sample references a corrupt/truncated/missing file,
+        # log it once and fall back to another random sample — never crash the run.
+        n = len(self.samples)
+        for _try in range(24):
+            j = idx if _try == 0 else random.randrange(n)
+            try:
+                return self._getitem_impl(j)
+            except Exception as _e:  # noqa: BLE001 - any decode/IO failure -> skip sample
+                _log_bad_file(f"[sample idx={j} adapter={getattr(self.adapter, 'name', '?')}]", _e)
+        # Exhausted retries: surface the error (don't hide a genuine bug forever).
+        return self._getitem_impl(random.randrange(n))
+
+    def _getitem_impl(self, idx: int) -> dict:
+        s = self.samples[idx]
+
+        # ---- anchor view + shared SSI from anchor pointmap ----
+        a_view, a_pm = self._anchor_pm(s.anchor)
+        shift, scale = _ssi_params(a_pm, mask=a_view["mask"], mode=self.ssi_mode)
+        anchor_ext = self.adapter.extrinsic_cv(s.anchor)
+
+        views = [_build_view_fn(a_view["rgb"], a_view["mask"],
+                                _apply_ssi(a_pm, shift, scale), True, self.image_size)]
+        views[0]["cam"] = _cam_feature(np.eye(3), np.zeros(3), shift, scale)
+
+        # ---- choose aux views ----
+        pool = s.aux_pool
+        if not pool:
+            aux_refs = []                      # anchor-only sample (no other views available)
+        elif self.random_views:
+            k = random.randint(min(self.min_aux, len(pool)), min(self.max_aux, len(pool)))
+            aux_refs = random.sample(pool, k)
+        else:
+            aux_refs = pool[: self.num_aux]
+
+        for aux in aux_refs:
+            x = self.adapter.load_view(aux)
+            x_pm = _to_p3d(_unproject(x["depth_m"], x["intr"], self.depth_near, self.depth_far))
+            Rrel, trel = relative_p3d_from_ext(anchor_ext, self.adapter.extrinsic_cv(aux))
+            x_pm = _transform_pointmap(x_pm, Rrel, trel)
+            av = _build_view_fn(x["rgb"], x["mask"], _apply_ssi(x_pm, shift, scale),
+                                False, self.image_size)
+            av["cam"] = _cam_feature(Rrel, trel, shift, scale)
+            views.append(av)
+
+        # ---- GT layout (anchor cam, PyTorch3D) ----
+        R_cv, t_cv = self.adapter.object_pose_cv(s)
+        R_p3d, t_p3d = _D @ np.asarray(R_cv, np.float64), _D @ np.asarray(t_cv, np.float64)
+        canon = self.adapter.canonical(s.obj_key)
+        R, t, sc = canonical.gt_layout_anchor_frame(
+            R_p3d, t_p3d, np.asarray(canon["center_cad"]), float(canon["max_extent"]))
+        rot6d = normalize_rot6d(matrix_to_rot6d(torch.from_numpy(R).float())).float()
+
+        # ---- SSI-consistent pose targets (mirror dexycb_dataset exactly) ----
+        shift_t = torch.from_numpy(np.asarray(shift, np.float64)).float()
+        pm_scale_safe = torch.tensor(float(scale), dtype=torch.float32).clamp_min(MIN_VALID_PM_SCALE)
+        t_ssi = (torch.from_numpy(t).float() - shift_t) / pm_scale_safe
+        gt_scale_m = torch.tensor(float(sc), dtype=torch.float32)
+        gt_scale_ratio = (gt_scale_m / pm_scale_safe).clamp_min(MIN_VALID_PM_SCALE)
+        s_log3 = torch.log(gt_scale_ratio).expand(3).contiguous()
+
+        return {
+            "shape_latent": canon["latent"].float(),
+            "rot6d": rot6d,
+            "translation": t_ssi,
+            "scale": s_log3,
+            "pm_shift": shift_t,
+            "pm_scale": pm_scale_safe.reshape(1),
+            "views": views,
+            "object_name": s.obj_key,
+        }

migrator/code/mv-sam3d-for-6d-v2/mvsam3d/data/canonical.py

@@ -0,0 +1,76 @@
+"""Canonical-voxel construction and anchor-frame GT layout math.
+
+The occupancy is produced by the VENDORED ``voxelize_mesh`` (byte-identical to
+SAM-3D's ``inference_utils.voxelize_mesh``), so the SS encoder sees exactly the
+distribution it was trained on.  ``voxelize_mesh`` operates in the ROTATED
+(z-up→y-up) frame and returns that frame's ``scale = 1/max_extent`` and
+``center`` (the rotated-frame AABB center).  Per the locked design we recover
+the CAD-frame quantities the GT layout consumes:
+
+    max_extent = 1 / scale                 (rotation-invariant: M is axis-aligned)
+    center_cad = M^T @ center_rot          (rotated-frame center → CAD frame)
+
+GT layout (anchor camera, PyTorch3D).  A canonical voxel point ``v_canon`` maps
+to the camera as ``X_cam = R @ (s · v_canon) + t``.  Composing the voxelization
+with the DexYCB object pose ``X_cam = R_dex @ p_cad + t_dex`` gives:
+
+    s = max_extent                         (isotropic scalar)
+    R = R_dex @ M^T                        (absorbs the z-up→y-up grid rotation)
+    t = R_dex @ center_cad + t_dex
+
+with ``M = [[1,0,0],[0,0,-1],[0,1,0]]`` (the grid rotation baked into
+``voxelize_mesh``) and the OpenCV→PyTorch3D flip ``D = diag(-1,-1,1)`` applied to
+``(R_dex, t_dex)`` upstream in the dataset.
+"""
+from __future__ import annotations
+
+import numpy as np
+
+from sam3d_objects.pipeline.inference_utils import trimesh2o3d_mesh, voxelize_mesh
+
+# Fixed z-up → y-up rotation applied inside voxelize_mesh:
+#   verts are right-multiplied by M.T, i.e. v_rot = v_cad @ M.T  <=>  v_rot = M @ v_cad.
+M_ZUP_TO_YUP = np.array([[1, 0, 0], [0, 0, -1], [0, 1, 0]], dtype=np.float64)
+
+
+def voxelize_canonical(mesh, resolution: int = 64):
+    """CAD trimesh → (occupancy ``[1,R,R,R]`` long, max_extent float, center_cad (3,)).
+
+    ``occupancy`` is exactly the vendored ``voxelize_mesh`` output.  ``max_extent``
+    and ``center_cad`` are recovered in the ORIGINAL CAD frame from the rotated-
+    frame ``scale``/``center`` ``voxelize_mesh`` returns, and are what the GT
+    layout math consumes.
+    """
+    # voxelize_mesh consumes an open3d TriangleMesh (it assigns
+    # o3d.utility.Vector3dVector to mesh.vertices and calls the o3d voxelizer),
+    # so convert via the vendored trimesh2o3d_mesh first — exactly as the fork's
+    # pipeline does. trimesh2o3d_mesh builds a fresh o3d mesh, so the input
+    # trimesh is not mutated.
+    o3d_mesh = trimesh2o3d_mesh(mesh)
+    ss, scale, center_rot = voxelize_mesh(o3d_mesh, resolution=resolution)
+    if scale is None or center_rot is None:
+        # Mesh was already normalized to [-0.5, 0.5]; recover stats from raw verts.
+        verts = np.asarray(mesh.vertices, dtype=np.float64)
+        verts = verts @ M_ZUP_TO_YUP.T  # voxelize_mesh works in the rotated frame
+        vmin, vmax = verts.min(axis=0), verts.max(axis=0)
+        center_rot = (vmax + vmin) / 2.0
+        max_extent = float(np.max(vmax - vmin))
+    else:
+        max_extent = 1.0 / float(scale)
+        center_rot = np.asarray(center_rot, dtype=np.float64)
+    # center_rot is in the rotated (y-up) frame; bring it back to the CAD frame.
+    center_cad = M_ZUP_TO_YUP.T @ center_rot
+    return ss, max_extent, center_cad.astype(np.float64)
+
+
+def gt_layout_anchor_frame(R_dex: np.ndarray, t_dex: np.ndarray,
+                           center_cad: np.ndarray, max_extent: float):
+    """DexYCB object pose (anchor cam frame) → canonical (R, t, s).
+
+    R_dex (3,3), t_dex (3,): object→camera pose in the anchor camera frame
+    (already in the desired PyTorch3D convention).  Returns R (3,3), t (3,),
+    s (float).
+    """
+    R = R_dex @ M_ZUP_TO_YUP.T
+    t = R_dex @ np.asarray(center_cad, np.float64) + np.asarray(t_dex, np.float64)
+    return R.astype(np.float64), t.astype(np.float64), float(max_extent)

migrator/code/mv-sam3d-for-6d-v2/mvsam3d/data/dexycb_calib.py

@@ -0,0 +1,75 @@
+"""DexYCB camera calibration: per-camera extrinsics/intrinsics and the
+cross-camera transform that re-expresses an aux camera's pointmap and pose in
+the ANCHOR camera frame.
+
+DexYCB stores, per extrinsics group, a 3x4 ``[R_c | t_c]`` (row-major, 12
+floats) mapping the shared calibration/tag frame -> camera ``c`` in the
+**OpenCV** convention (x-right, y-down, z-fwd).  Exactly one camera is identity
+(the master/tag).  The whole training pipeline works in **PyTorch3D** frame
+(x-left, y-up, z-fwd), related by ``D = diag(-1,-1,1)`` (``D == D^{-1}``).
+
+For a 3D point in aux cam B (PyTorch3D) the anchor-frame point is:
+
+    X_tag      = R_B^T (D X_B^p3d - t_B)
+    X_A^p3d    = D (R_A X_tag + t_A)
+               = (D R_A R_B^T D) X_B^p3d + (D t_A - D R_A R_B^T t_B)
+
+i.e. a rigid transform ``X_A = Rrel @ X_B + trel`` with
+``Rrel = D R_A R_B^T D`` and ``trel = D t_A - D R_A R_B^T t_B``.
+"""
+from __future__ import annotations
+
+import os
+
+import numpy as np
+import yaml
+
+_D = np.diag([-1.0, -1.0, 1.0]).astype(np.float64)
+
+
+def _load_yaml(path):
+    with open(path) as f:
+        return yaml.load(f, Loader=yaml.UnsafeLoader)  # files contain python/tuple tags
+
+
+def load_extrinsics(calib_root: str, extrinsics_tag: str) -> dict[str, dict]:
+    """serial -> {"R": (3,3), "t": (3,)} (tag->cam, OpenCV)."""
+    path = os.path.join(calib_root, f"extrinsics_{extrinsics_tag}", "extrinsics.yml")
+    raw = _load_yaml(path)["extrinsics"]
+    out = {}
+    for serial, vals in raw.items():
+        m = np.asarray(vals, dtype=np.float64).reshape(3, 4)
+        out[str(serial)] = {"R": m[:, :3], "t": m[:, 3]}
+    return out
+
+
+def load_intrinsics(calib_root: str, serial: str) -> dict:
+    """color intrinsics fx,fy,ppx,ppy for a serial (640x480)."""
+    path = os.path.join(calib_root, "intrinsics", f"{serial}_640x480.yml")
+    return _load_yaml(path)["color"]
+
+
+def relative_p3d(ext: dict, anchor_serial: str, aux_serial: str):
+    """Return (Rrel, trel) mapping a PyTorch3D point in ``aux`` cam -> anchor cam.
+
+    X_anchor = Rrel @ X_aux + trel.
+    """
+    Ra, ta = ext[anchor_serial]["R"], ext[anchor_serial]["t"]
+    Rb, tb = ext[aux_serial]["R"], ext[aux_serial]["t"]
+    RaRbT = Ra @ Rb.T
+    Rrel = _D @ RaRbT @ _D
+    trel = _D @ ta - _D @ (RaRbT @ tb)
+    return Rrel.astype(np.float64), trel.astype(np.float64)
+
+
+def transform_pointmap_to_anchor(pm: np.ndarray, Rrel: np.ndarray, trel: np.ndarray) -> np.ndarray:
+    """Apply the rigid transform to an (H,W,3) pointmap, preserving NaN/invalid.
+
+    pm is in the aux camera's PyTorch3D frame; returns it in the anchor frame.
+    """
+    H, W, _ = pm.shape
+    flat = pm.reshape(-1, 3)
+    valid = np.isfinite(flat).all(axis=1)
+    out = flat.copy()
+    out[valid] = flat[valid] @ Rrel.T + trel
+    return out.reshape(H, W, 3)

migrator/code/mv-sam3d-for-6d-v2/mvsam3d/data/dexycb_dataset.py

@@ -0,0 +1,501 @@
+"""Multi-view (multi-camera) DexYCB dataset for the 6D-pose flow model.
+
+One sample = one (sequence, timestamp, object, anchor-camera).  The anchor
+camera defines the reference frame; the other synchronized cameras are aux
+views.  Everything is expressed in the **anchor camera's PyTorch3D frame**:
+
+  * Pointmaps are unprojected from each camera's aligned depth (metric, OpenCV),
+    converted to PyTorch3D (X' = diag(-1,-1,1) X), and aux cameras are rigidly
+    transformed into the anchor frame via DexYCB calibration extrinsics BEFORE a
+    single shared SSI (scale-shift invariant) normalization computed on the
+    anchor view.  Pointmaps deliberately KEEP NaN at invalid/background pixels.
+  * GT (R, t, s) is the anchor camera's object pose, OpenCV→PyTorch3D, mapped to
+    the canonical voxel frame (R_dex·Mᵀ, R_dex·c_cad + t_dex, max_extent) — see
+    canonical.gt_layout_anchor_frame.
+  * The 6D rotation is ``normalize_rot6d(matrix_to_rot6d(R))`` using the
+    VENDORED column-convention rotation utils (consistent with the model's
+    decode ``rot6d_to_matrix(denormalize_rot6d(·))``).
+  * The shape target is the precomputed canonical SS latent for the object
+    (cache/canonical/<object>.pt).
+
+There is NO shape-K/V; this dataset only emits plain conditioning tokens + flow
+targets.  Output dict feeds MVSam3DFlow.flow_step directly.
+"""
+from __future__ import annotations
+
+import glob
+import os
+import random
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from PIL import Image
+from torch.utils.data import Dataset
+
+from sam3d_objects.geometry.geometry import (
+    matrix_to_rot6d,
+    normalize_rot6d,
+)
+
+from . import canonical, dexycb_calib
+
+_D = np.diag([-1.0, -1.0, 1.0]).astype(np.float64)
+DEPTH_SCALE = 1000.0  # DexYCB depth PNG is uint16 millimetres
+IMG_SIZE = 518
+DEPTH_NEAR, DEPTH_FAR = 0.1, 3.0  # metres; clip implausible depth
+
+# DexYCB ycb_ids / seg use 1-based class indices (NOT the YCB numeric prefix).
+# This is the canonical DexYCB _YCB_CLASSES ordering; seg == ycb_id for the object.
+YCB_ID_TO_NAME = {
+    1: "002_master_chef_can", 2: "003_cracker_box", 3: "004_sugar_box",
+    4: "005_tomato_soup_can", 5: "006_mustard_bottle", 6: "007_tuna_fish_can",
+    7: "008_pudding_box", 8: "009_gelatin_box", 9: "010_potted_meat_can",
+    10: "011_banana", 11: "019_pitcher_base", 12: "021_bleach_cleanser",
+    13: "024_bowl", 14: "025_mug", 15: "035_power_drill", 16: "036_wood_block",
+    17: "037_scissors", 18: "040_large_marker", 19: "051_large_clamp",
+    20: "052_extra_large_clamp", 21: "061_foam_brick",
+}
+
+
+@dataclass
+class Sample:
+    seq_dir: str
+    frame: int
+    obj_id: int
+    obj_index: int          # index into the sequence's ycb_ids / pose_y
+    anchor_serial: str
+    aux_serials: List[str]  # FULL pool of non-anchor cameras; subset chosen per __getitem__
+    extrinsics_tag: str
+
+
+# --------------------------------------------------------------------------- #
+# geometry helpers
+# --------------------------------------------------------------------------- #
+def _unproject(depth_m: np.ndarray, intr: dict,
+               depth_near: float = DEPTH_NEAR, depth_far: float = DEPTH_FAR) -> np.ndarray:
+    """aligned depth (H,W) metres → (H,W,3) pointmap in OpenCV cam frame, NaN invalid."""
+    H, W = depth_m.shape
+    fx, fy, cx, cy = intr["fx"], intr["fy"], intr["ppx"], intr["ppy"]
+    us, vs = np.meshgrid(np.arange(W), np.arange(H))
+    z = depth_m
+    x = (us - cx) / fx * z
+    y = (vs - cy) / fy * z
+    pm = np.stack([x, y, z], axis=-1).astype(np.float64)
+    invalid = (z <= depth_near) | (z >= depth_far)
+    pm[invalid] = np.nan
+    return pm
+
+
+def _to_p3d(pm_cv: np.ndarray) -> np.ndarray:
+    """OpenCV pointmap → PyTorch3D (flip x,y)."""
+    return pm_cv * np.array([-1.0, -1.0, 1.0])
+
+
+from mvsam3d import paths as _mv_paths
+_FORK_ROOT = _mv_paths.FORK_ROOT
+
+
+def _load_object_centric_ssi():
+    """[raw SAM-3D] import the vendored ObjectCentricSSI from the fork.
+
+    This repo vendors a PARTIAL ``sam3d_objects`` subset that shadows the fork's
+    full package on sys.path, so ``sam3d_objects.data.dataset`` fails to import.
+    Fix: extend the partial packages' ``__path__`` with the fork's directories so
+    the missing subpackages resolve there (pure reuse — never reimplement).
+    Additive only: names that already resolve keep resolving to this repo."""
+    try:
+        from sam3d_objects.data.dataset.tdfy.img_and_mask_transforms import ObjectCentricSSI
+        return ObjectCentricSSI
+    except ModuleNotFoundError:
+        import sam3d_objects
+        import sam3d_objects.data
+        for pkg, sub in ((sam3d_objects, "sam3d_objects"),
+                         (sam3d_objects.data, os.path.join("sam3d_objects", "data"))):
+            p = os.path.join(_FORK_ROOT, sub)
+            if p not in pkg.__path__:
+                pkg.__path__.append(p)
+        from sam3d_objects.data.dataset.tdfy.img_and_mask_transforms import ObjectCentricSSI
+        return ObjectCentricSSI
+
+
+def _ssi_params(pm: np.ndarray, mask: np.ndarray | None = None, mode: str = "scene"):
+    """SSI params from a pointmap. Two selectable conventions (``mode``):
+
+    * ``"scene"`` (default, used by all pre-2026-07-17 checkpoints):
+      shift=[0,0,median_z], scale=mean|p-shift| over the whole scene. Matches the
+      fork's BASE ``ScaleShiftInvariant.get_scale_and_shift`` default
+      (NOT ObjectCentricSSI).
+    * ``"object_centric"``: [raw SAM-3D] vendored ``ObjectCentricSSI``
+      (use_scene_scale=True) — shift = 3D median of the points INSIDE the object
+      ``mask`` (object recentred to the origin), scale = scene scale. This is the
+      convention the release HF checkpoint's inference pipeline.yaml pairs with
+      the frozen condition encoder. Requires ``mask``.
+
+    A checkpoint must always be evaluated with the mode it was trained with —
+    t/s targets and the cam features live in this SSI space.
+    """
+    if mode == "object_centric":
+        if mask is None:
+            raise ValueError("_ssi_params(mode='object_centric') requires the object mask")
+        ObjectCentricSSI = _load_object_centric_ssi()
+        pm_t = torch.from_numpy(np.ascontiguousarray(pm)).float().permute(2, 0, 1)   # [3,H,W]
+        mask_t = torch.from_numpy(np.ascontiguousarray(mask)).float()[None]          # [1,H,W]
+        scale_t, shift_t = ObjectCentricSSI(use_scene_scale=True)._compute_scale_and_shift(pm_t, mask_t)
+        shift = shift_t.numpy().astype(np.float64)                                   # [3]
+        scale = max(float(scale_t.reshape(-1)[0]), 1e-6)                             # isotropic
+        return shift, scale
+    if mode != "scene":
+        raise ValueError(f"unknown ssi mode: {mode!r} (expected 'scene' or 'object_centric')")
+    flat = pm.reshape(-1, 3)
+    valid = np.isfinite(flat).all(axis=1)
+    if valid.sum() < 16:
+        return np.zeros(3), 1.0
+    pts = flat[valid]
+    shift = np.array([0.0, 0.0, np.median(pts[:, 2])])
+    scale = float(np.mean(np.abs(pts - shift)))
+    scale = max(scale, 1e-6)
+    return shift, scale
+
+
+def _apply_ssi(pm: np.ndarray, shift, scale) -> np.ndarray:
+    return (pm - shift) / scale
+
+
+# Per-view camera feature width fed to the model's mlp_cam: rot6d(6) + camera
+# centre in the anchor frame, SSI-normalized (3).
+CAM_FEAT_DIM = 9
+
+
+def _cam_feature(Rrel: np.ndarray, trel: np.ndarray, shift, scale) -> torch.Tensor:
+    """Pack a view's camera->anchor extrinsics into a 9-vector for ``mlp_cam``.
+
+    ``[0:6]`` = column-convention rot6d of ``Rrel`` (the aux->anchor rotation),
+    using the SAME vendored ``matrix_to_rot6d`` as the object-rotation target so
+    the convention is consistent (NOT ``normalize_rot6d`` — that's tuned to the
+    object distribution; raw rot6d columns are already unit-scaled).
+    ``[6:9]`` = the camera centre in the anchor frame (which is exactly ``trel``,
+    the image of the aux origin under ``X_anchor = Rrel X_aux + trel``), expressed
+    in the SAME SSI space as the conditioning pointmaps -> ``(trel - shift)/scale``.
+
+    Computed uniformly for every view: the anchor passes ``Rrel=I, trel=0`` (its
+    centre normalizes to ``-shift/scale``), so there is no special-casing.
+    """
+    rot6d = matrix_to_rot6d(torch.from_numpy(np.asarray(Rrel, np.float64)).float())  # [6]
+    centre = torch.from_numpy(np.asarray(trel, np.float64)).float()                  # [3]
+    shift_t = torch.from_numpy(np.asarray(shift, np.float64)).float()                # [3]
+    pos = (centre - shift_t) / float(scale)                                          # [3]
+    return torch.cat([rot6d, pos])                                                   # [9]
+
+
+def _mask_bbox(mask: np.ndarray, pad: float = 0.1):
+    """Square bbox around mask with padding → (r0,c0,r1,c1). Full frame if empty."""
+    H, W = mask.shape
+    ys, xs = np.where(mask > 0)
+    if ys.size == 0:
+        return 0, 0, H, W
+    r0, r1 = ys.min(), ys.max() + 1
+    c0, c1 = xs.min(), xs.max() + 1
+    h, w = r1 - r0, c1 - c0
+    side = max(h, w) * (1.0 + pad)
+    cy, cx = (r0 + r1) / 2.0, (c0 + c1) / 2.0
+    r0 = int(round(cy - side / 2)); r1 = int(round(cy + side / 2))
+    c0 = int(round(cx - side / 2)); c1 = int(round(cx + side / 2))
+    return r0, c0, r1, c1
+
+
+def _crop_resize(arr: torch.Tensor, box, size: int, mode: str, fill: float):
+    """arr [C,H,W] → [C,size,size]; crop to (possibly out-of-bounds) box then resize.
+
+    Out-of-bounds region is filled with `fill` (NaN-safe for pointmaps).  Nearest
+    interpolation is used for pointmaps/masks so NaN is preserved (never averaged).
+    """
+    C, H, W = arr.shape
+    r0, c0, r1, c1 = box
+    bh, bw = r1 - r0, c1 - c0
+    canvas = arr.new_full((C, bh, bw), fill)
+    sr0, sc0 = max(r0, 0), max(c0, 0)
+    sr1, sc1 = min(r1, H), min(c1, W)
+    if sr1 > sr0 and sc1 > sc0:
+        canvas[:, sr0 - r0:sr1 - r0, sc0 - c0:sc1 - c0] = arr[:, sr0:sr1, sc0:sc1]
+    interp = {"bilinear": "bilinear", "nearest": "nearest"}[mode]
+    kw = {"align_corners": False} if interp == "bilinear" else {}
+    out = F.interpolate(canvas[None], size=(size, size), mode=interp, **kw)[0]
+    return out
+
+
+def _full_resize(arr: torch.Tensor, size: int, mode: str, fill: float):
+    """Pad [C,H,W] to square (centered) then resize to size."""
+    C, H, W = arr.shape
+    s = max(H, W)
+    canvas = arr.new_full((C, s, s), fill)
+    r0, c0 = (s - H) // 2, (s - W) // 2
+    canvas[:, r0:r0 + H, c0:c0 + W] = arr
+    interp = {"bilinear": "bilinear", "nearest": "nearest"}[mode]
+    kw = {"align_corners": False} if interp == "bilinear" else {}
+    return F.interpolate(canvas[None], size=(size, size), mode=interp, **kw)[0]
+
+
+# --------------------------------------------------------------------------- #
+# dataset
+# --------------------------------------------------------------------------- #
+def _build_view_fn(rgb, mask, pm_anchor_frame, anchor: bool, image_size: int):
+    """Module-level view construction (shared by MultiViewDexYCB._build_view and
+    the dataset-agnostic MultiViewPoseDataset). Byte-identical logic."""
+    img_t = torch.from_numpy(rgb).permute(2, 0, 1).float()
+    msk_t = torch.from_numpy(mask)[None].float()
+    pm_t = torch.from_numpy(pm_anchor_frame).permute(2, 0, 1).float()
+    box = _mask_bbox(mask)
+    out = {
+        "image_cropped": _crop_resize(img_t, box, image_size, "bilinear", 0.0),
+        "mask_cropped": _crop_resize(msk_t, box, image_size, "nearest", 0.0),
+        "pointmap_cropped": _crop_resize(pm_t, box, image_size, "nearest", float("nan")),
+    }
+    if anchor:
+        out["image_full"] = _full_resize(img_t, image_size, "bilinear", 0.0)
+        out["mask_full"] = _full_resize(msk_t, image_size, "nearest", 0.0)
+        out["pointmap_full"] = _full_resize(pm_t, image_size, "nearest", float("nan"))
+    return out
+
+
+class MultiViewDexYCB(Dataset):
+    def __init__(
+        self,
+        dexycb_root: str = os.path.join(_mv_paths.DATA_ROOT, "dexycb"),
+        canonical_cache: str = _mv_paths.CANONICAL_CACHE,
+        num_aux_views: int = 7,
+        frame_stride: int = 5,
+        anchor_sweep: bool = True,
+        subjects: Optional[List[str]] = None,
+        image_size: int = IMG_SIZE,
+        random_views: bool = False,
+        min_aux_views: int = 1,
+        max_aux_views: int = 3,
+        depth_scale: float = DEPTH_SCALE,
+        depth_near: float = DEPTH_NEAR,
+        depth_far: float = DEPTH_FAR,
+        split: str = "train",
+        val_fraction: float = 0.0,
+        split_seed: int = 0,
+        ssi_mode: str = "scene",
+    ):
+        """When ``random_views`` is True, each __getitem__ draws a random aux count
+        ``k ~ U[min_aux_views, max_aux_views]`` and a random subset of that size
+        from the non-anchor cameras (total views = 1 + k, i.e. 2..4 by default).
+        When False, the first ``num_aux_views`` aux cameras are used."""
+        self.root = dexycb_root
+        self.calib_root = os.path.join(dexycb_root, "calibration")
+        self.cache = canonical_cache
+        self.num_aux = num_aux_views
+        self.image_size = image_size
+        self.random_views = random_views
+        self.min_aux = min_aux_views
+        self.max_aux = max_aux_views
+        self.depth_scale = depth_scale
+        self.depth_near = depth_near
+        self.depth_far = depth_far
+        # SSI convention for pointmaps + t/s targets + cam features ("scene" |
+        # "object_centric"); validated eagerly so a config typo fails at startup.
+        if ssi_mode not in ("scene", "object_centric"):
+            raise ValueError(f"ssi_mode must be 'scene' or 'object_centric', got {ssi_mode!r}")
+        self.ssi_mode = ssi_mode
+        if random_views and not (1 <= min_aux_views <= max_aux_views):
+            raise ValueError(
+                f"random_views requires 1 <= min_aux_views ({min_aux_views}) "
+                f"<= max_aux_views ({max_aux_views})"
+            )
+        self._canon: Dict[str, dict] = {}
+
+        seq_dirs = sorted(glob.glob(os.path.join(dexycb_root, "2020*-subject-*", "2020*")))
+        if subjects is not None:
+            seq_dirs = [d for d in seq_dirs if any(s in d for s in subjects)]
+
+        self.samples: List[Sample] = []
+        for seq_dir in seq_dirs:
+            meta = dexycb_calib._load_yaml(os.path.join(seq_dir, "meta.yml"))
+            serials = [str(s) for s in meta["serials"]]
+            ycb_ids = list(meta["ycb_ids"])
+            tag = str(meta["extrinsics"])
+            n = int(meta["num_frames"])
+            anchors = serials if anchor_sweep else serials[:1]
+            for frame in range(0, n, frame_stride):
+                for anchor in anchors:
+                    aux = [s for s in serials if s != anchor]   # full pool
+                    for oi, oid in enumerate(ycb_ids):
+                        if int(oid) not in YCB_ID_TO_NAME:
+                            continue
+                        self.samples.append(
+                            Sample(seq_dir, frame, int(oid), oi, anchor, aux, tag)
+                        )
+
+        # ---- deterministic train/val split -------------------------------
+        # A fixed-seed shuffle partitions the full sample list once; the same
+        # (val_fraction, split_seed) yields the same disjoint train/val sets, so
+        # validation never touches a training sample.  val_fraction=0 keeps all
+        # samples in "train" (no split).
+        if val_fraction > 0.0:
+            if not 0.0 < val_fraction < 1.0:
+                raise ValueError(f"val_fraction must be in (0,1), got {val_fraction}")
+            order = list(range(len(self.samples)))
+            random.Random(split_seed).shuffle(order)
+            n_val = max(1, int(round(len(order) * val_fraction)))
+            val_idx = set(order[:n_val])
+            if split == "val":
+                self.samples = [self.samples[i] for i in sorted(val_idx)]
+            elif split == "train":
+                self.samples = [self.samples[i] for i in range(len(self.samples)) if i not in val_idx]
+            else:
+                raise ValueError(f"split must be 'train' or 'val', got {split!r}")
+
+    def __len__(self):
+        return len(self.samples)
+
+    # ---- per-object canonical cache ----
+    def _canonical(self, obj_id: int) -> dict:
+        name = YCB_ID_TO_NAME[obj_id]
+        if name not in self._canon:
+            d = torch.load(os.path.join(self.cache, f"{name}.pt"), map_location="cpu",
+                           weights_only=False)
+            self._canon[name] = d
+        return self._canon[name]
+
+    # ---- raw IO ----
+    def _load_color(self, seq_dir, serial, frame) -> np.ndarray:
+        p = os.path.join(seq_dir, serial, f"color_{frame:06d}.jpg")
+        return np.asarray(Image.open(p).convert("RGB"), dtype=np.float32) / 255.0
+
+    def _load_depth(self, seq_dir, serial, frame) -> np.ndarray:
+        p = os.path.join(seq_dir, serial, f"aligned_depth_to_color_{frame:06d}.png")
+        return np.asarray(Image.open(p), dtype=np.float32) / self.depth_scale
+
+    def _load_seg(self, seq_dir, serial, frame, obj_id) -> np.ndarray:
+        p = os.path.join(seq_dir, serial, f"labels_{frame:06d}.npz")
+        seg = np.load(p)["seg"]
+        return (seg == obj_id).astype(np.float32)
+
+    def _pose_y(self, seq_dir, serial, frame, obj_index):
+        p = os.path.join(seq_dir, serial, f"labels_{frame:06d}.npz")
+        py = np.load(p)["pose_y"][obj_index]      # (3,4) OpenCV CAD->cam
+        return py[:, :3].astype(np.float64), py[:, 3].astype(np.float64)
+
+    # ---- view tensor construction ----
+    def _build_view(self, rgb, mask, pm_anchor_frame, anchor: bool):
+        """rgb [H,W,3] in [0,1]; mask [H,W]; pm [H,W,3] (anchor frame, already SSI).
+
+        Returns a dict {image_cropped, mask_cropped, pointmap_cropped, [+ *_full]}.
+        """
+        # Delegates to the module-level _build_view_fn (byte-identical). Pointmaps
+        # intentionally KEEP NaN at invalid/background pixels so the pretrained
+        # PointPatchEmbed's isfinite() valid_mask + invalid_xyz_token path works.
+        return _build_view_fn(rgb, mask, pm_anchor_frame, anchor, self.image_size)
+
+    def __getitem__(self, idx: int) -> dict:
+        s = self.samples[idx]
+        ext = dexycb_calib.load_extrinsics(self.calib_root, s.extrinsics_tag)
+
+        # ---- anchor view + shared SSI from anchor pointmap ----
+        a_rgb = self._load_color(s.seq_dir, s.anchor_serial, s.frame)
+        a_mask = self._load_seg(s.seq_dir, s.anchor_serial, s.frame, s.obj_id)
+        a_intr = dexycb_calib.load_intrinsics(self.calib_root, s.anchor_serial)
+        a_pm = _to_p3d(_unproject(self._load_depth(s.seq_dir, s.anchor_serial, s.frame), a_intr,
+                                  self.depth_near, self.depth_far))
+        shift, scale = _ssi_params(a_pm, mask=a_mask, mode=self.ssi_mode)
+
+        views = [self._build_view(a_rgb, a_mask, _apply_ssi(a_pm, shift, scale), anchor=True)]
+        # Anchor camera == the reference frame: relative transform is identity.
+        views[0]["cam"] = _cam_feature(np.eye(3), np.zeros(3), shift, scale)
+
+        # ---- choose which aux cameras to use (random count + subset, or fixed) ----
+        pool = s.aux_serials
+        if self.random_views:
+            k = random.randint(self.min_aux, min(self.max_aux, len(pool)))
+            aux_serials = random.sample(pool, k)
+        else:
+            aux_serials = pool[: self.num_aux]
+
+        # ---- aux views: transform pointmap into anchor frame, share SSI ----
+        for aux in aux_serials:
+            x_rgb = self._load_color(s.seq_dir, aux, s.frame)
+            x_mask = self._load_seg(s.seq_dir, aux, s.frame, s.obj_id)
+            x_intr = dexycb_calib.load_intrinsics(self.calib_root, aux)
+            x_pm = _to_p3d(_unproject(self._load_depth(s.seq_dir, aux, s.frame), x_intr,
+                                      self.depth_near, self.depth_far))
+            Rrel, trel = dexycb_calib.relative_p3d(ext, s.anchor_serial, aux)
+            x_pm = dexycb_calib.transform_pointmap_to_anchor(x_pm, Rrel, trel)
+            av = self._build_view(x_rgb, x_mask, _apply_ssi(x_pm, shift, scale), anchor=False)
+            # Bind this aux view to its camera pose (aux->anchor extrinsics).
+            av["cam"] = _cam_feature(Rrel, trel, shift, scale)
+            views.append(av)
+
+        # ---- GT layout (anchor cam, PyTorch3D) ----
+        R_cv, t_cv = self._pose_y(s.seq_dir, s.anchor_serial, s.frame, s.obj_index)
+        R_p3d, t_p3d = _D @ R_cv, _D @ t_cv          # OpenCV → PyTorch3D
+        canon = self._canonical(s.obj_id)
+        R, t, sc = canonical.gt_layout_anchor_frame(
+            R_p3d, t_p3d, np.asarray(canon["center_cad"]), float(canon["max_extent"])
+        )
+        # 6D rotation: VENDORED column-convention encode, normalized to the
+        # pretrained backbone's distribution. Consistent with the model's decode
+        # rot6d_to_matrix(denormalize_rot6d(·)).
+        R_t = torch.from_numpy(R).float()
+        rot6d = normalize_rot6d(matrix_to_rot6d(R_t)).float()   # [6]
+
+        # ---- SSI-consistent pose targets ----------------------------------
+        # Mirror the fork stage1 model_wrapper._build_pose_latents EXACTLY
+        # (sam3d-for-6d stage1 model_wrapper.py lines 560-581);
+        # the conditioning pointmaps were normalized by the shared anchor SSI
+        # (shift [3], scale scalar), so the t/s targets share that SSI space.
+        # Guards use the same constants as the fork:
+        #   min_valid_pointmap_scale = 1e-6  (clamps pm_scale and the scale ratio)
+        # No mean/std on t or s (fork normalize=False; NOT the ScaleShiftInvariant
+        # pose-target/inference decoder). 3-channel isotropic scale latent.
+        MIN_VALID_PM_SCALE = 1e-6
+        shift_t = torch.from_numpy(np.asarray(shift, dtype=np.float64)).float()   # [3]
+        # fork line 560: pm_scale_safe = pm_scale.float().clamp_min(min_valid_pointmap_scale)
+        pm_scale_safe = torch.tensor(float(scale), dtype=torch.float32).clamp_min(MIN_VALID_PM_SCALE)
+        # fork line 561: gt_trans_ssi = (gt_trans_metric - pm_shift) / pm_scale_safe
+        t_ssi = (torch.from_numpy(t).float() - shift_t) / pm_scale_safe          # [3]
+        # fork lines 580-581: gt_scale_ratio = (gt_scale_m / pm_scale_scalar).clamp_min(min_valid_pointmap_scale)
+        #                     gt_scale_log = log(gt_scale_ratio).expand(...,3)
+        gt_scale_m = torch.tensor(float(sc), dtype=torch.float32)                # scalar metric extent
+        gt_scale_ratio = (gt_scale_m / pm_scale_safe).clamp_min(MIN_VALID_PM_SCALE)
+        s_log3 = torch.log(gt_scale_ratio).expand(3).contiguous()               # [3] isotropic
+
+        return {
+            "shape_latent": canon["latent"].float(),            # [8,16,16,16]
+            "rot6d": rot6d,                                     # [6]
+            "translation": t_ssi,                              # [3] SSI space
+            "scale": s_log3,                                   # [3] log-ratio, isotropic
+            "pm_shift": shift_t,                               # [3] SSI shift
+            "pm_scale": pm_scale_safe.reshape(1),              # [1] SSI scale (clamped, fork-safe)
+            "views": views,
+            "object_name": YCB_ID_TO_NAME[s.obj_id],
+        }
+
+
+def collate_multiview(batch: List[dict]) -> dict:
+    """Stack a list of samples. ``views`` becomes a list (len=n_views) of dicts of
+    batched [B,C,H,W] tensors; the rest stack on dim 0.
+
+    Samples may carry different view counts (random-view training).  All views at
+    a given position share the same group structure (position 0 = anchor with 6
+    groups, positions >=1 = aux with 3 cropped groups), so the batch is truncated
+    to the minimum view count present.  At batch_size=1 this is a no-op."""
+    n_views = min(len(b["views"]) for b in batch)
+    out = {
+        "shape_latent": torch.stack([b["shape_latent"] for b in batch]),
+        "rot6d": torch.stack([b["rot6d"] for b in batch]),
+        "translation": torch.stack([b["translation"] for b in batch]),
+        "scale": torch.stack([b["scale"] for b in batch]),
+        "pm_shift": torch.stack([b["pm_shift"] for b in batch]),
+        "pm_scale": torch.stack([b["pm_scale"] for b in batch]),
+        "object_name": [b["object_name"] for b in batch],
+    }
+    views = []
+    for v in range(n_views):
+        keys = batch[0]["views"][v].keys()
+        views.append({k: torch.stack([b["views"][v][k] for b in batch]) for k in keys})
+    out["views"] = views
+    return out

migrator/code/mv-sam3d-for-6d-v2/mvsam3d/data/integrated.py

@@ -0,0 +1,120 @@
+"""Integrated multi-dataset training set builder + held-out enforcement.
+
+Reads a ``data.datasets`` list from the training config, instantiates one
+MultiViewPoseDataset per entry (via its adapter), enforces the held-out manifest
+(configs/heldout.yaml), concatenates, and provides a diversity-weighted sampler.
+
+HELD-OUT SEMANTICS (two distinct kinds — the distinction is load-bearing):
+  * OOD datasets (ho3d, ycbm, hope): whole-dataset held-out for evaluation. Their
+    objects MAY overlap training objects (e.g. HO-3D is YCB). We do NOT enforce
+    object-disjointness for these — doing so would delete every YCB object from
+    DexYCB/YCB-V training. They are simply never built with split:train.
+  * unseen-object datasets (housecat6d, graspnet test_novel): held out AND their
+    object identities reserved — removed from EVERY train dataset (global
+    object-disjointness). Their objects are non-YCB, so this is safe.
+"""
+from __future__ import annotations
+
+import os
+from typing import Dict, List
+
+import numpy as np
+import torch
+import yaml
+from torch.utils.data import ConcatDataset, WeightedRandomSampler
+
+from .adapters import ADAPTERS
+from .base_dataset import MultiViewPoseDataset
+
+
+import inspect
+
+
+def _adapter_kwargs(adapter_cls, entry: dict) -> dict:
+    """Pass through only the entry keys the adapter's __init__ actually accepts.
+    `role` is ROUTING (train vs heldout) and is never passed to the adapter;
+    `data_split` maps to the adapter's on-disk `split` (test/train/val)."""
+    accepted = set(inspect.signature(adapter_cls.__init__).parameters) - {"self", "root", "canonical_cache"}
+    kw = {k: entry[k] for k in accepted if k in entry and k not in ("role", "data_split")}
+    if "split" in accepted and "data_split" in entry:
+        kw["split"] = entry["data_split"]
+    return kw
+
+
+def load_heldout(path: str | None) -> dict:
+    if not path or not os.path.exists(path):
+        return {"ood_datasets": [], "unseen_object_datasets": [],
+                "unseen_object_keys_extra": [], "never_train": []}
+    with open(path) as f:
+        return yaml.safe_load(f) or {}
+
+
+def reserved_object_keys(heldout: dict, canonical_cache: str) -> set:
+    """Union of obj_keys from unseen-object datasets + explicit extras.
+    These keys are removed from ALL training datasets."""
+    keys = set(heldout.get("unseen_object_keys_extra", []) or [])
+    for name in heldout.get("unseen_object_datasets", []) or []:
+        ds_root = heldout["roots"][name]
+        ad = ADAPTERS[name](ds_root, canonical_cache, **heldout.get("adapter_kwargs", {}).get(name, {}))
+        for s in ad.list_samples():
+            keys.add(s.obj_key)
+    return keys
+
+
+def build_integrated_dataset(data_cfg: dict, split: str, *, image_size: int,
+                             random_views: bool, min_aux: int, max_aux: int,
+                             num_aux: int, depth_near: float, depth_far: float,
+                             ssi_mode: str = "scene",
+                             heldout_path: str | None = None):
+    """Build a ConcatDataset over the configured datasets for `split`, applying
+    held-out rules. Returns (dataset, per_sample_weights or None)."""
+    cache = data_cfg["canonical_cache"]
+    heldout = load_heldout(heldout_path)
+    never_train = set(heldout.get("never_train", []) or [])
+    reserved = reserved_object_keys(heldout, cache) if split == "train" else set()
+
+    parts: List[MultiViewPoseDataset] = []
+    weights: List[float] = []
+    report = []
+    for entry in data_cfg["datasets"]:
+        name = entry["name"]
+        adapter_name = entry.get("adapter", name)
+        role = entry.get("role", "train")   # routing: "train" | "heldout"
+        # only build datasets whose role matches the request
+        if split == "train":
+            if role != "train" or name in never_train:
+                continue
+        else:  # val / held-out
+            if role == "train":
+                continue
+        ad = ADAPTERS[adapter_name](entry["root"], cache, **_adapter_kwargs(ADAPTERS[adapter_name], entry))
+        ds = MultiViewPoseDataset(ad, image_size=image_size,
+                                  random_views=random_views if split == "train" else False,
+                                  num_aux_views=num_aux, min_aux_views=min_aux,
+                                  max_aux_views=max_aux, depth_near=depth_near, depth_far=depth_far,
+                                  ssi_mode=ssi_mode)
+        # object-disjointness filter (train only)
+        if split == "train" and reserved:
+            kept = [i for i, s in enumerate(ds.samples) if s.obj_key not in reserved]
+            ds.samples = [ds.samples[i] for i in kept]
+        if len(ds) == 0:
+            continue
+        parts.append(ds)
+        w = float(entry.get("weight", 1.0))
+        weights.extend([w / len(ds)] * len(ds))   # diversity weight: per-dataset mixing ratio
+        report.append(f"{name}:{len(ds)}(w={w})")
+
+    if not parts:
+        raise RuntimeError(f"No datasets for split={split}")
+    concat = ConcatDataset(parts)
+    print(f"[integrated] {split}: " + " ".join(report) + f" -> total {len(concat)}", flush=True)
+    sample_weights = torch.tensor(weights, dtype=torch.double) if split == "train" else None
+    return concat, sample_weights
+
+
+def make_weighted_sampler(sample_weights, *, world_size: int, rank: int, seed: int = 0):
+    """Diversity-weighted sampler. For DDP, each rank draws an independent
+    weighted stream (seeded by rank) — the mixing ratio is preserved per rank."""
+    g = torch.Generator().manual_seed(seed + rank)
+    return WeightedRandomSampler(sample_weights, num_samples=len(sample_weights),
+                                 replacement=True, generator=g)

migrator/code/mv-sam3d-for-6d-v2/mvsam3d/infer/__init__.py

@@ -0,0 +1,17 @@
+"""Inference package for the multi-view 6D-pose flow model (v2).
+
+Reusable, data-source-decoupled inference that mirrors the EXACT inference path
+used by ``mvsam3d/train/validate.run_validation``:
+
+  * build the unwrapped ``MVSam3DFlow`` with the pretrained warm-start so the
+    FROZEN condition fuser + warm-started backbone are reconstructed, THEN
+  * restore the trained tensors (backbone + mlp_cam) with the vendored
+    ``load_checkpoint`` (v2 checkpoints store ONLY trainable params), THEN
+  * run ``inner.predict(batch, num_steps=...)`` under ``eval()`` + AMP autocast
+    + ``no_grad`` — byte-for-byte the same sampler/decoder validation uses.
+"""
+from .predict import (  # noqa: F401
+    build_inference_model,
+    load_config,
+    predict_pose,
+)

migrator/code/mv-sam3d-for-6d-v2/mvsam3d/infer/predict.py

@@ -0,0 +1,373 @@
+"""Reusable eval-time inference for the multi-view 6D-pose flow model (v2).
+
+This module is the single source of truth for "load a trained v2 checkpoint and
+predict (R, t, s, shape) for a batch".  It reproduces the inference path of
+``mvsam3d/train/validate.run_validation`` EXACTLY:
+
+  * model construction mirrors ``mvsam3d/train/train.main`` —
+    ``MVSam3DFlow(... pretrained_ckpt=<ss_generator.ckpt> ...)`` first (which
+    reconstructs the FROZEN condition fuser + warm-starts the backbone), then the
+    vendored ``load_checkpoint`` overwrites ONLY the trained tensors (backbone +
+    mlp_cam) on top of that warm-start.  v2 checkpoints under cache/ckpt store
+    trainable params only, so the warm-start MUST happen first or the frozen
+    condition pathway would be left uninitialized.
+  * prediction mirrors ``run_validation``: ``inner.eval()``, the SAME AMP
+    autocast (bf16 on cuda by default), ``torch.no_grad()``, and a single
+    ``inner.predict(batch, num_steps=...)`` call — same vendored Euler sampler,
+    same ``num_steps`` semantics, same ``decode_pose_*`` helpers.
+
+The pose-prediction entrypoint ``predict_pose(model, batch, num_steps)`` is
+deliberately DECOUPLED from any data source: it accepts an already-collated batch
+dict (``views`` + ``pm_shift`` / ``pm_scale`` + ...), so an HO3D relative-pose
+eval harness can import and call it with its own batches.  The CLI below wires a
+DexYCB batch source around it purely for the smoke test / standalone runs.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from typing import Dict, List, Optional, Tuple
+
+import torch
+import yaml
+
+from sam3d_objects.training.stage1_flow_lora.checkpointing import load_checkpoint
+from mvsam3d.model.flow_model import MVSam3DFlow
+from mvsam3d import paths as _paths
+
+DEFAULT_CONFIG = _paths.DEFAULT_CONFIG
+
+
+# --------------------------------------------------------------------------- #
+# config + small utils
+# --------------------------------------------------------------------------- #
+def load_config(path: str) -> dict:
+    """Load the training YAML (same keys train.py reads)."""
+    with open(path) as f:
+        return _paths.expand(yaml.safe_load(f))
+
+
+def _move(obj, device):
+    """Recursively move tensors in a (possibly nested) batch to ``device``."""
+    if torch.is_tensor(obj):
+        return obj.to(device, non_blocking=True)
+    if isinstance(obj, dict):
+        return {k: _move(v, device) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [_move(v, device) for v in obj]
+    return obj
+
+
+# --------------------------------------------------------------------------- #
+# model construction + checkpoint restore (mirrors train.main order)
+# --------------------------------------------------------------------------- #
+def build_inference_model(
+    cfg: dict,
+    checkpoint_path: str,
+    device,
+    *,
+    verbose: bool = True,
+) -> Tuple[MVSam3DFlow, Dict]:
+    """Build ``MVSam3DFlow`` with the pretrained warm-start, then restore the
+    trained tensors from ``checkpoint_path`` with the vendored ``load_checkpoint``.
+
+    Returns ``(model_in_eval_mode, load_report)`` where ``load_report`` carries the
+    restored ``step`` plus the ``load_state_dict`` missing/unexpected key report.
+
+    Construction order is IDENTICAL to ``train.main``:
+      1. ``MVSam3DFlow(..., pretrained_ckpt=model_cfg['pretrained_ckpt'], ...)``
+         — reconstructs the frozen fuser + warm-starts the backbone.
+      2. vendored ``load_checkpoint(checkpoint_path, model)`` — overwrites the
+         trainable tensors (backbone + mlp_cam) on top of the warm-start.
+    """
+    model_cfg = cfg["model"]
+    cond_cfg = cfg["condition"]
+    loss_cfg = cfg["loss_weights"]
+    data_cfg = cfg["data"]
+
+    # max_views derivation copied verbatim from train.main (API-compat only; the
+    # per-view bias now comes from mlp_cam, so this is not a hard upper bound).
+    max_aux = (data_cfg["max_aux_views"] if data_cfg["random_views"]
+               else data_cfg["num_aux_views"])
+    max_views = max_aux + 1
+
+    # (1) warm-start: pretrained_ckpt reconstructs frozen fuser + backbone torso.
+    model = MVSam3DFlow(
+        model_channels=model_cfg["model_channels"],
+        cond_channels=model_cfg["cond_channels"],
+        num_blocks=model_cfg["num_blocks"],
+        num_heads=model_cfg["num_heads"],
+        max_views=max_views,
+        loss_weights=loss_cfg,
+        sigma_min=model_cfg["sigma_min"],
+        time_scale=model_cfg["time_scale"],
+        timestep_sampler=model_cfg["timestep_sampler"],
+        timestep_lognorm_mean=model_cfg["timestep_lognorm_mean"],
+        timestep_lognorm_std=model_cfg["timestep_lognorm_std"],
+        pretrained_ckpt=model_cfg["pretrained_ckpt"],
+        condition_kwargs=cond_cfg,
+    ).to(device)
+
+    # (2) restore TRAINABLES on top of the warm-start, via the vendored loader.
+    # We transiently wrap the model's load_state_dict to capture the
+    # IncompatibleKeys (missing/unexpected) that load_checkpoint discards, so we
+    # can report them. The actual restore is still done by the vendored
+    # load_checkpoint (single 1x checkpoint read; its m.load_state_dict call picks
+    # up the instance-level override below).
+    inner = model.module if hasattr(model, "module") else model
+    _orig_load = inner.load_state_dict
+    captured: Dict = {}
+
+    def _capturing_load(state_dict, strict=False, **kw):
+        res = _orig_load(state_dict, strict=strict, **kw)
+        captured["restored_tensors"] = len(state_dict)
+        captured["missing_keys"] = list(getattr(res, "missing_keys", []))
+        captured["unexpected_keys"] = list(getattr(res, "unexpected_keys", []))
+        return res
+
+    inner.load_state_dict = _capturing_load  # instance attr shadows bound method
+    try:
+        step, history = load_checkpoint(
+            checkpoint_path=checkpoint_path, model=model, map_location="cpu",
+        )
+    finally:
+        inner.load_state_dict = _orig_load
+
+    model.eval()  # same eval() mode validation uses for the sampler
+
+    # Sanity: every trainable param name should be present in the restored set
+    # (i.e. NOT in missing_keys). missing_keys here are the frozen fuser / DINO
+    # tensors that the trainable-only checkpoint legitimately does not carry.
+    trainable_names = {n for n, p in inner.named_parameters() if p.requires_grad}
+    missing_trainable = sorted(trainable_names & set(captured.get("missing_keys", [])))
+
+    report = {
+        "step": int(step),
+        "restored_tensors": captured.get("restored_tensors", 0),
+        "n_missing_keys": len(captured.get("missing_keys", [])),
+        "n_unexpected_keys": len(captured.get("unexpected_keys", [])),
+        "unexpected_keys": captured.get("unexpected_keys", []),
+        "n_trainable_params": len(trainable_names),
+        "missing_trainable_params": missing_trainable,
+    }
+    if verbose:
+        print(f"[infer] built MVSam3DFlow + warm-start from "
+              f"{model_cfg['pretrained_ckpt']}")
+        print(f"[infer] load_checkpoint({checkpoint_path}) -> step={report['step']}; "
+              f"restored {report['restored_tensors']} trainable tensors; "
+              f"{report['n_missing_keys']} keys not in ckpt (frozen fuser/DINO, "
+              f"expected); {report['n_unexpected_keys']} unexpected.")
+        if missing_trainable:
+            print(f"[infer][WARN] {len(missing_trainable)} TRAINABLE params were "
+                  f"NOT restored (left at warm-start/init): {missing_trainable[:8]}"
+                  + (" ..." if len(missing_trainable) > 8 else ""))
+        else:
+            print("[infer] all trainable params restored from the checkpoint: OK")
+    return model, report
+
+
+# --------------------------------------------------------------------------- #
+# the decoupled pose-prediction entrypoint
+# --------------------------------------------------------------------------- #
+@torch.no_grad()
+def predict_pose(
+    model,
+    batch: Dict,
+    num_steps: int,
+    *,
+    amp_dtype: str = "bf16",
+    device=None,
+) -> Dict[str, torch.Tensor]:
+    """Run the model's own rectified-flow sampler on one collated ``batch`` and
+    return the decoded metric-space pose + shape latent.
+
+    Wraps ``inner.predict(batch, num_steps=...)`` EXACTLY as
+    ``validate.run_validation`` does: unwrap to ``inner``, ``inner.eval()``, the
+    same AMP autocast (bf16 on cuda by default; matches train/val ``amp_dtype``),
+    and ``no_grad``.  Outputs are cast to float32 to match the metric space the
+    validation metrics are computed in (``pred[...].float()``).
+
+    Args:
+      model: an ``MVSam3DFlow`` (or DDP-wrapped); unwrapped internally.
+      batch: an already-collated batch dict with at least ``views``, ``pm_shift``,
+             ``pm_scale`` (the data source is irrelevant — HO3D, DexYCB, ...).
+      num_steps: Euler ODE steps (same semantics as ``val_num_steps``).
+      amp_dtype: "bf16" | "fp16" | "float16" (autocast dtype on cuda).
+
+    Returns dict:
+      rotation     [B,3,3] float32  (decode_pose_rotation; orthonormal, det +1),
+      translation  [B,3]   float32  (metric, decode_pose_translation),
+      scale        [B]     float32  (metric, decode_pose_scale),
+      shape_latent [B,8,16,16,16] float32.
+    """
+    inner = model.module if hasattr(model, "module") else model
+    was_training = inner.training
+    inner.eval()
+    if device is None:
+        device = next(inner.parameters()).device
+    batch = _move(batch, device)
+
+    use_amp = (device.type == "cuda"
+               and amp_dtype.lower() in {"bf16", "fp16", "float16"})
+    amp_dt = torch.bfloat16 if amp_dtype.lower() == "bf16" else torch.float16
+    autocast = (torch.autocast(device_type="cuda", dtype=amp_dt)
+                if use_amp else torch.autocast(device_type="cpu", enabled=False))
+
+    try:
+        with autocast:
+            pred = inner.predict(batch, num_steps=num_steps)
+        out = {
+            "rotation": pred["rotation"].float(),          # [B,3,3]
+            "translation": pred["translation"].float(),    # [B,3] metric
+            "scale": pred["scale"].float(),                # [B]   metric
+            "shape_latent": pred["shape_latent"].float(),  # [B,8,16,16,16]
+        }
+    finally:
+        if was_training:
+            inner.train()
+    return out
+
+
+# --------------------------------------------------------------------------- #
+# CLI batch source (DexYCB) — used for the smoke / standalone runs only.
+# The pose entrypoint above does NOT depend on any of this.
+# --------------------------------------------------------------------------- #
+def _build_dexycb(data_cfg: dict, *, subjects, frame_stride, num_aux_views,
+                  split: str = "val"):
+    """Construct MultiViewDexYCB with deterministic views (random_views=False),
+    mirroring train.build_dataset for the val split.  ``subjects`` / ``frame_stride``
+    / ``num_aux_views`` may be overridden for a fast smoke."""
+    from mvsam3d.data.dexycb_dataset import MultiViewDexYCB
+    return MultiViewDexYCB(
+        dexycb_root=data_cfg["dexycb_root"],
+        canonical_cache=data_cfg["canonical_cache"],
+        num_aux_views=num_aux_views,
+        frame_stride=frame_stride,
+        anchor_sweep=data_cfg["anchor_sweep"],
+        subjects=subjects,
+        image_size=data_cfg["image_size"],
+        random_views=False,                       # deterministic views (like val)
+        min_aux_views=data_cfg["min_aux_views"],
+        max_aux_views=data_cfg["max_aux_views"],
+        depth_scale=data_cfg["depth_scale"],
+        depth_near=data_cfg["depth_near"],
+        depth_far=data_cfg["depth_far"],
+        split=split,
+        val_fraction=data_cfg["val_fraction"],
+        split_seed=data_cfg["split_seed"],
+        ssi_mode=data_cfg.get("ssi_mode", "scene"),
+    )
+
+
+def _orthonormality(R: torch.Tensor) -> Tuple[float, float]:
+    """(det, ||RᵀR - I||) for a [3,3] rotation, for a finite/orthonormal check."""
+    det = float(torch.det(R))
+    err = float((R.transpose(-1, -2) @ R - torch.eye(3, device=R.device, dtype=R.dtype)).abs().max())
+    return det, err
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Predict (R,t,s,shape) for DexYCB samples and write to disk.")
+    ap.add_argument("--config", default=DEFAULT_CONFIG, help="training YAML config")
+    ap.add_argument("--checkpoint", required=True, help="v2 checkpoint (cache/ckpt/step_*.pt)")
+    ap.add_argument("--out", required=True, help="output .pt path (a .json sibling is also written)")
+    ap.add_argument("--num-steps", type=int, default=None,
+                    help="Euler ODE steps (default: config train.val_num_steps)")
+    ap.add_argument("--num-samples", type=int, default=4, help="number of samples to predict")
+    ap.add_argument("--idx", type=int, nargs="*", default=None,
+                    help="explicit sample indices (overrides --num-samples)")
+    ap.add_argument("--subjects", type=str, nargs="*", default=None,
+                    help="subject substrings to filter (e.g. subject-01); default = config")
+    ap.add_argument("--frame-stride", type=int, default=None,
+                    help="override data.frame_stride (larger = fewer/faster samples)")
+    ap.add_argument("--num-aux-views", type=int, default=None,
+                    help="override fixed aux-view count (smaller = faster)")
+    ap.add_argument("--split", default="val", choices=["train", "val"])
+    ap.add_argument("--seed", type=int, default=1234, help="RNG seed for the sampler noise z0")
+    args = ap.parse_args()
+
+    cfg = load_config(args.config)
+    data_cfg = cfg["data"]
+    num_steps = args.num_steps if args.num_steps is not None else cfg["train"]["val_num_steps"]
+    amp_dtype = cfg["train"]["amp_dtype"]
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    # Deterministic sampler noise (validate.py seeds VAL_SEED before sampling).
+    torch.manual_seed(args.seed)
+    if device.type == "cuda":
+        torch.cuda.manual_seed_all(args.seed)
+
+    model, report = build_inference_model(cfg, args.checkpoint, device, verbose=True)
+
+    from mvsam3d.data.dexycb_dataset import collate_multiview
+    subjects = args.subjects if args.subjects else data_cfg["subjects"]
+    frame_stride = args.frame_stride if args.frame_stride is not None else data_cfg["frame_stride"]
+    num_aux = (args.num_aux_views if args.num_aux_views is not None
+               else (data_cfg["max_aux_views"] if data_cfg["random_views"]
+                     else data_cfg["num_aux_views"]))
+    ds = _build_dexycb(data_cfg, subjects=subjects, frame_stride=frame_stride,
+                       num_aux_views=num_aux, split=args.split)
+    print(f"[infer] dataset: {len(ds)} samples (subjects={subjects} "
+          f"frame_stride={frame_stride} num_aux_views={num_aux} split={args.split})")
+    if len(ds) == 0:
+        raise RuntimeError("dataset is empty for the given filters")
+
+    idxs: List[int] = (args.idx if args.idx is not None
+                       else list(range(min(args.num_samples, len(ds)))))
+
+    records = []
+    for i in idxs:
+        batch = collate_multiview([ds[i]])              # B=1
+        pred = predict_pose(model, batch, num_steps=num_steps, amp_dtype=amp_dtype,
+                            device=device)
+        R = pred["rotation"][0]
+        det, ortho_err = _orthonormality(R)
+        finite = bool(torch.isfinite(R).all() and torch.isfinite(pred["translation"]).all()
+                      and torch.isfinite(pred["scale"]).all())
+        rec = {
+            "idx": int(i),
+            "object_name": batch["object_name"][0],
+            "rotation": R.cpu(),
+            "translation": pred["translation"][0].cpu(),
+            "scale": float(pred["scale"][0]),
+            "shape_latent_shape": list(pred["shape_latent"].shape),
+            "det": det,
+            "ortho_err": ortho_err,
+            "finite": finite,
+        }
+        records.append(rec)
+        print(f"[infer] idx={i:>5d} obj={rec['object_name']:<22s} "
+              f"det={det:+.4f} ortho_err={ortho_err:.2e} "
+              f"t={rec['translation'].tolist()} s={rec['scale']:.4f} finite={finite}")
+
+    # ---- write .pt (full tensors) + .json (human-readable summary) ----
+    os.makedirs(os.path.dirname(os.path.abspath(args.out)) or ".", exist_ok=True)
+    torch.save({"load_report": report, "num_steps": num_steps,
+                "checkpoint": os.path.abspath(args.checkpoint),
+                "records": records}, args.out)
+    json_path = os.path.splitext(args.out)[0] + ".json"
+    with open(json_path, "w") as f:
+        json.dump({
+            "checkpoint": os.path.abspath(args.checkpoint),
+            "step": report["step"],
+            "num_steps": num_steps,
+            "load_report": {k: report[k] for k in
+                            ("step", "restored_tensors", "n_missing_keys",
+                             "n_unexpected_keys", "n_trainable_params",
+                             "missing_trainable_params")},
+            "predictions": [{
+                "idx": r["idx"], "object_name": r["object_name"],
+                "rotation": r["rotation"].tolist(),
+                "translation": r["translation"].tolist(),
+                "scale": r["scale"], "det": r["det"], "ortho_err": r["ortho_err"],
+                "finite": r["finite"], "shape_latent_shape": r["shape_latent_shape"],
+            } for r in records],
+        }, f, indent=2)
+    print(f"[infer] wrote {os.path.abspath(args.out)}")
+    print(f"[infer] wrote {os.path.abspath(json_path)}")
+
+
+if __name__ == "__main__":
+    main()