Les quatre vues passent sous multivue/ : l'arborescence du depot devient celle du disque (part 2)

multivue/LISEZ-MOI.md

@@ -0,0 +1,23 @@
+# Les poids du multivue
+
+Quatre fichiers obligatoires et un optionnel. Aucun n'est telecharge par le
+moteur : c'est l'application qui les pose, ou la personne qui installe.
+
+    structure_mv_fp8.safetensors   fusionne les quatre vues en un volume
+    forme_512_mv_fp8.safetensors   premiere descente de la forme
+    forme_1024_mv_fp8.safetensors  seconde descente, celle qui rend la surface
+    champ.safetensors              rattrape le detail entre deux jetons
+
+    cameras/                       mesure sous quel angle chaque photo a ete
+                                   prise, et en deduit si les deux profils
+                                   sont a l'endroit ou inverses.
+
+CE DERNIER A ETE ECRIT « OPTIONNEL » ICI, ET C'ETAIT FAUX (corrige le 8
+septembre 2026). Sans lui le moteur SUPPOSE la lateralite, et quand il suppose
+mal l'objet sort avec un visage sculpte a l'avant ET a l'arriere. Mesure sur
+quatre sujets : les quatre demandent l'inverse de la convention. Le moteur
+tourne encore sans ce dossier, mais il devine, et il le dit dans son resultat.
+A ne pas retirer pour gagner 517 Mo.
+
+Chaque .safetensors va avec son .json, qui decrit l'architecture : sans lui le
+modele ne peut pas etre construit.

multivue/cameras/source/src/depth_anything_3/model/gs_adapter.py

@@ -0,0 +1,200 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+import torch
+from einops import einsum, rearrange, repeat
+from torch import nn
+
+from depth_anything_3.model.utils.transform import cam_quat_xyzw_to_world_quat_wxyz
+from depth_anything_3.specs import Gaussians
+from depth_anything_3.utils.geometry import affine_inverse, get_world_rays, sample_image_grid
+from depth_anything_3.utils.pose_align import batch_align_poses_umeyama
+from depth_anything_3.utils.sh_helpers import rotate_sh
+
+
+class GaussianAdapter(nn.Module):
+
+    def __init__(
+        self,
+        sh_degree: int = 0,
+        pred_color: bool = False,
+        pred_offset_depth: bool = False,
+        pred_offset_xy: bool = True,
+        gaussian_scale_min: float = 1e-5,
+        gaussian_scale_max: float = 30.0,
+    ):
+        super().__init__()
+        self.sh_degree = sh_degree
+        self.pred_color = pred_color
+        self.pred_offset_depth = pred_offset_depth
+        self.pred_offset_xy = pred_offset_xy
+        self.gaussian_scale_min = gaussian_scale_min
+        self.gaussian_scale_max = gaussian_scale_max
+
+        # Create a mask for the spherical harmonics coefficients. This ensures that at
+        # initialization, the coefficients are biased towards having a large DC
+        # component and small view-dependent components.
+        if not pred_color:
+            self.register_buffer(
+                "sh_mask",
+                torch.ones((self.d_sh,), dtype=torch.float32),
+                persistent=False,
+            )
+            for degree in range(1, sh_degree + 1):
+                self.sh_mask[degree**2 : (degree + 1) ** 2] = 0.1 * 0.25**degree
+
+    def forward(
+        self,
+        extrinsics: torch.Tensor,  # "*#batch 4 4"
+        intrinsics: torch.Tensor,  # "*#batch 3 3"
+        depths: torch.Tensor,  # "*#batch"
+        opacities: torch.Tensor,  # "*#batch" | "*#batch _"
+        raw_gaussians: torch.Tensor,  # "*#batch _"
+        image_shape: tuple[int, int],
+        eps: float = 1e-8,
+        gt_extrinsics: Optional[torch.Tensor] = None,  # "*#batch 4 4"
+        **kwargs,
+    ) -> Gaussians:
+        device = extrinsics.device
+        dtype = raw_gaussians.dtype
+        H, W = image_shape
+        b, v = raw_gaussians.shape[:2]
+
+        # get cam2worlds and intr_normed to adapt to 3DGS codebase
+        cam2worlds = affine_inverse(extrinsics)
+        intr_normed = intrinsics.clone().detach()
+        intr_normed[..., 0, :] /= W
+        intr_normed[..., 1, :] /= H
+
+        # 1. compute 3DGS means
+        # 1.1) offset the predicted depth if needed
+        if self.pred_offset_depth:
+            gs_depths = depths + raw_gaussians[..., -1]
+            raw_gaussians = raw_gaussians[..., :-1]
+        else:
+            gs_depths = depths
+        # 1.2) align predicted poses with GT if needed
+        if gt_extrinsics is not None and not torch.equal(extrinsics, gt_extrinsics):
+            try:
+                _, _, pose_scales = batch_align_poses_umeyama(
+                    gt_extrinsics.detach().float(),
+                    extrinsics.detach().float(),
+                )
+            except Exception:
+                pose_scales = torch.ones_like(extrinsics[:, 0, 0, 0])
+            pose_scales = torch.clamp(pose_scales, min=1 / 3.0, max=3.0)
+            cam2worlds[:, :, :3, 3] = cam2worlds[:, :, :3, 3] * rearrange(
+                pose_scales, "b -> b () ()"
+            )  # [b, i, j]
+            gs_depths = gs_depths * rearrange(pose_scales, "b -> b () () ()")  # [b, v, h, w]
+        # 1.3) casting xy in image space
+        xy_ray, _ = sample_image_grid((H, W), device)
+        xy_ray = xy_ray[None, None, ...].expand(b, v, -1, -1, -1)  # b v h w xy
+        # offset xy if needed
+        if self.pred_offset_xy:
+            pixel_size = 1 / torch.tensor((W, H), dtype=xy_ray.dtype, device=device)
+            offset_xy = raw_gaussians[..., :2]
+            xy_ray = xy_ray + offset_xy * pixel_size
+            raw_gaussians = raw_gaussians[..., 2:]  # skip the offset_xy
+        # 1.4) unproject depth + xy to world ray
+        origins, directions = get_world_rays(
+            xy_ray,
+            repeat(cam2worlds, "b v i j -> b v h w i j", h=H, w=W),
+            repeat(intr_normed, "b v i j -> b v h w i j", h=H, w=W),
+        )
+        gs_means_world = origins + directions * gs_depths[..., None]
+        gs_means_world = rearrange(gs_means_world, "b v h w d -> b (v h w) d")
+
+        # 2. compute other GS attributes
+        scales, rotations, sh = raw_gaussians.split((3, 4, 3 * self.d_sh), dim=-1)
+
+        # 2.1) 3DGS scales
+        # make the scale invarient to resolution
+        scale_min = self.gaussian_scale_min
+        scale_max = self.gaussian_scale_max
+        scales = scale_min + (scale_max - scale_min) * scales.sigmoid()
+        pixel_size = 1 / torch.tensor((W, H), dtype=dtype, device=device)
+        multiplier = self.get_scale_multiplier(intr_normed, pixel_size)
+        gs_scales = scales * gs_depths[..., None] * multiplier[..., None, None, None]
+        gs_scales = rearrange(gs_scales, "b v h w d -> b (v h w) d")
+
+        # 2.2) 3DGS quaternion (world space)
+        # due to historical issue, assume quaternion in order xyzw, not wxyz
+        # Normalize the quaternion features to yield a valid quaternion.
+        rotations = rotations / (rotations.norm(dim=-1, keepdim=True) + eps)
+        # rotate them to world space
+        cam_quat_xyzw = rearrange(rotations, "b v h w c -> b (v h w) c")
+        c2w_mat = repeat(
+            cam2worlds,
+            "b v i j -> b (v h w) i j",
+            h=H,
+            w=W,
+        )
+        world_quat_wxyz = cam_quat_xyzw_to_world_quat_wxyz(cam_quat_xyzw, c2w_mat)
+        gs_rotations_world = world_quat_wxyz  # b (v h w) c
+
+        # 2.3) 3DGS color / SH coefficient (world space)
+        sh = rearrange(sh, "... (xyz d_sh) -> ... xyz d_sh", xyz=3)
+        if not self.pred_color:
+            sh = sh * self.sh_mask
+
+        if self.pred_color or self.sh_degree == 0:
+            # predict pre-computed color or predict only DC band, no need to transform
+            gs_sh_world = sh
+        else:
+            gs_sh_world = rotate_sh(sh, cam2worlds[:, :, None, None, None, :3, :3])
+        gs_sh_world = rearrange(gs_sh_world, "b v h w xyz d_sh -> b (v h w) xyz d_sh")
+
+        # 2.4) 3DGS opacity
+        gs_opacities = rearrange(opacities, "b v h w ... -> b (v h w) ...")
+
+        return Gaussians(
+            means=gs_means_world,
+            harmonics=gs_sh_world,
+            opacities=gs_opacities,
+            scales=gs_scales,
+            rotations=gs_rotations_world,
+        )
+
+    def get_scale_multiplier(
+        self,
+        intrinsics: torch.Tensor,  # "*#batch 3 3"
+        pixel_size: torch.Tensor,  # "*#batch 2"
+        multiplier: float = 0.1,
+    ) -> torch.Tensor:  # " *batch"
+        xy_multipliers = multiplier * einsum(
+            intrinsics[..., :2, :2].float().inverse().to(intrinsics),
+            pixel_size,
+            "... i j, j -> ... i",
+        )
+        return xy_multipliers.sum(dim=-1)
+
+    @property
+    def d_sh(self) -> int:
+        return 1 if self.pred_color else (self.sh_degree + 1) ** 2
+
+    @property
+    def d_in(self) -> int:
+        # provided as reference to the gs_dpt output dim
+        raw_gs_dim = 0
+        if self.pred_offset_xy:
+            raw_gs_dim += 2
+        raw_gs_dim += 3  # scales
+        raw_gs_dim += 4  # quaternion
+        raw_gs_dim += 3 * self.d_sh  # color
+        if self.pred_offset_depth:
+            raw_gs_dim += 1
+
+        return raw_gs_dim

multivue/cameras/source/src/depth_anything_3/model/gsdpt.py

@@ -0,0 +1,133 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict as TyDict
+from typing import List, Sequence
+import torch
+import torch.nn as nn
+
+from depth_anything_3.model.dpt import DPT
+from depth_anything_3.model.utils.head_utils import activate_head_gs, custom_interpolate
+
+
+class GSDPT(DPT):
+
+    def __init__(
+        self,
+        dim_in: int,
+        patch_size: int = 14,
+        output_dim: int = 4,
+        activation: str = "linear",
+        conf_activation: str = "sigmoid",
+        features: int = 256,
+        out_channels: Sequence[int] = (256, 512, 1024, 1024),
+        pos_embed: bool = True,
+        feature_only: bool = False,
+        down_ratio: int = 1,
+        conf_dim: int = 1,
+        norm_type: str = "idt",  # use to match legacy GS-DPT head, "idt" / "layer"
+        fusion_block_inplace: bool = False,
+    ) -> None:
+        super().__init__(
+            dim_in=dim_in,
+            patch_size=patch_size,
+            output_dim=output_dim,
+            activation=activation,
+            conf_activation=conf_activation,
+            features=features,
+            out_channels=out_channels,
+            pos_embed=pos_embed,
+            down_ratio=down_ratio,
+            head_name="raw_gs",
+            use_sky_head=False,
+            norm_type=norm_type,
+            fusion_block_inplace=fusion_block_inplace,
+        )
+        self.conf_dim = conf_dim
+        if conf_dim and conf_dim > 1:
+            assert (
+                conf_activation == "linear"
+            ), "use linear prediction when using view-dependent opacity"
+
+        merger_out_dim = features if feature_only else features // 2
+        self.images_merger = nn.Sequential(
+            nn.Conv2d(3, merger_out_dim // 4, 3, 1, 1),  # fewer channels first
+            nn.GELU(),
+            nn.Conv2d(merger_out_dim // 4, merger_out_dim // 2, 3, 1, 1),
+            nn.GELU(),
+            nn.Conv2d(merger_out_dim // 2, merger_out_dim, 3, 1, 1),
+            nn.GELU(),
+        )
+
+    # -------------------------------------------------------------------------
+    # Internal forward (single chunk)
+    # -------------------------------------------------------------------------
+    def _forward_impl(
+        self,
+        feats: List[torch.Tensor],
+        H: int,
+        W: int,
+        patch_start_idx: int,
+        images: torch.Tensor,
+    ) -> TyDict[str, torch.Tensor]:
+        B, _, C = feats[0].shape
+        ph, pw = H // self.patch_size, W // self.patch_size
+        resized_feats = []
+        for stage_idx, take_idx in enumerate(self.intermediate_layer_idx):
+            x = feats[take_idx][:, patch_start_idx:]  # [B*S, N_patch, C]
+            x = self.norm(x)
+            x = x.permute(0, 2, 1).reshape(B, C, ph, pw)  # [B*S, C, ph, pw]
+
+            x = self.projects[stage_idx](x)
+            if self.pos_embed:
+                x = self._add_pos_embed(x, W, H)
+            x = self.resize_layers[stage_idx](x)  # Align scale
+            resized_feats.append(x)
+
+        # 2) Fusion pyramid (main branch only)
+        fused = self._fuse(resized_feats)
+        fused = self.scratch.output_conv1(fused)
+
+        # 3) Upsample to target resolution, optionally add position encoding again
+        h_out = int(ph * self.patch_size / self.down_ratio)
+        w_out = int(pw * self.patch_size / self.down_ratio)
+
+        fused = custom_interpolate(fused, (h_out, w_out), mode="bilinear", align_corners=True)
+
+        # inject the image information here
+        fused = fused + self.images_merger(images)
+
+        if self.pos_embed:
+            fused = self._add_pos_embed(fused, W, H)
+
+        # 4) Shared neck1
+        # feat = self.scratch.output_conv1(fused)
+        feat = fused
+
+        # 5) Main head: logits -> activate_head or single channel activation
+        main_logits = self.scratch.output_conv2(feat)
+        outs: TyDict[str, torch.Tensor] = {}
+        if self.has_conf:
+            pred, conf = activate_head_gs(
+                main_logits,
+                activation=self.activation,
+                conf_activation=self.conf_activation,
+                conf_dim=self.conf_dim,
+            )
+            outs[self.head_main] = pred.squeeze(1)
+            outs[f"{self.head_main}_conf"] = conf.squeeze(1)
+        else:
+            outs[self.head_main] = self._apply_activation_single(main_logits).squeeze(1)
+
+        return outs

multivue/cameras/source/src/depth_anything_3/model/reference_view_selector.py

@@ -0,0 +1,223 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Reference View Selection Strategies
+
+This module provides different strategies for selecting a reference view
+from multiple input views in multi-view depth estimation.
+"""
+
+import torch
+from typing import Literal
+
+
+RefViewStrategy = Literal["first", "middle", "saddle_balanced", "saddle_sim_range"]
+
+
+def select_reference_view(
+    x: torch.Tensor,
+    strategy: RefViewStrategy = "saddle_balanced",
+) -> torch.Tensor:
+    """
+    Select a reference view from multiple views using the specified strategy.
+    
+    Args:
+        x: Input tensor of shape (B, S, N, C) where
+           B = batch size
+           S = number of views
+           N = number of tokens
+           C = channel dimension
+        strategy: Selection strategy, one of:
+            - "first": Always select the first view
+            - "middle": Select the middle view
+            - "saddle_balanced": Select view with balanced features across multiple metrics
+            - "saddle_sim_range": Select view with largest similarity range
+    
+    Returns:
+        b_idx: Tensor of shape (B,) containing the selected view index for each batch
+    """
+    B, S, N, C = x.shape
+    
+    # For single view, no reordering needed
+    if S <= 1:
+        return torch.zeros(B, dtype=torch.long, device=x.device)
+    
+    # Simple position-based strategies
+    if strategy == "first":
+        return torch.zeros(B, dtype=torch.long, device=x.device)
+    
+    elif strategy == "middle":
+        return torch.full((B,), S // 2, dtype=torch.long, device=x.device)
+    
+    # Feature-based strategies require normalized class tokens
+    # Extract and normalize class tokens (first token of each view)
+    img_class_feat = x[:, :, 0] / x[:, :, 0].norm(dim=-1, keepdim=True)  # B S C
+    
+    if strategy == "saddle_balanced":
+        # Select view with balanced features across multiple metrics
+        # Compute similarity matrix
+        sim = torch.matmul(img_class_feat, img_class_feat.transpose(1, 2))  # B S S
+        sim_no_diag = sim - torch.eye(S, device=sim.device).unsqueeze(0)
+        sim_score = sim_no_diag.sum(dim=-1) / (S - 1)  # B S
+        
+        feat_norm = x[:, :, 0].norm(dim=-1)  # B S
+        feat_var = img_class_feat.var(dim=-1)  # B S
+        
+        # Normalize all metrics to [0, 1]
+        def normalize_metric(metric):
+            min_val = metric.min(dim=1, keepdim=True).values
+            max_val = metric.max(dim=1, keepdim=True).values
+            return (metric - min_val) / (max_val - min_val + 1e-8)
+        
+        sim_score_norm = normalize_metric(sim_score)
+        norm_norm = normalize_metric(feat_norm)
+        var_norm = normalize_metric(feat_var)
+        
+        # Select view closest to the median (0.5) across all metrics
+        balance_score = (
+            (sim_score_norm - 0.5).abs() +
+            (norm_norm - 0.5).abs() +
+            (var_norm - 0.5).abs()
+        )
+        b_idx = balance_score.argmin(dim=1)
+        
+    elif strategy == "saddle_sim_range":
+        # Select view with largest similarity range (max - min)
+        sim = torch.matmul(img_class_feat, img_class_feat.transpose(1, 2))  # B S S
+        sim_no_diag = sim - torch.eye(S, device=sim.device).unsqueeze(0)
+        
+        sim_max = sim_no_diag.max(dim=-1).values  # B S
+        sim_min = sim_no_diag.min(dim=-1).values  # B S
+        sim_range = sim_max - sim_min
+        b_idx = sim_range.argmax(dim=1)
+    
+    else:
+        raise ValueError(
+            f"Unknown reference view selection strategy: {strategy}. "
+            f"Must be one of: 'first', 'middle', 'saddle_balanced', 'saddle_sim_range'"
+        )
+    
+    return b_idx
+
+
+def reorder_by_reference(
+    x: torch.Tensor,
+    b_idx: torch.Tensor,
+) -> torch.Tensor:
+    """
+    Reorder views to place the selected reference view first.
+    
+    Args:
+        x: Input tensor of shape (B, S, N, C)
+        b_idx: Reference view indices of shape (B,)
+    
+    Returns:
+        Reordered tensor with reference view at position 0
+    
+    Example:
+        If b_idx = [2] and S = 5 (views [0,1,2,3,4]),
+        result order is [2,0,1,3,4] (ref_idx first, then others in order)
+    """
+    B, S = x.shape[0], x.shape[1]
+    
+    # For single view, no reordering needed
+    if S <= 1:
+        return x
+    
+    # Create position indices: (B, S) where each row is [0, 1, 2, ..., S-1]
+    positions = torch.arange(S, device=x.device).unsqueeze(0).expand(B, -1)  # B S
+    
+    # For each position, determine which original index it should take
+    # Position 0 gets ref_idx
+    # Position 1 to ref_idx gets indices 0 to ref_idx-1
+    # Position ref_idx+1 to S-1 gets indices ref_idx+1 to S-1
+    
+    b_idx_expanded = b_idx.unsqueeze(1)  # B 1
+    
+    # Create the reordering indices
+    # For positions 1 to ref_idx: map to indices 0 to ref_idx-1 (shift by -1)
+    # For positions > ref_idx: keep the same
+    reorder_indices = positions.clone()
+    reorder_indices = torch.where(
+        (positions > 0) & (positions <= b_idx_expanded),
+        positions - 1,
+        positions
+    )
+    # Set position 0 to ref_idx
+    reorder_indices[:, 0] = b_idx
+    
+    # Gather using advanced indexing
+    batch_indices = torch.arange(B, device=x.device).unsqueeze(1)  # B 1
+    x_reordered = x[batch_indices, reorder_indices]
+    
+    return x_reordered
+
+
+def restore_original_order(
+    x: torch.Tensor,
+    b_idx: torch.Tensor,
+) -> torch.Tensor:
+    """
+    Restore original view order after processing.
+    
+    Args:
+        x: Reordered tensor of shape (B, S, ...)
+        b_idx: Original reference view indices of shape (B,)
+    
+    Returns:
+        Tensor with original view order restored
+    
+    Example:
+        If original order was [0, 1, 2, 3, 4] and b_idx=2,
+        reordered becomes [2, 0, 1, 3, 4] (reference at position 0),
+        restore should return [0, 1, 2, 3, 4] (original order).
+    """
+    B, S = x.shape[0], x.shape[1]
+    
+    # For single view, no restoration needed
+    if S <= 1:
+        return x
+    
+    # Create target position indices: (B, S) where each row is [0, 1, 2, ..., S-1]
+    target_positions = torch.arange(S, device=x.device).unsqueeze(0).expand(B, -1)  # B S
+    
+    # For each target position, determine which current position it comes from
+    # Target position 0 to ref_idx-1 <- Current position 1 to ref_idx (shift by +1)
+    # Target position ref_idx <- Current position 0
+    # Target position ref_idx+1 to S-1 <- Current position ref_idx+1 to S-1 (no change)
+    
+    b_idx_expanded = b_idx.unsqueeze(1)  # B 1
+    
+    # Create the restore indices
+    restore_indices = torch.where(
+        target_positions < b_idx_expanded,
+        target_positions + 1,  # Positions before ref_idx come from current position + 1
+        target_positions        # Positions after ref_idx stay the same
+    )
+    # Target position = ref_idx comes from current position 0
+    # Use scatter to set specific positions
+    restore_indices = torch.scatter(
+        restore_indices,
+        dim=1,
+        index=b_idx_expanded,
+        src=torch.zeros_like(b_idx_expanded)
+    )
+    
+    # Gather using advanced indexing
+    batch_indices = torch.arange(B, device=x.device).unsqueeze(1)  # B 1
+    x_restored = x[batch_indices, restore_indices]
+    
+    return x_restored
+

multivue/cameras/source/src/depth_anything_3/model/utils/attention.py

@@ -0,0 +1,109 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Modified from: https://github.com/huggingface/pytorch-image-models/blob/main/timm/models/vision_transformer.py#L103-L110 # noqa
+
+from typing import Callable, Optional, Union
+import torch
+import torch.nn.functional as F
+from torch import Tensor, nn
+
+
+class Attention(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int = 8,
+        qkv_bias: bool = True,
+        proj_bias: bool = True,
+        attn_drop: float = 0.0,
+        proj_drop: float = 0.0,
+        norm_layer: nn.Module = nn.LayerNorm,
+        qk_norm: bool = False,
+        rope=None,
+    ) -> None:
+        super().__init__()
+        assert dim % num_heads == 0, "dim should be divisible by num_heads"
+        self.num_heads = num_heads
+        self.head_dim = dim // num_heads
+        self.scale = self.head_dim**-0.5
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.q_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity()
+        self.k_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity()
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim, bias=proj_bias)
+        self.proj_drop = nn.Dropout(proj_drop)
+        self.rope = rope
+
+    def forward(self, x: Tensor, pos=None, attn_mask=None) -> Tensor:
+        # Debug breakpoint removed for production
+        B, N, C = x.shape
+        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
+        q, k, v = qkv.unbind(0)
+        q, k = self.q_norm(q), self.k_norm(k)
+        q = self.rope(q, pos) if self.rope is not None else q
+        k = self.rope(k, pos) if self.rope is not None else k
+        x = F.scaled_dot_product_attention(
+            q,
+            k,
+            v,
+            dropout_p=self.attn_drop.p if self.training else 0.0,
+            attn_mask=attn_mask,
+        )
+        x = x.transpose(1, 2).reshape(B, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+
+class LayerScale(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        init_values: Union[float, Tensor] = 1e-5,
+        inplace: bool = False,
+    ) -> None:
+        super().__init__()
+        self.inplace = inplace
+        self.gamma = nn.Parameter(init_values * torch.ones(dim))
+
+    def forward(self, x: Tensor) -> Tensor:
+        return x.mul_(self.gamma) if self.inplace else x * self.gamma
+
+
+class Mlp(nn.Module):
+    def __init__(
+        self,
+        in_features: int,
+        hidden_features: Optional[int] = None,
+        out_features: Optional[int] = None,
+        act_layer: Callable[..., nn.Module] = nn.GELU,
+        drop: float = 0.0,
+        bias: bool = True,
+    ) -> None:
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features, bias=bias)
+        self.act = act_layer()
+        self.fc2 = nn.Linear(hidden_features, out_features, bias=bias)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x: Tensor) -> Tensor:
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x

multivue/cameras/source/src/depth_anything_3/model/utils/block.py

@@ -0,0 +1,81 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import Callable
+from torch import Tensor, nn
+
+from .attention import Attention, LayerScale, Mlp
+
+
+class Block(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int,
+        mlp_ratio: float = 4.0,
+        qkv_bias: bool = True,
+        proj_bias: bool = True,
+        ffn_bias: bool = True,
+        drop: float = 0.0,
+        attn_drop: float = 0.0,
+        init_values=None,
+        drop_path: float = 0.0,
+        act_layer: Callable[..., nn.Module] = nn.GELU,
+        norm_layer: Callable[..., nn.Module] = nn.LayerNorm,
+        attn_class: Callable[..., nn.Module] = Attention,
+        ffn_layer: Callable[..., nn.Module] = Mlp,
+        qk_norm: bool = False,
+        rope=None,
+    ) -> None:
+        super().__init__()
+
+        self.norm1 = norm_layer(dim)
+
+        self.attn = attn_class(
+            dim,
+            num_heads=num_heads,
+            qkv_bias=qkv_bias,
+            proj_bias=proj_bias,
+            attn_drop=attn_drop,
+            proj_drop=drop,
+            qk_norm=qk_norm,
+            rope=rope,
+        )
+
+        self.ls1 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = ffn_layer(
+            in_features=dim,
+            hidden_features=mlp_hidden_dim,
+            act_layer=act_layer,
+            drop=drop,
+            bias=ffn_bias,
+        )
+        self.ls2 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
+
+        self.sample_drop_ratio = 0.0  # Equivalent to always having drop_path=0
+
+    def forward(self, x: Tensor, pos=None, attn_mask=None) -> Tensor:
+        def attn_residual_func(x: Tensor, pos=None, attn_mask=None) -> Tensor:
+            return self.ls1(self.attn(self.norm1(x), pos=pos, attn_mask=attn_mask))
+
+        def ffn_residual_func(x: Tensor) -> Tensor:
+            return self.ls2(self.mlp(self.norm2(x)))
+
+        # drop_path is always 0, so always take the else branch
+        x = x + attn_residual_func(x, pos=pos, attn_mask=attn_mask)
+        x = x + ffn_residual_func(x)
+        return x

multivue/cameras/source/src/depth_anything_3/model/utils/gs_renderer.py

@@ -0,0 +1,340 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from math import isqrt
+from typing import Literal, Optional
+import torch
+from einops import rearrange, repeat
+from tqdm import tqdm
+
+from depth_anything_3.specs import Gaussians
+from depth_anything_3.utils.camera_trj_helpers import (
+    interpolate_extrinsics,
+    interpolate_intrinsics,
+    render_dolly_zoom_path,
+    render_stabilization_path,
+    render_wander_path,
+    render_wobble_inter_path,
+)
+from depth_anything_3.utils.geometry import affine_inverse, as_homogeneous, get_fov
+from depth_anything_3.utils.logger import logger
+
+try:
+    from gsplat import rasterization
+except ImportError:
+    logger.warn(
+        "Dependency `gsplat` is required for rendering 3DGS. "
+        "Install via: pip install git+https://github.com/nerfstudio-project/"
+        "gsplat.git@0b4dddf04cb687367602c01196913cde6a743d70"
+    )
+
+
+def render_3dgs(
+    extrinsics: torch.Tensor,  # "batch_views 4 4", w2c
+    intrinsics: torch.Tensor,  # "batch_views 3 3", normalized
+    image_shape: tuple[int, int],
+    gaussian: Gaussians,
+    background_color: Optional[torch.Tensor] = None,  # "batch_views 3"
+    use_sh: bool = True,
+    num_view: int = 1,
+    color_mode: Literal["RGB+D", "RGB+ED"] = "RGB+D",
+    **kwargs,
+) -> tuple[
+    torch.Tensor,  # "batch_views 3 height width"
+    torch.Tensor,  # "batch_views height width"
+]:
+    # extract gaussian params
+    gaussian_means = gaussian.means
+    gaussian_scales = gaussian.scales
+    gaussian_quats = gaussian.rotations
+    gaussian_opacities = gaussian.opacities
+    gaussian_sh_coefficients = gaussian.harmonics
+    b, _, _ = extrinsics.shape
+
+    if background_color is None:
+        background_color = repeat(torch.tensor([0.0, 0.0, 0.0]), "c -> b c", b=b).to(
+            gaussian_sh_coefficients
+        )
+
+    if use_sh:
+        _, _, _, n = gaussian_sh_coefficients.shape
+        degree = isqrt(n) - 1
+        shs = rearrange(gaussian_sh_coefficients, "b g xyz n -> b g n xyz").contiguous()
+    else:  # use color
+        shs = (
+            gaussian_sh_coefficients.squeeze(-1).sigmoid().contiguous()
+        )  # (b, g, c), normed to (0, 1)
+
+    h, w = image_shape
+
+    fov_x, fov_y = get_fov(intrinsics).unbind(dim=-1)
+    tan_fov_x = (0.5 * fov_x).tan()
+    tan_fov_y = (0.5 * fov_y).tan()
+    focal_length_x = w / (2 * tan_fov_x)
+    focal_length_y = h / (2 * tan_fov_y)
+
+    view_matrix = extrinsics.float()
+
+    all_images = []
+    all_radii = []
+    all_depths = []
+    # render view in a batch based, each batch contains one scene
+    # assume the Gaussian parameters are originally repeated along the view dim
+    batch_scene = b // num_view
+
+    def index_i_gs_attr(full_attr, idx):
+        # return rearrange(full_attr, "(b v) ... -> b v ...", v=num_view)[idx, 0]
+        return full_attr[idx]
+
+    for i in range(batch_scene):
+        K = repeat(
+            torch.tensor(
+                [
+                    [0, 0, w / 2.0],
+                    [0, 0, h / 2.0],
+                    [0, 0, 1],
+                ]
+            ),
+            "i j -> v i j",
+            v=num_view,
+        ).to(gaussian_means)
+        K[:, 0, 0] = focal_length_x.reshape(batch_scene, num_view)[i]
+        K[:, 1, 1] = focal_length_y.reshape(batch_scene, num_view)[i]
+
+        i_means = index_i_gs_attr(gaussian_means, i)  # [N, 3]
+        i_scales = index_i_gs_attr(gaussian_scales, i)
+        i_quats = index_i_gs_attr(gaussian_quats, i)
+        i_opacities = index_i_gs_attr(gaussian_opacities, i)  # [N,]
+        i_colors = index_i_gs_attr(shs, i)  # [N, K, 3]
+        i_viewmats = rearrange(view_matrix, "(b v) ... -> b v ...", v=num_view)[i]  # [v, 4, 4]
+        i_backgrounds = rearrange(background_color, "(b v) ... -> b v ...", v=num_view)[
+            i
+        ]  # [v, 3]
+
+        render_colors, render_alphas, info = rasterization(
+            means=i_means,
+            quats=i_quats,  # [N, 4]
+            scales=i_scales,  # [N, 3]
+            opacities=i_opacities,
+            colors=i_colors,
+            viewmats=i_viewmats,  # [v, 4, 4]
+            Ks=K,  # [v, 3, 3]
+            backgrounds=i_backgrounds,
+            render_mode=color_mode,
+            width=w,
+            height=h,
+            packed=False,
+            sh_degree=degree if use_sh else None,
+        )
+        depth = render_colors[..., -1].unbind(dim=0)
+
+        image = rearrange(render_colors[..., :3], "v h w c -> v c h w").unbind(dim=0)
+        radii = info["radii"].unbind(dim=0)
+        try:
+            info["means2d"].retain_grad()  # [1, N, 2]
+        except Exception:
+            pass
+        all_images.extend(image)
+        all_depths.extend(depth)
+        all_radii.extend(radii)
+
+    return torch.stack(all_images), torch.stack(all_depths)
+
+
+def run_renderer_in_chunk_w_trj_mode(
+    gaussians: Gaussians,
+    extrinsics: torch.Tensor,  # world2cam, "batch view 4 4" | "batch view 3 4"
+    intrinsics: torch.Tensor,  # unnormed intrinsics, "batch view 3 3"
+    image_shape: tuple[int, int],
+    chunk_size: Optional[int] = 8,
+    trj_mode: Literal[
+        "original",
+        "smooth",
+        "interpolate",
+        "interpolate_smooth",
+        "wander",
+        "dolly_zoom",
+        "extend",
+        "wobble_inter",
+    ] = "smooth",
+    input_shape: Optional[tuple[int, int]] = None,
+    enable_tqdm: Optional[bool] = False,
+    **kwargs,
+) -> tuple[
+    torch.Tensor,  # color, "batch view 3 height width"
+    torch.Tensor,  # depth, "batch view height width"
+]:
+    cam2world = affine_inverse(as_homogeneous(extrinsics))
+    if input_shape is not None:
+        in_h, in_w = input_shape
+    else:
+        in_h, in_w = image_shape
+    intr_normed = intrinsics.clone().detach()
+    intr_normed[..., 0, :] /= in_w
+    intr_normed[..., 1, :] /= in_h
+    if extrinsics.shape[1] <= 1:
+        assert trj_mode in [
+            "wander",
+            "dolly_zoom",
+        ], "Please set trj_mode to 'wander' or 'dolly_zoom' when n_views=1"
+
+    def _smooth_trj_fn_batch(raw_c2ws, k_size=50):
+        try:
+            smooth_c2ws = torch.stack(
+                [render_stabilization_path(c2w_i, k_size) for c2w_i in raw_c2ws],
+                dim=0,
+            )
+        except Exception as e:
+            print(f"[DEBUG] Path smoothing failed with error: {e}.")
+            smooth_c2ws = raw_c2ws
+        return smooth_c2ws
+
+    # get rendered trj
+    if trj_mode == "original":
+        tgt_c2w = cam2world
+        tgt_intr = intr_normed
+    elif trj_mode == "smooth":
+        tgt_c2w = _smooth_trj_fn_batch(cam2world)
+        tgt_intr = intr_normed
+    elif trj_mode in ["interpolate", "interpolate_smooth", "extend"]:
+        inter_len = 8
+        total_len = (cam2world.shape[1] - 1) * inter_len
+        if total_len > 24 * 18:  # no more than 18s
+            inter_len = max(1, 24 * 10 // (cam2world.shape[1] - 1))
+        if total_len < 24 * 2:  # no less than 2s
+            inter_len = max(1, 24 * 2 // (cam2world.shape[1] - 1))
+
+        if inter_len > 2:
+            t = torch.linspace(0, 1, inter_len, dtype=torch.float32, device=cam2world.device)
+            t = (torch.cos(torch.pi * (t + 1)) + 1) / 2
+            tgt_c2w_b = []
+            tgt_intr_b = []
+            for b_idx in range(cam2world.shape[0]):
+                tgt_c2w = []
+                tgt_intr = []
+                for cur_idx in range(cam2world.shape[1] - 1):
+                    tgt_c2w.append(
+                        interpolate_extrinsics(
+                            cam2world[b_idx, cur_idx], cam2world[b_idx, cur_idx + 1], t
+                        )[(0 if cur_idx == 0 else 1) :]
+                    )
+                    tgt_intr.append(
+                        interpolate_intrinsics(
+                            intr_normed[b_idx, cur_idx], intr_normed[b_idx, cur_idx + 1], t
+                        )[(0 if cur_idx == 0 else 1) :]
+                    )
+                tgt_c2w_b.append(torch.cat(tgt_c2w))
+                tgt_intr_b.append(torch.cat(tgt_intr))
+            tgt_c2w = torch.stack(tgt_c2w_b)  # b v 4 4
+            tgt_intr = torch.stack(tgt_intr_b)  # b v 3 3
+        else:
+            tgt_c2w = cam2world
+            tgt_intr = intr_normed
+        if trj_mode in ["interpolate_smooth", "extend"]:
+            tgt_c2w = _smooth_trj_fn_batch(tgt_c2w)
+        if trj_mode == "extend":
+            # apply dolly_zoom and wander in the middle frame
+            assert cam2world.shape[0] == 1, "extend only supports for batch_size=1 currently."
+            mid_idx = tgt_c2w.shape[1] // 2
+            c2w_wd, intr_wd = render_wander_path(
+                tgt_c2w[0, mid_idx],
+                tgt_intr[0, mid_idx],
+                h=in_h,
+                w=in_w,
+                num_frames=max(36, min(60, mid_idx // 2)),
+                max_disp=24.0,
+            )
+            c2w_dz, intr_dz = render_dolly_zoom_path(
+                tgt_c2w[0, mid_idx],
+                tgt_intr[0, mid_idx],
+                h=in_h,
+                w=in_w,
+                num_frames=max(36, min(60, mid_idx // 2)),
+            )
+            tgt_c2w = torch.cat(
+                [
+                    tgt_c2w[:, :mid_idx],
+                    c2w_wd.unsqueeze(0),
+                    c2w_dz.unsqueeze(0),
+                    tgt_c2w[:, mid_idx:],
+                ],
+                dim=1,
+            )
+            tgt_intr = torch.cat(
+                [
+                    tgt_intr[:, :mid_idx],
+                    intr_wd.unsqueeze(0),
+                    intr_dz.unsqueeze(0),
+                    tgt_intr[:, mid_idx:],
+                ],
+                dim=1,
+            )
+    elif trj_mode in ["wander", "dolly_zoom"]:
+        if trj_mode == "wander":
+            render_fn = render_wander_path
+            extra_kwargs = {"max_disp": 24.0}
+        else:
+            render_fn = render_dolly_zoom_path
+            extra_kwargs = {"D_focus": 30.0, "max_disp": 2.0}
+        tgt_c2w = []
+        tgt_intr = []
+        for b_idx in range(cam2world.shape[0]):
+            c2w_i, intr_i = render_fn(
+                cam2world[b_idx, 0], intr_normed[b_idx, 0], h=in_h, w=in_w, **extra_kwargs
+            )
+            tgt_c2w.append(c2w_i)
+            tgt_intr.append(intr_i)
+        tgt_c2w = torch.stack(tgt_c2w)
+        tgt_intr = torch.stack(tgt_intr)
+    elif trj_mode == "wobble_inter":
+        tgt_c2w, tgt_intr = render_wobble_inter_path(
+            cam2world=cam2world,
+            intr_normed=intr_normed,
+            inter_len=10,
+            n_skip=3,
+        )
+    else:
+        raise Exception(f"trj mode [{trj_mode}] is not implemented.")
+
+    _, v = tgt_c2w.shape[:2]
+    tgt_extr = affine_inverse(tgt_c2w)
+    if chunk_size is None:
+        chunk_size = v
+    chunk_size = min(v, chunk_size)
+    all_colors = []
+    all_depths = []
+    for chunk_idx in tqdm(
+        range(math.ceil(v / chunk_size)),
+        desc="Rendering novel views",
+        disable=(not enable_tqdm),
+        leave=False,
+    ):
+        s = int(chunk_idx * chunk_size)
+        e = int((chunk_idx + 1) * chunk_size)
+        cur_n_view = tgt_extr[:, s:e].shape[1]
+        color, depth = render_3dgs(
+            extrinsics=rearrange(tgt_extr[:, s:e], "b v ... -> (b v) ..."),  # w2c
+            intrinsics=rearrange(tgt_intr[:, s:e], "b v ... -> (b v) ..."),  # normed
+            image_shape=image_shape,
+            gaussian=gaussians,
+            num_view=cur_n_view,
+            **kwargs,
+        )
+        all_colors.append(rearrange(color, "(b v) ... -> b v ...", v=cur_n_view))
+        all_depths.append(rearrange(depth, "(b v) ... -> b v ...", v=cur_n_view))
+    all_colors = torch.cat(all_colors, dim=1)
+    all_depths = torch.cat(all_depths, dim=1)
+
+    return all_colors, all_depths

multivue/cameras/source/src/depth_anything_3/model/utils/head_utils.py

@@ -0,0 +1,230 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Tuple, Union
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+# -----------------------------------------------------------------------------
+# Activation functions
+# -----------------------------------------------------------------------------
+
+
+def activate_head_gs(out, activation="norm_exp", conf_activation="expp1", conf_dim=None):
+    """
+    Process network output to extract GS params and density values.
+    Density could be view-dependent as SH coefficient
+
+
+    Args:
+        out: Network output tensor (B, C, H, W)
+        activation: Activation type for 3D points
+        conf_activation: Activation type for confidence values
+
+    Returns:
+        Tuple of (3D points tensor, confidence tensor)
+    """
+    # Move channels from last dim to the 4th dimension => (B, H, W, C)
+    fmap = out.permute(0, 2, 3, 1)  # B,H,W,C expected
+
+    # Split into xyz (first C-1 channels) and confidence (last channel)
+    conf_dim = 1 if conf_dim is None else conf_dim
+    xyz = fmap[:, :, :, :-conf_dim]
+    conf = fmap[:, :, :, -1] if conf_dim == 1 else fmap[:, :, :, -conf_dim:]
+
+    if activation == "norm_exp":
+        d = xyz.norm(dim=-1, keepdim=True).clamp(min=1e-8)
+        xyz_normed = xyz / d
+        pts3d = xyz_normed * torch.expm1(d)
+    elif activation == "norm":
+        pts3d = xyz / xyz.norm(dim=-1, keepdim=True)
+    elif activation == "exp":
+        pts3d = torch.exp(xyz)
+    elif activation == "relu":
+        pts3d = F.relu(xyz)
+    elif activation == "sigmoid":
+        pts3d = torch.sigmoid(xyz)
+    elif activation == "linear":
+        pts3d = xyz
+    else:
+        raise ValueError(f"Unknown activation: {activation}")
+
+    if conf_activation == "expp1":
+        conf_out = 1 + conf.exp()
+    elif conf_activation == "expp0":
+        conf_out = conf.exp()
+    elif conf_activation == "sigmoid":
+        conf_out = torch.sigmoid(conf)
+    elif conf_activation == "linear":
+        conf_out = conf
+    else:
+        raise ValueError(f"Unknown conf_activation: {conf_activation}")
+
+    return pts3d, conf_out
+
+
+# -----------------------------------------------------------------------------
+# Other utilities
+# -----------------------------------------------------------------------------
+
+
+class Permute(nn.Module):
+    """nn.Module wrapper around Tensor.permute for cleaner nn.Sequential usage."""
+
+    dims: Tuple[int, ...]
+
+    def __init__(self, dims: Tuple[int, ...]) -> None:
+        super().__init__()
+        self.dims = dims
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:  # type: ignore[override]
+        return x.permute(*self.dims)
+
+
+def position_grid_to_embed(
+    pos_grid: torch.Tensor, embed_dim: int, omega_0: float = 100
+) -> torch.Tensor:
+    """
+    Convert 2D position grid (HxWx2) to sinusoidal embeddings (HxWxC)
+
+    Args:
+        pos_grid: Tensor of shape (H, W, 2) containing 2D coordinates
+        embed_dim: Output channel dimension for embeddings
+
+    Returns:
+        Tensor of shape (H, W, embed_dim) with positional embeddings
+    """
+    H, W, grid_dim = pos_grid.shape
+    assert grid_dim == 2
+    pos_flat = pos_grid.reshape(-1, grid_dim)  # Flatten to (H*W, 2)
+
+    # Process x and y coordinates separately
+    emb_x = make_sincos_pos_embed(embed_dim // 2, pos_flat[:, 0], omega_0=omega_0)  # [1, H*W, D/2]
+    emb_y = make_sincos_pos_embed(embed_dim // 2, pos_flat[:, 1], omega_0=omega_0)  # [1, H*W, D/2]
+
+    # Combine and reshape
+    emb = torch.cat([emb_x, emb_y], dim=-1)  # [1, H*W, D]
+
+    return emb.view(H, W, embed_dim)  # [H, W, D]
+
+
+def make_sincos_pos_embed(embed_dim: int, pos: torch.Tensor, omega_0: float = 100) -> torch.Tensor:
+    """
+    This function generates a 1D positional embedding from a given grid using sine and cosine functions. # noqa
+
+    Args:
+    - embed_dim: The embedding dimension.
+    - pos: The position to generate the embedding from.
+
+    Returns:
+    - emb: The generated 1D positional embedding.
+    """
+    assert embed_dim % 2 == 0
+    omega = torch.arange(embed_dim // 2, dtype=torch.float32, device=pos.device)
+    omega /= embed_dim / 2.0
+    omega = 1.0 / omega_0**omega  # (D/2,)
+
+    pos = pos.reshape(-1)  # (M,)
+    out = torch.einsum("m,d->md", pos, omega)  # (M, D/2), outer product
+
+    emb_sin = torch.sin(out)  # (M, D/2)
+    emb_cos = torch.cos(out)  # (M, D/2)
+
+    emb = torch.cat([emb_sin, emb_cos], dim=1)  # (M, D)
+    return emb.float()
+
+
+# Inspired by https://github.com/microsoft/moge
+
+
+def create_uv_grid(
+    width: int,
+    height: int,
+    aspect_ratio: float = None,
+    dtype: torch.dtype = None,
+    device: torch.device = None,
+) -> torch.Tensor:
+    """
+    Create a normalized UV grid of shape (width, height, 2).
+
+    The grid spans horizontally and vertically according to an aspect ratio,
+    ensuring the top-left corner is at (-x_span, -y_span) and the bottom-right
+    corner is at (x_span, y_span), normalized by the diagonal of the plane.
+
+    Args:
+        width (int): Number of points horizontally.
+        height (int): Number of points vertically.
+        aspect_ratio (float, optional): Width-to-height ratio. Defaults to width/height.
+        dtype (torch.dtype, optional): Data type of the resulting tensor.
+        device (torch.device, optional): Device on which the tensor is created.
+
+    Returns:
+        torch.Tensor: A (width, height, 2) tensor of UV coordinates.
+    """
+    # Derive aspect ratio if not explicitly provided
+    if aspect_ratio is None:
+        aspect_ratio = float(width) / float(height)
+
+    # Compute normalized spans for X and Y
+    diag_factor = (aspect_ratio**2 + 1.0) ** 0.5
+    span_x = aspect_ratio / diag_factor
+    span_y = 1.0 / diag_factor
+
+    # Establish the linspace boundaries
+    left_x = -span_x * (width - 1) / width
+    right_x = span_x * (width - 1) / width
+    top_y = -span_y * (height - 1) / height
+    bottom_y = span_y * (height - 1) / height
+
+    # Generate 1D coordinates
+    x_coords = torch.linspace(left_x, right_x, steps=width, dtype=dtype, device=device)
+    y_coords = torch.linspace(top_y, bottom_y, steps=height, dtype=dtype, device=device)
+
+    # Create 2D meshgrid (width x height) and stack into UV
+    uu, vv = torch.meshgrid(x_coords, y_coords, indexing="xy")
+    uv_grid = torch.stack((uu, vv), dim=-1)
+
+    return uv_grid
+
+
+# -----------------------------------------------------------------------------
+# Interpolation (safe interpolation, avoid INT_MAX overflow)
+# -----------------------------------------------------------------------------
+def custom_interpolate(
+    x: torch.Tensor,
+    size: Union[Tuple[int, int], None] = None,
+    scale_factor: Union[float, None] = None,
+    mode: str = "bilinear",
+    align_corners: bool = True,
+) -> torch.Tensor:
+    """
+    Safe interpolation implementation to avoid INT_MAX overflow in torch.nn.functional.interpolate.
+    """
+    if size is None:
+        assert scale_factor is not None, "Either size or scale_factor must be provided."
+        size = (int(x.shape[-2] * scale_factor), int(x.shape[-1] * scale_factor))
+
+    INT_MAX = 1610612736
+    total = size[0] * size[1] * x.shape[0] * x.shape[1]
+
+    if total > INT_MAX:
+        chunks = torch.chunk(x, chunks=(total // INT_MAX) + 1, dim=0)
+        outs = [
+            nn.functional.interpolate(c, size=size, mode=mode, align_corners=align_corners)
+            for c in chunks
+        ]
+        return torch.cat(outs, dim=0).contiguous()
+
+    return nn.functional.interpolate(x, size=size, mode=mode, align_corners=align_corners)

multivue/cameras/source/src/depth_anything_3/model/utils/transform.py

@@ -0,0 +1,208 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+import torch.nn.functional as F
+
+
+def extri_intri_to_pose_encoding(
+    extrinsics,
+    intrinsics,
+    image_size_hw=None,
+):
+    """Convert camera extrinsics and intrinsics to a compact pose encoding."""
+
+    # extrinsics: BxSx3x4
+    # intrinsics: BxSx3x3
+    R = extrinsics[:, :, :3, :3]  # BxSx3x3
+    T = extrinsics[:, :, :3, 3]  # BxSx3
+
+    quat = mat_to_quat(R)
+    # Note the order of h and w here
+    H, W = image_size_hw
+    fov_h = 2 * torch.atan((H / 2) / intrinsics[..., 1, 1])
+    fov_w = 2 * torch.atan((W / 2) / intrinsics[..., 0, 0])
+    pose_encoding = torch.cat([T, quat, fov_h[..., None], fov_w[..., None]], dim=-1).float()
+
+    return pose_encoding
+
+
+def pose_encoding_to_extri_intri(
+    pose_encoding,
+    image_size_hw=None,
+):
+    """Convert a pose encoding back to camera extrinsics and intrinsics."""
+
+    T = pose_encoding[..., :3]
+    quat = pose_encoding[..., 3:7]
+    fov_h = pose_encoding[..., 7]
+    fov_w = pose_encoding[..., 8]
+
+    R = quat_to_mat(quat)
+    extrinsics = torch.cat([R, T[..., None]], dim=-1)
+
+    H, W = image_size_hw
+    fy = (H / 2.0) / torch.clamp(torch.tan(fov_h / 2.0), 1e-6)
+    fx = (W / 2.0) / torch.clamp(torch.tan(fov_w / 2.0), 1e-6)
+    intrinsics = torch.zeros(pose_encoding.shape[:2] + (3, 3), device=pose_encoding.device)
+    intrinsics[..., 0, 0] = fx
+    intrinsics[..., 1, 1] = fy
+    intrinsics[..., 0, 2] = W / 2
+    intrinsics[..., 1, 2] = H / 2
+    intrinsics[..., 2, 2] = 1.0  # Set the homogeneous coordinate to 1
+
+    return extrinsics, intrinsics
+
+
+def quat_to_mat(quaternions: torch.Tensor) -> torch.Tensor:
+    """
+    Quaternion Order: XYZW or say ijkr, scalar-last
+
+    Convert rotations given as quaternions to rotation matrices.
+    Args:
+        quaternions: quaternions with real part last,
+            as tensor of shape (..., 4).
+
+    Returns:
+        Rotation matrices as tensor of shape (..., 3, 3).
+    """
+    i, j, k, r = torch.unbind(quaternions, -1)
+    two_s = 2.0 / (quaternions * quaternions).sum(-1)
+
+    o = torch.stack(
+        (
+            1 - two_s * (j * j + k * k),
+            two_s * (i * j - k * r),
+            two_s * (i * k + j * r),
+            two_s * (i * j + k * r),
+            1 - two_s * (i * i + k * k),
+            two_s * (j * k - i * r),
+            two_s * (i * k - j * r),
+            two_s * (j * k + i * r),
+            1 - two_s * (i * i + j * j),
+        ),
+        -1,
+    )
+    return o.reshape(quaternions.shape[:-1] + (3, 3))
+
+
+def mat_to_quat(matrix: torch.Tensor) -> torch.Tensor:
+    """
+    Convert rotations given as rotation matrices to quaternions.
+
+    Args:
+        matrix: Rotation matrices as tensor of shape (..., 3, 3).
+
+    Returns:
+        quaternions with real part last, as tensor of shape (..., 4).
+        Quaternion Order: XYZW or say ijkr, scalar-last
+    """
+    if matrix.size(-1) != 3 or matrix.size(-2) != 3:
+        raise ValueError(f"Invalid rotation matrix shape {matrix.shape}.")
+
+    batch_dim = matrix.shape[:-2]
+    m00, m01, m02, m10, m11, m12, m20, m21, m22 = torch.unbind(
+        matrix.reshape(batch_dim + (9,)), dim=-1
+    )
+
+    q_abs = _sqrt_positive_part(
+        torch.stack(
+            [
+                1.0 + m00 + m11 + m22,
+                1.0 + m00 - m11 - m22,
+                1.0 - m00 + m11 - m22,
+                1.0 - m00 - m11 + m22,
+            ],
+            dim=-1,
+        )
+    )
+
+    quat_by_rijk = torch.stack(
+        [
+            torch.stack([q_abs[..., 0] ** 2, m21 - m12, m02 - m20, m10 - m01], dim=-1),
+            torch.stack([m21 - m12, q_abs[..., 1] ** 2, m10 + m01, m02 + m20], dim=-1),
+            torch.stack([m02 - m20, m10 + m01, q_abs[..., 2] ** 2, m12 + m21], dim=-1),
+            torch.stack([m10 - m01, m20 + m02, m21 + m12, q_abs[..., 3] ** 2], dim=-1),
+        ],
+        dim=-2,
+    )
+
+    flr = torch.tensor(0.1).to(dtype=q_abs.dtype, device=q_abs.device)
+    quat_candidates = quat_by_rijk / (2.0 * q_abs[..., None].max(flr))
+
+    out = quat_candidates[F.one_hot(q_abs.argmax(dim=-1), num_classes=4) > 0.5, :].reshape(
+        batch_dim + (4,)
+    )
+
+    out = out[..., [1, 2, 3, 0]]
+
+    out = standardize_quaternion(out)
+
+    return out
+
+
+def _sqrt_positive_part(x: torch.Tensor) -> torch.Tensor:
+    """
+    Returns torch.sqrt(torch.max(0, x))
+    but with a zero subgradient where x is 0.
+    """
+    ret = torch.zeros_like(x)
+    positive_mask = x > 0
+    if torch.is_grad_enabled():
+        ret[positive_mask] = torch.sqrt(x[positive_mask])
+    else:
+        ret = torch.where(positive_mask, torch.sqrt(x), ret)
+    return ret
+
+
+def standardize_quaternion(quaternions: torch.Tensor) -> torch.Tensor:
+    """
+    Convert a unit quaternion to a standard form: one in which the real
+    part is non negative.
+
+    Args:
+        quaternions: Quaternions with real part last,
+            as tensor of shape (..., 4).
+
+    Returns:
+        Standardized quaternions as tensor of shape (..., 4).
+    """
+    return torch.where(quaternions[..., 3:4] < 0, -quaternions, quaternions)
+
+
+def cam_quat_xyzw_to_world_quat_wxyz(cam_quat_xyzw, c2w):
+    # cam_quat_xyzw: (b, n, 4) in xyzw
+    # c2w: (b, n, 4, 4)
+    b, n = cam_quat_xyzw.shape[:2]
+    # 1. xyzw -> wxyz
+    cam_quat_wxyz = torch.cat(
+        [
+            cam_quat_xyzw[..., 3:4],  # w
+            cam_quat_xyzw[..., 0:1],  # x
+            cam_quat_xyzw[..., 1:2],  # y
+            cam_quat_xyzw[..., 2:3],  # z
+        ],
+        dim=-1,
+    )
+    # 2. Quaternion to matrix
+    cam_quat_wxyz_flat = cam_quat_wxyz.reshape(-1, 4)
+    rotmat_cam = quat_to_mat(cam_quat_wxyz_flat).reshape(b, n, 3, 3)
+    # 3. Transform to world space
+    rotmat_c2w = c2w[..., :3, :3]
+    rotmat_world = torch.matmul(rotmat_c2w, rotmat_cam)
+    # 4. Matrix to quaternion (wxyz)
+    rotmat_world_flat = rotmat_world.reshape(-1, 3, 3)
+    world_quat_wxyz_flat = mat_to_quat(rotmat_world_flat)
+    world_quat_wxyz = world_quat_wxyz_flat.reshape(b, n, 4)
+    return world_quat_wxyz

multivue/cameras/source/src/depth_anything_3/registry.py

@@ -0,0 +1,50 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import OrderedDict
+from pathlib import Path
+
+
+def get_all_models() -> OrderedDict:
+    """
+    Scans all YAML files in the configs directory and returns a sorted dictionary where:
+    - Keys are model names (YAML filenames without the .yaml extension)
+    - Values are absolute paths to the corresponding YAML files
+    """
+    # Get path to the configs directory within the da3 package
+    # Works both in development and after pip installation
+    # configs_dir = files("depth_anything_3").joinpath("configs")
+    configs_dir = Path(__file__).resolve().parent / "configs"
+
+    # Ensure path is a Path object for consistent cross-platform handling
+    configs_dir = Path(configs_dir)
+
+    model_entries = []
+    # Iterate through all items in the configs directory
+    for item in configs_dir.iterdir():
+        # Filter for YAML files (excluding directories)
+        if item.is_file() and item.suffix == ".yaml":
+            # Extract model name (filename without .yaml extension)
+            model_name = item.stem
+            # Get absolute path (resolve() handles symlinks)
+            file_abs_path = str(item.resolve())
+            model_entries.append((model_name, file_abs_path))
+
+    # Sort entries by model name and convert to OrderedDict
+    sorted_entries = sorted(model_entries, key=lambda x: x[0])
+    return OrderedDict(sorted_entries)
+
+
+# Global registry for external imports
+MODEL_REGISTRY = get_all_models()

multivue/cameras/source/src/depth_anything_3/services/__init__.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Services module for Depth Anything 3.
+"""
+
+from depth_anything_3.services.backend import create_app, start_server
+
+__all__ = [
+    start_server,
+    create_app,
+]

multivue/cameras/source/src/depth_anything_3/services/backend.py

@@ -0,0 +1,1556 @@
+# flake8: noqa: E501
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Model backend service for Depth Anything 3.
+Provides HTTP API for model inference with persistent model loading.
+"""
+
+import hmac
+import html
+import os
+import posixpath
+import re
+import secrets
+import time
+import uuid
+
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, Dict, List, Optional
+from urllib.parse import quote
+import numpy as np
+
+import uvicorn
+from fastapi import Depends, FastAPI, Header, HTTPException
+from fastapi.responses import FileResponse, HTMLResponse
+from pydantic import BaseModel, field_validator
+
+from ..api import DepthAnything3
+from ..utils.export import SUPPORTED_EXPORT_FORMATS
+from ..utils.memory import (
+    get_gpu_memory_info,
+    cleanup_cuda_memory,
+    check_memory_availability,
+    estimate_memory_requirement,
+)
+
+
+class InferenceRequest(BaseModel):
+    """Request model for inference API."""
+
+    image_paths: List[str]
+    export_dir: Optional[str] = None
+    export_format: str = "mini_npz-glb"
+    extrinsics: Optional[List[List[List[float]]]] = None
+    intrinsics: Optional[List[List[List[float]]]] = None
+    process_res: int = 504
+    process_res_method: str = "upper_bound_resize"
+    export_feat_layers: List[int] = []
+    align_to_input_ext_scale: bool = True
+    # GLB export parameters
+    conf_thresh_percentile: float = 40.0
+    num_max_points: int = 1_000_000
+    show_cameras: bool = True
+    # Feat_vis export parameters
+    feat_vis_fps: int = 15
+
+    @field_validator("export_format")
+    @classmethod
+    def _validate_export_format(cls, value: str) -> str:
+        """Restrict export_format to the server's known/trusted export dispatcher formats,
+        instead of letting an arbitrary client-supplied string reach the export() dispatcher."""
+        tokens = value.split("-") if value else [value]
+        unknown = sorted(set(tokens) - SUPPORTED_EXPORT_FORMATS)
+        if unknown:
+            raise ValueError(
+                f"Unsupported export_format value(s): {unknown}. "
+                f"Supported formats: {sorted(SUPPORTED_EXPORT_FORMATS)}"
+            )
+        return value
+
+
+class InferenceResponse(BaseModel):
+    """Response model for inference API."""
+
+    success: bool
+    message: str
+    task_id: Optional[str] = None
+    export_dir: Optional[str] = None
+    export_format: str = "mini_npz-glb"
+    processing_time: Optional[float] = None
+
+
+class TaskStatus(BaseModel):
+    """Task status model."""
+
+    task_id: str
+    status: str  # "pending", "running", "completed", "failed"
+    message: str
+    progress: Optional[float] = None  # 0.0 to 1.0
+    created_at: float
+    started_at: Optional[float] = None
+    completed_at: Optional[float] = None
+    export_dir: Optional[str] = None
+    request: Optional[InferenceRequest] = None  # Store the original request
+
+    # Essential task parameters
+    num_images: Optional[int] = None  # Number of input images
+    export_format: Optional[str] = None  # Export format
+    process_res_method: Optional[str] = None  # Processing resolution method
+    video_path: Optional[str] = None  # Source video path
+
+
+def _render_task_card(task: "TaskStatus", css_class: str) -> str:
+    """Render a single task as an HTML fragment for the dashboard.
+
+    Every field is HTML-escaped: task fields originate from a client request and
+    are rendered on a page other users may view.
+    """
+    e = html.escape
+
+    video_row = ""
+    if task.video_path:
+        video_row = f"<br><small>Video: {e(task.video_path)}</small>"
+
+    return f"""
+                <div class="task-item {e(css_class)}">
+                    <div class="task-header">
+                        <span class="task-id">{e(task.task_id)}</span>
+                        <span class="task-status status-{e(task.status)}">{e(task.status)}</span>
+                    </div>
+                    <div class="task-message">{e(task.message)}</div>
+                    <div class="task-params">
+                        <small>
+                            Images: {e(str(task.num_images)) if task.num_images is not None else 'N/A'} |
+                            Format: {e(task.export_format) if task.export_format else 'N/A'} |
+                            Method: {e(task.process_res_method) if task.process_res_method else 'N/A'} |
+                            Export Dir: {e(task.export_dir) if task.export_dir else 'N/A'}
+                        </small>
+                        {video_row}
+                    </div>
+                </div>
+                """
+
+
+class ModelBackend:
+    """Model backend service with persistent model loading."""
+
+    def __init__(self, model_dir: str, device: str = "cuda"):
+        self.model_dir = model_dir
+        self.device = device
+        self.model = None
+        self.model_loaded = False
+        self.load_time = None
+        self.load_start_time = None  # Time when model loading started
+        self.load_completed_time = None  # Time when model loading completed
+        self.last_used = None
+
+    def load_model(self):
+        """Load model if not already loaded."""
+        if self.model_loaded and self.model is not None:
+            self.last_used = time.time()
+            return self.model
+
+        try:
+            print(f"Loading model from {self.model_dir}...")
+            self.load_start_time = time.time()
+            start_time = time.time()
+
+            self.model = DepthAnything3.from_pretrained(self.model_dir).to(self.device)
+            self.model.eval()
+
+            self.model_loaded = True
+            self.load_time = time.time() - start_time
+            self.load_completed_time = time.time()
+            self.last_used = time.time()
+
+            print(f"Model loaded successfully in {self.load_time:.2f}s")
+            return self.model
+
+        except Exception as e:
+            print(f"Failed to load model: {e}")
+            raise e
+
+    def get_model(self):
+        """Get model, loading if necessary."""
+        if not self.model_loaded:
+            return self.load_model()
+        self.last_used = time.time()
+        return self.model
+
+    def get_status(self) -> Dict[str, Any]:
+        """Get backend status information."""
+        # Calculate uptime from when model loading completed
+        uptime = 0
+        if self.model_loaded and self.load_completed_time:
+            uptime = time.time() - self.load_completed_time
+
+        return {
+            "model_loaded": self.model_loaded,
+            "model_dir": self.model_dir,
+            "device": self.device,
+            "load_time": self.load_time,
+            "last_used": self.last_used,
+            "uptime": uptime,
+        }
+
+
+# Global backend instance
+_backend: Optional[ModelBackend] = None
+_app: Optional[FastAPI] = None
+_gallery_dir: Optional[str] = None
+_tasks: Dict[str, TaskStatus] = {}
+_executor = ThreadPoolExecutor(max_workers=1)  # Restrict to single-task execution
+_running_task_id: Optional[str] = None  # Currently running task ID
+_task_queue: List[str] = []  # Pending task queue
+
+# Task cleanup configuration
+MAX_TASK_HISTORY = 100  # Maximum number of tasks to keep in memory
+CLEANUP_INTERVAL = 300  # Cleanup interval in seconds (5 minutes)
+
+
+def _process_next_task():
+    """Process the next task in the queue."""
+    global _task_queue, _running_task_id
+
+    if not _task_queue or _running_task_id is not None:
+        return
+
+    # Get next task from queue
+    task_id = _task_queue.pop(0)
+
+    # Get task request from tasks dict (we need to store the request)
+    if task_id not in _tasks:
+        return
+
+    # Submit task to executor
+    _executor.submit(_run_inference_task, task_id)
+
+
+# get_gpu_memory_info imported from depth_anything_3.utils.memory
+
+
+# cleanup_cuda_memory imported from depth_anything_3.utils.memory
+
+
+# check_memory_availability imported from depth_anything_3.utils.memory
+
+
+# estimate_memory_requirement imported from depth_anything_3.utils.memory
+
+
+def _run_inference_task(task_id: str):
+    """Run inference task in background thread with OOM protection."""
+    global _tasks, _backend, _running_task_id, _task_queue
+
+    model = None
+    inference_started = False
+    start_time = time.time()
+
+    try:
+        # Get task request
+        if task_id not in _tasks or _tasks[task_id].request is None:
+            print(f"[{task_id}] Task not found or request missing")
+            return
+
+        request = _tasks[task_id].request
+        num_images = len(request.image_paths)
+
+        # Set current running task
+        _running_task_id = task_id
+
+        # Update task status to running
+        _tasks[task_id].status = "running"
+        _tasks[task_id].started_at = start_time
+        _tasks[task_id].message = f"[{task_id}] Starting inference on {num_images} frames..."
+        print(f"[{task_id}] Starting inference on {num_images} frames")
+
+        # Pre-inference cleanup to ensure maximum available memory
+        print(f"[{task_id}] Pre-inference cleanup...")
+        cleanup_cuda_memory()
+
+        # Check memory availability
+        estimated_memory = estimate_memory_requirement(num_images, request.process_res)
+        mem_available, mem_msg = check_memory_availability(estimated_memory)
+        print(f"[{task_id}] {mem_msg}")
+
+        if not mem_available:
+            # Try aggressive cleanup
+            print(f"[{task_id}] Insufficient memory, attempting aggressive cleanup...")
+            cleanup_cuda_memory()
+            time.sleep(0.5)  # Give system time to reclaim memory
+
+            # Check again
+            mem_available, mem_msg = check_memory_availability(estimated_memory)
+            if not mem_available:
+                raise RuntimeError(
+                    f"Insufficient GPU memory after cleanup. {mem_msg}\n"
+                    f"Suggestions:\n"
+                    f"  1. Reduce process_res (current: {request.process_res})\n"
+                    f"  2. Process fewer images at once (current: {num_images})\n"
+                    f"  3. Clear other GPU processes"
+                )
+
+        # Get model (with error handling)
+        print(f"[{task_id}] Loading model...")
+        _tasks[task_id].message = f"[{task_id}] Loading model..."
+        _tasks[task_id].progress = 0.1
+
+        try:
+            model = _backend.get_model()
+        except RuntimeError as e:
+            if "out of memory" in str(e).lower():
+                cleanup_cuda_memory()
+                raise RuntimeError(
+                    f"OOM during model loading: {str(e)}\n"
+                    f"Try reducing the batch size or resolution."
+                )
+            raise
+
+        print(f"[{task_id}] Model loaded successfully")
+        _tasks[task_id].progress = 0.2
+
+        # Prepare inference parameters
+        inference_kwargs = {
+            "image": request.image_paths,
+            "export_format": request.export_format,
+            "process_res": request.process_res,
+            "process_res_method": request.process_res_method,
+            "export_feat_layers": request.export_feat_layers,
+            "align_to_input_ext_scale": request.align_to_input_ext_scale,
+            "conf_thresh_percentile": request.conf_thresh_percentile,
+            "num_max_points": request.num_max_points,
+            "show_cameras": request.show_cameras,
+            "feat_vis_fps": request.feat_vis_fps,
+        }
+
+        if request.export_dir:
+            # Re-check against the current configuration: this task may have sat in the
+            # queue for a while after the initial request-time check, and configuration
+            # can differ if the process was restarted with a different gallery_dir.
+            validate_export_dir(request.export_dir, _gallery_dir)
+            inference_kwargs["export_dir"] = request.export_dir
+
+        if request.extrinsics:
+            inference_kwargs["extrinsics"] = np.array(request.extrinsics, dtype=np.float32)
+
+        if request.intrinsics:
+            inference_kwargs["intrinsics"] = np.array(request.intrinsics, dtype=np.float32)
+
+        # Run inference with timing
+        inference_start_time = time.time()
+        print(f"[{task_id}] Running model inference...")
+        _tasks[task_id].message = f"[{task_id}] Running model inference on {num_images} images..."
+        _tasks[task_id].progress = 0.3
+
+        inference_started = True
+
+        try:
+            model.inference(**inference_kwargs)
+            inference_time = time.time() - inference_start_time
+            avg_time_per_image = inference_time / num_images if num_images > 0 else 0
+
+            print(
+                f"[{task_id}] Inference completed in {inference_time:.2f}s "
+                f"({avg_time_per_image:.2f}s per image)"
+            )
+
+        except RuntimeError as e:
+            if "out of memory" in str(e).lower():
+                cleanup_cuda_memory()
+                raise RuntimeError(
+                    f"OOM during inference: {str(e)}\n"
+                    f"Settings: {num_images} images, resolution={request.process_res}\n"
+                    f"Suggestions:\n"
+                    f"  1. Reduce process_res to {int(request.process_res * 0.75)}\n"
+                    f"  2. Process images in smaller batches\n"
+                    f"  3. Use process_res_method='resize' instead of 'upper_bound_resize'"
+                )
+            raise
+
+        _tasks[task_id].progress = 0.9
+
+        # Post-inference cleanup
+        print(f"[{task_id}] Post-inference cleanup...")
+        cleanup_cuda_memory()
+
+        # Calculate total processing time
+        total_time = time.time() - start_time
+
+        # Update task status to completed
+        _tasks[task_id].status = "completed"
+        _tasks[task_id].completed_at = time.time()
+        _tasks[task_id].message = (
+            f"[{task_id}] Completed in {total_time:.2f}s " f"({avg_time_per_image:.2f}s per image)"
+        )
+        _tasks[task_id].progress = 1.0
+        _tasks[task_id].export_dir = request.export_dir
+
+        # Clear running state
+        _running_task_id = None
+
+        # Process next task in queue
+        _process_next_task()
+
+        print(f"[{task_id}] Task completed successfully")
+        print(
+            f"[{task_id}] Total time: {total_time:.2f}s, "
+            f"Inference time: {inference_time:.2f}s, "
+            f"Avg per image: {avg_time_per_image:.2f}s"
+        )
+
+    except Exception as e:
+        # Update task status to failed
+        error_msg = str(e)
+        total_time = time.time() - start_time
+
+        print(f"[{task_id}] Task failed after {total_time:.2f}s: {error_msg}")
+
+        # Always attempt cleanup on failure
+        cleanup_cuda_memory()
+
+        _tasks[task_id].status = "failed"
+        _tasks[task_id].completed_at = time.time()
+        _tasks[task_id].message = f"[{task_id}] Failed after {total_time:.2f}s: {error_msg}"
+
+        # Clear running state
+        _running_task_id = None
+
+        # Process next task in queue
+        _process_next_task()
+
+    finally:
+        # Final cleanup in finally block to ensure it always runs
+        # This is critical for releasing resources even if unexpected errors occur
+        try:
+            if inference_started:
+                print(f"[{task_id}] Final cleanup in finally block...")
+                cleanup_cuda_memory()
+        except Exception as e:
+            print(f"[{task_id}] Warning: Finally block cleanup failed: {e}")
+
+        # Schedule cleanup after task completion
+        _schedule_task_cleanup()
+
+
+def _cleanup_old_tasks():
+    """Clean up old completed/failed tasks to prevent memory buildup."""
+    global _tasks
+
+    current_time = time.time()
+    tasks_to_remove = []
+
+    # Find tasks to remove - more aggressive cleanup
+    for task_id, task in _tasks.items():
+        # Remove completed/failed tasks older than 10 minutes (instead of 1 hour)
+        if (
+            task.status in ["completed", "failed"]
+            and task.completed_at
+            and current_time - task.completed_at > 600
+        ):  # 10 minutes
+            tasks_to_remove.append(task_id)
+
+    # Remove old tasks
+    for task_id in tasks_to_remove:
+        del _tasks[task_id]
+        print(f"[CLEANUP] Removed old task: {task_id}")
+
+    # If still too many tasks, remove oldest completed/failed tasks
+    if len(_tasks) > MAX_TASK_HISTORY:
+        completed_tasks = [
+            (task_id, task)
+            for task_id, task in _tasks.items()
+            if task.status in ["completed", "failed"]
+        ]
+        completed_tasks.sort(key=lambda x: x[1].completed_at or 0)
+
+        excess_count = len(_tasks) - MAX_TASK_HISTORY
+        for i in range(min(excess_count, len(completed_tasks))):
+            task_id = completed_tasks[i][0]
+            del _tasks[task_id]
+            print(f"[CLEANUP] Removed excess task: {task_id}")
+
+    # Count active tasks (only pending and running)
+    active_count = sum(1 for task in _tasks.values() if task.status in ["pending", "running"])
+    print(
+        "[CLEANUP] Task cleanup completed. "
+        f"Total tasks: {len(_tasks)}, Active tasks: {active_count}"
+    )
+
+
+def _schedule_task_cleanup():
+    """Schedule task cleanup in background."""
+
+    def cleanup_worker():
+        try:
+            time.sleep(2)  # Small delay to ensure task status is updated
+            _cleanup_old_tasks()
+        except Exception as e:
+            print(f"[CLEANUP] Cleanup worker failed: {e}")
+
+    # Run cleanup in background thread
+    _executor.submit(cleanup_worker)
+
+
+# ============================================================================
+# Gallery utilities (extracted from gallery.py)
+# ============================================================================
+
+GALLERY_IMAGE_EXTS = (".png", ".jpg", ".jpeg", ".webp", ".bmp")
+
+
+def _load_gallery_html() -> str:
+    """
+    Load and modify gallery HTML to work under /gallery/ subdirectory.
+    Replaces API paths from root to /gallery/ prefix.
+    """
+    from ..services.gallery import HTML_PAGE
+
+    # Replace API paths to be under /gallery/ subdirectory
+    html = (
+        HTML_PAGE.replace("fetch('/manifest.json'", "fetch('/gallery/manifest.json'")
+        .replace("fetch('/manifest/'+", "fetch('/gallery/manifest/'+")
+        .replace(
+            "if(location.pathname!=\"/\")history.replaceState(null,'','/'+location.search)",
+            "if(!location.pathname.startsWith(\"/gallery\"))history.replaceState(null,'','/gallery/'+location.search)",
+        )
+    )
+
+    return html
+
+
+def _gallery_url_join(*parts: str) -> str:
+    """Join URL parts safely."""
+    norm = posixpath.join(*[p.replace("\\", "/") for p in parts])
+    segs = [s for s in norm.split("/") if s not in ("", ".")]
+    return "/".join(quote(s) for s in segs)
+
+
+def _is_plain_name(name: str) -> bool:
+    """Check if name is safe for use in paths."""
+    return all(c not in name for c in ("/", "\\")) and name not in (".", "..")
+
+
+def build_group_list(root_dir: str) -> dict:
+    """Build list of groups from gallery directory."""
+    groups = []
+    try:
+        for gname in sorted(os.listdir(root_dir)):
+            gpath = os.path.join(root_dir, gname)
+            if not os.path.isdir(gpath):
+                continue
+            has_scene = False
+            try:
+                for sname in os.listdir(gpath):
+                    spath = os.path.join(gpath, sname)
+                    if not os.path.isdir(spath):
+                        continue
+                    if os.path.exists(os.path.join(spath, "scene.glb")) and os.path.exists(
+                        os.path.join(spath, "scene.jpg")
+                    ):
+                        has_scene = True
+                        break
+            except Exception:
+                pass
+            if has_scene:
+                groups.append({"id": gname, "title": gname})
+    except Exception as e:
+        print(f"[warn] build_group_list failed: {e}")
+    return {"groups": groups}
+
+
+def build_group_manifest(root_dir: str, group: str) -> dict:
+    """Build manifest for a specific group."""
+    items = []
+    gpath = os.path.join(root_dir, group)
+    try:
+        if not os.path.isdir(gpath):
+            return {"group": group, "items": []}
+        for sname in sorted(os.listdir(gpath)):
+            spath = os.path.join(gpath, sname)
+            if not os.path.isdir(spath):
+                continue
+            glb_fs = os.path.join(spath, "scene.glb")
+            jpg_fs = os.path.join(spath, "scene.jpg")
+            if not (os.path.exists(glb_fs) and os.path.exists(jpg_fs)):
+                continue
+            depth_images = []
+            dpath = os.path.join(spath, "depth_vis")
+            if os.path.isdir(dpath):
+                files = [
+                    f
+                    for f in os.listdir(dpath)
+                    if os.path.splitext(f)[1].lower() in GALLERY_IMAGE_EXTS
+                ]
+                for fn in sorted(files):
+                    depth_images.append(
+                        "/gallery/" + _gallery_url_join(group, sname, "depth_vis", fn)
+                    )
+            items.append(
+                {
+                    "id": sname,
+                    "title": sname,
+                    "model": "/gallery/" + _gallery_url_join(group, sname, "scene.glb"),
+                    "thumbnail": "/gallery/" + _gallery_url_join(group, sname, "scene.jpg"),
+                    "depth_images": depth_images,
+                }
+            )
+    except Exception as e:
+        print(f"[warn] build_group_manifest failed for {group}: {e}")
+    return {"group": group, "items": items}
+
+
+# ============================================================================
+# Inference request validation
+# ============================================================================
+
+# export_dir is expected to be a simple relative path; anything else is rejected below.
+_EXPORT_DIR_METACHAR_RE = re.compile(r"[;&|`$\n\r<>\x00]")
+
+# Hosts considered "local only" for the purposes of deciding whether an unauthenticated
+# backend is safe to start.
+_LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1", "0:0:0:0:0:0:0:1"}
+
+
+def validate_export_dir(export_dir: Optional[str], gallery_dir: Optional[str]) -> Optional[str]:
+    """Validate a client-supplied export_dir before it reaches any inference/export code.
+
+    export_dir must be a simple relative path that resolves within the server's
+    configured gallery directory.
+
+    Raises:
+        ValueError: with a human-readable reason, if export_dir is rejected.
+    """
+    if not export_dir:
+        return export_dir
+
+    if _EXPORT_DIR_METACHAR_RE.search(export_dir):
+        raise ValueError("export_dir contains disallowed characters")
+
+    if os.path.isabs(export_dir) or export_dir.startswith("~"):
+        raise ValueError("export_dir must be a relative path")
+
+    if any(part == ".." for part in export_dir.replace("\\", "/").split("/")):
+        raise ValueError("export_dir must not contain '..' path traversal")
+
+    if not gallery_dir:
+        raise ValueError(
+            "export_dir is not permitted: no gallery directory is configured on this server"
+        )
+
+    gallery_root = os.path.realpath(gallery_dir)
+    resolved = os.path.realpath(export_dir)
+    if resolved != gallery_root and not resolved.startswith(gallery_root + os.sep):
+        raise ValueError(
+            "export_dir must resolve within the server's configured gallery directory"
+        )
+
+    return export_dir
+
+
+def create_app(
+    model_dir: str,
+    device: str = "cuda",
+    gallery_dir: Optional[str] = None,
+    api_key: Optional[str] = None,
+) -> FastAPI:
+    """Create FastAPI application with model backend.
+
+    Note: the non-loopback-without-authentication guard lives in start_server(), not
+    here. Callers that take this app and serve it themselves (e.g. via a custom
+    uvicorn.run() or by mounting it into a larger ASGI app) are responsible for their
+    own host-binding and authentication decisions.
+    """
+    global _backend, _app, _gallery_dir
+
+    _backend = ModelBackend(model_dir, device)
+    _app = FastAPI(
+        title="Depth Anything 3 Backend",
+        description="Model inference service for Depth Anything 3",
+        version="1.0.0",
+    )
+
+    # Module-level (not just closure-local) so background tasks can re-check it too.
+    _gallery_dir = gallery_dir
+
+    # Optional shared-secret gate for task-submitting endpoints. Falls back to the
+    # DA3_BACKEND_API_KEY env var so the key never needs to appear in a command line.
+    _api_key = api_key or os.environ.get("DA3_BACKEND_API_KEY")
+
+    async def _require_api_key(x_api_key: Optional[str] = Header(default=None, alias="X-API-Key")):
+        """FastAPI dependency enforcing the API key when one is configured."""
+        if _api_key and not hmac.compare_digest(x_api_key or "", _api_key):
+            raise HTTPException(status_code=401, detail="Missing or invalid X-API-Key header")
+
+    @_app.get("/", response_class=HTMLResponse)
+    async def root():
+        """Home page with navigation to dashboard and gallery."""
+        html_content = (
+            """
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Depth Anything 3 Backend</title>
+    <style>
+        :root {
+            --tech-blue: #00d4ff;
+            --tech-cyan: #00ffcc;
+            --tech-purple: #7877c6;
+        }
+
+        * {
+            box-sizing: border-box;
+        }
+
+        /* Dark mode styles */
+        @media (prefers-color-scheme: dark) {
+            body {
+                margin: 0;
+                font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+                background: linear-gradient(135deg, #0a0a0a 0%, #1a1a2e 50%, #16213e 100%);
+                color: #e8eaed;
+                min-height: 100vh;
+                display: flex;
+                align-items: center;
+                justify-content: center;
+                position: relative;
+                overflow-x: hidden;
+            }
+
+            body::before {
+                content: '';
+                position: fixed;
+                top: 0;
+                left: 0;
+                right: 0;
+                bottom: 0;
+                background:
+                    radial-gradient(circle at 20% 80%, rgba(120, 119, 198, 0.3) 0%, transparent 50%),
+                    radial-gradient(circle at 80% 20%, rgba(255, 119, 198, 0.3) 0%, transparent 50%),
+                    radial-gradient(circle at 40% 40%, rgba(120, 219, 255, 0.2) 0%, transparent 50%);
+                animation: techPulse 8s ease-in-out infinite;
+                z-index: -1;
+            }
+
+            .container {
+                max-width: 800px;
+                padding: 40px;
+                text-align: center;
+                z-index: 1;
+            }
+
+            h1 {
+                font-size: 3em;
+                margin: 0 0 20px 0;
+                background: linear-gradient(45deg, var(--tech-blue), var(--tech-cyan), var(--tech-purple));
+                background-size: 400% 400%;
+                -webkit-background-clip: text;
+                background-clip: text;
+                color: transparent;
+                animation: techGradient 3s ease infinite;
+                text-shadow: 0 0 30px rgba(0, 212, 255, 0.5);
+            }
+
+            .subtitle {
+                font-size: 1.2em;
+                opacity: 0.8;
+                margin-bottom: 50px;
+                color: #a0a0a0;
+            }
+
+            .nav-grid {
+                display: grid;
+                grid-template-columns: repeat(auto-fit, minmax(280px, 1fr));
+                gap: 24px;
+                margin-top: 40px;
+            }
+
+            .nav-card {
+                background: rgba(0, 0, 0, 0.3);
+                border: 1px solid rgba(0, 212, 255, 0.2);
+                border-radius: 16px;
+                padding: 30px;
+                text-decoration: none;
+                color: inherit;
+                transition: all 0.3s ease;
+                backdrop-filter: blur(10px);
+            }
+
+            .nav-card:hover {
+                transform: translateY(-4px);
+                border-color: var(--tech-blue);
+                box-shadow: 0 8px 25px rgba(0, 212, 255, 0.2);
+            }
+
+            .nav-card h2 {
+                margin: 0 0 15px 0;
+                font-size: 1.8em;
+                color: var(--tech-blue);
+            }
+
+            .nav-card p {
+                margin: 0;
+                opacity: 0.8;
+                line-height: 1.6;
+            }
+        }
+
+        /* Light mode styles */
+        @media (prefers-color-scheme: light) {
+            body {
+                margin: 0;
+                font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+                background: linear-gradient(135deg, #f8fafc 0%, #e2e8f0 50%, #cbd5e1 100%);
+                color: #1e293b;
+                min-height: 100vh;
+                display: flex;
+                align-items: center;
+                justify-content: center;
+                position: relative;
+                overflow-x: hidden;
+            }
+
+            body::before {
+                content: '';
+                position: fixed;
+                top: 0;
+                left: 0;
+                right: 0;
+                bottom: 0;
+                background:
+                    radial-gradient(circle at 20% 80%, rgba(0, 212, 255, 0.1) 0%, transparent 50%),
+                    radial-gradient(circle at 80% 20%, rgba(0, 102, 255, 0.1) 0%, transparent 50%),
+                    radial-gradient(circle at 40% 40%, rgba(0, 255, 204, 0.08) 0%, transparent 50%);
+                animation: techPulse 8s ease-in-out infinite;
+                z-index: -1;
+            }
+
+            .container {
+                max-width: 800px;
+                padding: 40px;
+                text-align: center;
+                z-index: 1;
+            }
+
+            h1 {
+                font-size: 3em;
+                margin: 0 0 20px 0;
+                background: linear-gradient(45deg, #0066ff, #00d4ff, #00ffcc);
+                background-size: 400% 400%;
+                -webkit-background-clip: text;
+                background-clip: text;
+                color: transparent;
+                animation: techGradient 3s ease infinite;
+                text-shadow: 0 0 20px rgba(0, 102, 255, 0.3);
+            }
+
+            .subtitle {
+                font-size: 1.2em;
+                opacity: 0.8;
+                margin-bottom: 50px;
+                color: #64748b;
+            }
+
+            .nav-grid {
+                display: grid;
+                grid-template-columns: repeat(auto-fit, minmax(280px, 1fr));
+                gap: 24px;
+                margin-top: 40px;
+            }
+
+            .nav-card {
+                background: rgba(255, 255, 255, 0.8);
+                border: 1px solid rgba(0, 212, 255, 0.3);
+                border-radius: 16px;
+                padding: 30px;
+                text-decoration: none;
+                color: inherit;
+                transition: all 0.3s ease;
+                backdrop-filter: blur(10px);
+                box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+            }
+
+            .nav-card:hover {
+                transform: translateY(-4px);
+                border-color: #0066ff;
+                box-shadow: 0 8px 25px rgba(0, 102, 255, 0.2);
+            }
+
+            .nav-card h2 {
+                margin: 0 0 15px 0;
+                font-size: 1.8em;
+                color: #0066ff;
+            }
+
+            .nav-card p {
+                margin: 0;
+                opacity: 0.8;
+                line-height: 1.6;
+            }
+        }
+
+        @keyframes techPulse {
+            0%, 100% { opacity: 0.5; }
+            50% { opacity: 0.8; }
+        }
+
+        @keyframes techGradient {
+            0% { background-position: 0% 50%; }
+            50% { background-position: 100% 50%; }
+            100% { background-position: 0% 50%; }
+        }
+
+        .footer {
+            margin-top: 50px;
+            opacity: 0.6;
+            font-size: 0.9em;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <h1>Depth Anything 3</h1>
+        <p class="subtitle">Model Backend Service</p>
+        <div class="nav-grid">
+            <a href="/dashboard" class="nav-card">
+                <h2>📊 Dashboard</h2>
+                <p>Monitor backend status, model information, and inference tasks in real-time.</p>
+            </a>
+            """
+            + (
+                '<a href="/gallery/" class="nav-card">'
+                "<h2>🎨 Gallery</h2>"
+                "<p>Browse 3D reconstructions and depth visualizations from processed scenes.</p>"
+                "</a>"
+                if _gallery_dir and os.path.exists(_gallery_dir)
+                else ""
+            )
+            + """
+        </div>
+        <div class="footer">
+            <p>Depth Anything 3 Backend API</p>
+        </div>
+    </div>
+</body>
+</html>
+        """
+        )
+        return HTMLResponse(html_content)
+
+    @_app.get("/dashboard", response_class=HTMLResponse)
+    async def dashboard():
+        """HTML dashboard for monitoring backend status and tasks."""
+        if _backend is None:
+            return HTMLResponse("<h1>Backend not initialized</h1>", status_code=500)
+
+        # Get backend status
+        status = _backend.get_status()
+
+        # Safely format status values
+        if status["load_time"] is not None:
+            load_time_str = f"{status['load_time']:.2f}s"
+        else:
+            load_time_str = "Not loaded"
+
+        if status["uptime"] is not None:
+            uptime_str = f"{status['uptime']:.2f}s"
+        else:
+            uptime_str = "Not running"
+
+        # Get tasks information
+        active_tasks = [task for task in _tasks.values() if task.status in ["pending", "running"]]
+        completed_tasks = [
+            task for task in _tasks.values() if task.status in ["completed", "failed"]
+        ]
+
+        # Generate task HTML
+        if active_tasks:
+            active_tasks_html = "".join(
+                _render_task_card(task, "running") for task in active_tasks
+            )
+        else:
+            active_tasks_html = "<p>No active tasks</p>"
+
+        if completed_tasks:
+            completed_tasks_html = "".join(
+                _render_task_card(task, "completed") for task in completed_tasks[-10:]
+            )
+        else:
+            completed_tasks_html = "<p>No completed tasks</p>"
+
+        # Generate HTML
+        html_content = f"""
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Depth Anything 3 Backend Dashboard</title>
+    <style>
+        body {{
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+            margin: 0;
+            padding: 20px;
+            background-color: #f5f5f5;
+        }}
+        .container {{
+            max-width: 1200px;
+            margin: 0 auto;
+        }}
+        .header {{
+            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+            color: white;
+            padding: 20px;
+            border-radius: 10px;
+            margin-bottom: 20px;
+            text-align: center;
+        }}
+        .status-grid {{
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
+            gap: 20px;
+            margin-bottom: 30px;
+        }}
+        .status-card {{
+            background: white;
+            padding: 20px;
+            border-radius: 10px;
+            box-shadow: 0 2px 10px rgba(0,0,0,0.1);
+        }}
+        .status-card h3 {{
+            margin-top: 0;
+            color: #333;
+        }}
+        .status-item {{
+            display: flex;
+            justify-content: space-between;
+            margin: 10px 0;
+            padding: 8px 0;
+            border-bottom: 1px solid #eee;
+        }}
+        .status-item:last-child {{
+            border-bottom: none;
+        }}
+        .status-value {{
+            font-weight: bold;
+            color: #666;
+        }}
+        .status-online {{
+            color: #28a745;
+        }}
+        .status-offline {{
+            color: #dc3545;
+        }}
+        .tasks-section {{
+            background: white;
+            padding: 20px;
+            border-radius: 10px;
+            box-shadow: 0 2px 10px rgba(0,0,0,0.1);
+            margin-bottom: 20px;
+        }}
+        .task-item {{
+            background: #f8f9fa;
+            padding: 15px;
+            margin: 10px 0;
+            border-radius: 8px;
+            border-left: 4px solid #007bff;
+        }}
+        .task-item.completed {{
+            border-left-color: #28a745;
+        }}
+        .task-item.failed {{
+            border-left-color: #dc3545;
+        }}
+        .task-item.running {{
+            border-left-color: #ffc107;
+        }}
+        .task-header {{
+            display: flex;
+            justify-content: space-between;
+            align-items: center;
+            margin-bottom: 8px;
+        }}
+        .task-id {{
+            font-family: monospace;
+            font-size: 12px;
+            color: #666;
+        }}
+        .task-status {{
+            padding: 4px 8px;
+            border-radius: 4px;
+            font-size: 12px;
+            font-weight: bold;
+        }}
+        .status-pending {{
+            background: #fff3cd;
+            color: #856404;
+        }}
+        .status-running {{
+            background: #d4edda;
+            color: #155724;
+        }}
+        .status-completed {{
+            background: #d1ecf1;
+            color: #0c5460;
+        }}
+        .status-failed {{
+            background: #f8d7da;
+            color: #721c24;
+        }}
+        .refresh-btn {{
+            background: #007bff;
+            color: white;
+            border: none;
+            padding: 10px 20px;
+            border-radius: 5px;
+            cursor: pointer;
+            font-size: 14px;
+        }}
+        .refresh-btn:hover {{
+            background: #0056b3;
+        }}
+        .auto-refresh {{
+            margin-left: 10px;
+        }}
+        .timestamp {{
+            font-size: 12px;
+            color: #666;
+            margin-top: 10px;
+        }}
+        .task-message {{
+            font-size: 14px;
+            color: #333;
+            margin-bottom: 8px;
+        }}
+        .task-params {{
+            font-size: 12px;
+            color: #666;
+            background: #f8f9fa;
+            padding: 6px 8px;
+            border-radius: 4px;
+            margin-top: 8px;
+        }}
+    </style>
+</head>
+<body>
+    <div class="container">
+        <div class="header">
+            <h1>Depth Anything 3 Backend Dashboard</h1>
+            <p>Real-time monitoring of model status and inference tasks</p>
+        </div>
+
+        <div class="status-grid">
+            <div class="status-card">
+                <h3>Model Status</h3>
+                <div class="status-item">
+                    <span>Status:</span>
+                    <span class="status-value {'status-online' if status['model_loaded'] else 'status-offline'}">
+                        {'Online' if status['model_loaded'] else 'Offline'}
+                    </span>
+                </div>
+                <div class="status-item">
+                    <span>Model Directory:</span>
+                    <span class="status-value">{html.escape(str(status['model_dir']))}</span>
+                </div>
+                <div class="status-item">
+                    <span>Device:</span>
+                    <span class="status-value">{html.escape(str(status['device']))}</span>
+                </div>
+                <div class="status-item">
+                    <span>Load Time:</span>
+                    <span class="status-value">{load_time_str}</span>
+                </div>
+                <div class="status-item">
+                    <span>Uptime:</span>
+                    <span class="status-value">{uptime_str}</span>
+                </div>
+            </div>
+
+            <div class="status-card">
+                <h3>Task Summary</h3>
+                <div class="status-item">
+                    <span>Active Tasks:</span>
+                    <span class="status-value">{len(active_tasks)}</span>
+                </div>
+                <div class="status-item">
+                    <span>Completed Tasks:</span>
+                    <span class="status-value">{len(completed_tasks)}</span>
+                </div>
+                <div class="status-item">
+                    <span>Total Tasks:</span>
+                    <span class="status-value">{len(_tasks)}</span>
+                </div>
+            </div>
+        </div>
+
+        <div class="tasks-section">
+            <h3>Active Tasks</h3>
+            <button class="refresh-btn" onclick="location.reload()">Refresh</button>
+            <label class="auto-refresh">
+                <input type="checkbox" id="autoRefresh" onchange="toggleAutoRefresh()"> Auto-refresh (5s)
+            </label>
+            <div class="timestamp">Last updated: <span id="lastUpdate">{time.strftime('%Y-%m-%d %H:%M:%S')}</span></div>
+
+            {active_tasks_html}
+        </div>
+
+        <div class="tasks-section">
+            <h3>Recent Completed Tasks</h3>
+            {completed_tasks_html}
+        </div>
+    </div>
+
+    <script>
+        let autoRefreshInterval;
+
+        function toggleAutoRefresh() {{
+            const checkbox = document.getElementById('autoRefresh');
+            if (checkbox.checked) {{
+                autoRefreshInterval = setInterval(() => {{
+                    location.reload();
+                }}, 5000);
+            }} else {{
+                clearInterval(autoRefreshInterval);
+            }}
+        }}
+
+        // Update timestamp every second
+        setInterval(() => {{
+            const now = new Date();
+            document.getElementById('lastUpdate').textContent = now.toLocaleString();
+        }}, 1000);
+    </script>
+</body>
+</html>
+        """
+
+        return HTMLResponse(html_content)
+
+    @_app.get("/status")
+    async def get_status():
+        """Get backend status with GPU memory information."""
+        if _backend is None:
+            raise HTTPException(status_code=500, detail="Backend not initialized")
+
+        status = _backend.get_status()
+
+        # Add GPU memory information
+        gpu_memory = get_gpu_memory_info()
+        if gpu_memory:
+            status["gpu_memory"] = {
+                "total_gb": round(gpu_memory["total_gb"], 2),
+                "allocated_gb": round(gpu_memory["allocated_gb"], 2),
+                "reserved_gb": round(gpu_memory["reserved_gb"], 2),
+                "free_gb": round(gpu_memory["free_gb"], 2),
+                "utilization_percent": round(gpu_memory["utilization"], 1),
+            }
+        else:
+            status["gpu_memory"] = None
+
+        return status
+
+    @_app.post(
+        "/inference", response_model=InferenceResponse, dependencies=[Depends(_require_api_key)]
+    )
+    async def run_inference(request: InferenceRequest):
+        """Submit inference task and return task ID."""
+        global _running_task_id
+
+        if _backend is None:
+            raise HTTPException(status_code=500, detail="Backend not initialized")
+
+        if request.export_dir:
+            try:
+                validate_export_dir(request.export_dir, _gallery_dir)
+            except ValueError as e:
+                raise HTTPException(status_code=400, detail=str(e))
+
+        # Generate unique task ID
+        task_id = str(uuid.uuid4())
+
+        # Create task status
+        if _running_task_id is not None:
+            status_msg = f"[{task_id}] Task queued (waiting for {_running_task_id} to complete)"
+        else:
+            status_msg = f"[{task_id}] Task submitted"
+
+        _tasks[task_id] = TaskStatus(
+            task_id=task_id,
+            status="pending",
+            message=status_msg,
+            created_at=time.time(),
+            export_dir=request.export_dir,
+            request=request,
+            # Record essential parameters
+            num_images=len(request.image_paths),
+            export_format=request.export_format,
+            process_res_method=request.process_res_method,
+            video_path=(
+                request.image_paths[0] if request.image_paths else None
+            ),  # Use first image path as video reference
+        )
+
+        # Add task to queue
+        _task_queue.append(task_id)
+
+        # If no task is running, start processing the queue
+        if _running_task_id is None:
+            _process_next_task()
+
+        return InferenceResponse(
+            success=True,
+            message="Task submitted successfully",
+            task_id=task_id,
+            export_dir=request.export_dir,
+            export_format=request.export_format,
+        )
+
+    @_app.get("/task/{task_id}", response_model=TaskStatus)
+    async def get_task_status(task_id: str):
+        """Get task status by task ID."""
+        if task_id not in _tasks:
+            raise HTTPException(status_code=404, detail="Task not found")
+
+        return _tasks[task_id]
+
+    @_app.get("/gpu-memory")
+    async def get_gpu_memory():
+        """Get detailed GPU memory information."""
+        gpu_memory = get_gpu_memory_info()
+        if gpu_memory is None:
+            return {
+                "available": False,
+                "message": "CUDA not available or memory info cannot be retrieved",
+            }
+
+        return {
+            "available": True,
+            "total_gb": round(gpu_memory["total_gb"], 2),
+            "allocated_gb": round(gpu_memory["allocated_gb"], 2),
+            "reserved_gb": round(gpu_memory["reserved_gb"], 2),
+            "free_gb": round(gpu_memory["free_gb"], 2),
+            "utilization_percent": round(gpu_memory["utilization"], 1),
+            "status": (
+                "healthy"
+                if gpu_memory["utilization"] < 80
+                else "warning" if gpu_memory["utilization"] < 95 else "critical"
+            ),
+        }
+
+    @_app.get("/tasks")
+    async def list_tasks():
+        """List all tasks."""
+        # Separate active and completed tasks
+        active_tasks = [task for task in _tasks.values() if task.status in ["pending", "running"]]
+        completed_tasks = [
+            task for task in _tasks.values() if task.status in ["completed", "failed"]
+        ]
+
+        return {
+            "tasks": list(_tasks.values()),
+            "active_tasks": active_tasks,
+            "completed_tasks": completed_tasks,
+            "active_count": len(active_tasks),
+            "total_count": len(_tasks),
+        }
+
+    @_app.post("/cleanup")
+    async def manual_cleanup():
+        """Manually trigger task cleanup."""
+        try:
+            _cleanup_old_tasks()
+            return {"message": "Cleanup completed", "active_tasks": len(_tasks)}
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Cleanup failed: {str(e)}")
+
+    @_app.delete("/task/{task_id}")
+    async def delete_task(task_id: str):
+        """Delete a specific task."""
+        if task_id not in _tasks:
+            raise HTTPException(status_code=404, detail="Task not found")
+
+        # Only allow deletion of completed/failed tasks
+        if _tasks[task_id].status not in ["completed", "failed"]:
+            raise HTTPException(status_code=400, detail="Cannot delete running or pending tasks")
+
+        del _tasks[task_id]
+        return {"message": f"Task {task_id} deleted successfully"}
+
+    @_app.post("/reload")
+    async def reload_model():
+        """Reload the model."""
+        if _backend is None:
+            raise HTTPException(status_code=500, detail="Backend not initialized")
+
+        try:
+            _backend.model = None
+            _backend.model_loaded = False
+            _backend.load_model()
+            return {"message": "Model reloaded successfully"}
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Failed to reload model: {str(e)}")
+
+    # ============================================================================
+    # Gallery routes
+    # ============================================================================
+
+    if _gallery_dir and os.path.exists(_gallery_dir):
+        # Load gallery HTML page (with modified paths for /gallery/ subdirectory)
+        _gallery_html = _load_gallery_html()
+
+        @_app.get("/gallery/", response_class=HTMLResponse)
+        @_app.get("/gallery", response_class=HTMLResponse)
+        async def gallery_home():
+            """Gallery home page."""
+            return HTMLResponse(_gallery_html)
+
+        @_app.get("/gallery/manifest.json")
+        async def gallery_manifest():
+            """Get gallery group list."""
+            try:
+                return build_group_list(_gallery_dir)
+            except Exception as e:
+                raise HTTPException(
+                    status_code=500, detail=f"Failed to build group list: {str(e)}"
+                )
+
+        @_app.get("/gallery/manifest/{group}.json")
+        async def gallery_group_manifest(group: str):
+            """Get manifest for a specific group."""
+            if not _is_plain_name(group):
+                raise HTTPException(status_code=400, detail="Invalid group name")
+            try:
+                return build_group_manifest(_gallery_dir, group)
+            except Exception as e:
+                raise HTTPException(
+                    status_code=500, detail=f"Failed to build group manifest: {str(e)}"
+                )
+
+        @_app.get("/gallery/{path:path}")
+        async def gallery_files(path: str):
+            """Serve gallery static files (GLB, JPG, etc.)."""
+            # Security check: prevent directory traversal
+            path_parts = path.split("/")
+            if any(not _is_plain_name(part) for part in path_parts if part):
+                raise HTTPException(status_code=400, detail="Invalid path")
+
+            file_path = os.path.join(_gallery_dir, *path_parts)
+
+            # Ensure the file is within gallery directory
+            real_file_path = os.path.realpath(file_path)
+            real_gallery_dir = os.path.realpath(_gallery_dir)
+            if not real_file_path.startswith(real_gallery_dir):
+                raise HTTPException(status_code=403, detail="Access denied")
+
+            if not os.path.exists(file_path) or not os.path.isfile(file_path):
+                raise HTTPException(status_code=404, detail="File not found")
+
+            return FileResponse(file_path)
+
+    return _app
+
+
+def start_server(
+    model_dir: str,
+    device: str = "cuda",
+    host: str = "127.0.0.1",
+    port: int = 8000,
+    gallery_dir: Optional[str] = None,
+    api_key: Optional[str] = None,
+    allow_unauthenticated: bool = False,
+):
+    """Start the backend server."""
+    api_key = api_key or os.environ.get("DA3_BACKEND_API_KEY")
+    auto_generated_key = False
+
+    if host not in _LOOPBACK_HOSTS and not api_key and not allow_unauthenticated:
+        # No key was configured for a host reachable beyond this machine: generate one
+        # for this run rather than requiring the operator to come up with one first.
+        api_key = secrets.token_urlsafe(32)
+        auto_generated_key = True
+
+    app = create_app(model_dir, device, gallery_dir, api_key=api_key)
+
+    print("Starting Depth Anything 3 Backend...")
+    print(f"Model directory: {model_dir}")
+    print(f"Device: {device}")
+    print(f"Server: http://{host}:{port}")
+    print(f"Dashboard: http://{host}:{port}/dashboard")
+    print(f"API Status: http://{host}:{port}/status")
+
+    if gallery_dir and os.path.exists(gallery_dir):
+        print(f"Gallery: http://{host}:{port}/gallery/")
+
+    if auto_generated_key:
+        print("=" * 60)
+        print("No --api-key was set, so one was generated for this run:")
+        print(f"  X-API-Key: {api_key}")
+        print("Set DA3_BACKEND_API_KEY to this value wherever you submit jobs from")
+        print("(e.g. `da3 ... --use-backend`), or pass --api-key / set")
+        print("DA3_BACKEND_API_KEY before starting the backend for a key that stays")
+        print("the same across restarts. Use --allow-unauthenticated to skip this.")
+    elif api_key:
+        print("Authentication: X-API-Key header required for /inference")
+
+    print("=" * 60)
+    print("Backend is running! You can now:")
+    print(f"  • Open home page: http://{host}:{port}")
+    print(f"  • Open dashboard: http://{host}:{port}/dashboard")
+    print(f"  • Check API status: http://{host}:{port}/status")
+
+    if gallery_dir and os.path.exists(gallery_dir):
+        print(f"  • Browse gallery: http://{host}:{port}/gallery/")
+
+    print("  • Submit inference tasks via API")
+    print("=" * 60)
+
+    uvicorn.run(app, host=host, port=port, log_level="info")
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Depth Anything 3 Backend Server")
+    parser.add_argument("--model-dir", required=True, help="Model directory path")
+    parser.add_argument("--device", default="cuda", help="Device to use")
+    parser.add_argument("--host", default="127.0.0.1", help="Host to bind to")
+    parser.add_argument("--port", type=int, default=8000, help="Port to bind to")
+    parser.add_argument("--gallery-dir", help="Gallery directory path (optional)")
+    parser.add_argument(
+        "--api-key",
+        default=None,
+        help="Require this API key (X-API-Key header) on inference requests. Falls back "
+        "to the DA3_BACKEND_API_KEY env var, or an auto-generated key when binding to a "
+        "non-loopback host.",
+    )
+    parser.add_argument(
+        "--allow-unauthenticated",
+        action="store_true",
+        help="Skip authentication entirely on a non-loopback host, instead of using an "
+        "auto-generated API key.",
+    )
+
+    args = parser.parse_args()
+    start_server(
+        args.model_dir,
+        args.device,
+        args.host,
+        args.port,
+        args.gallery_dir,
+        api_key=args.api_key,
+        allow_unauthenticated=args.allow_unauthenticated,
+    )

multivue/cameras/source/src/depth_anything_3/services/gallery.py

@@ -0,0 +1,806 @@
+#!/usr/bin/env python3
+# flake8: noqa: E501
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Depth Anything 3 Gallery Server (two-level, single-file)
+Now supports paginated depth preview (4 per page).
+"""
+
+import argparse
+import json
+import mimetypes
+import os
+import posixpath
+import sys
+from functools import partial
+from http import HTTPStatus
+from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+from urllib.parse import quote, unquote
+
+# ------------------------------ Embedded HTML ------------------------------ #
+
+HTML_PAGE = r"""<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <title>Depth Anything 3 Gallery</title>
+  <meta name="viewport" content="width=device-width, initial-scale=1" />
+  <link rel="icon" href="https://i.postimg.cc/rFSzGJ7J/light-icon.jpg" media="(prefers-color-scheme: light)">
+  <link rel="icon" href="https://i.postimg.cc/P5gZfJsf/dark-icon.jpg" media="(prefers-color-scheme: dark)">
+  <script type="module" src="https://unpkg.com/@google/model-viewer/dist/model-viewer.min.js"></script>
+  <style>
+    :root {
+      --gap:16px; --card-radius:16px; --shadow:0 8px 24px rgba(0,0,0,.12);
+      --maxW:1036px; --maxH:518px;
+      --tech-blue: #00d4ff;
+      --tech-cyan: #00ffcc;
+      --tech-purple: #7877c6;
+    }
+
+    *{ box-sizing:border-box }
+
+    /* Dark mode tech theme */
+    @media (prefers-color-scheme: dark) {
+      body{
+        margin:0; font:16px/1.5 system-ui,-apple-system,Segoe UI,Roboto,sans-serif;
+        background: linear-gradient(135deg, #0a0a0a 0%, #1a1a2e 50%, #16213e 100%);
+        color:#e8eaed;
+        position: relative;
+        overflow-x: hidden;
+      }
+
+      body::before {
+        content: '';
+        position: fixed;
+        top: 0;
+        left: 0;
+        right: 0;
+        bottom: 0;
+        background:
+          radial-gradient(circle at 20% 80%, rgba(120, 119, 198, 0.3) 0%, transparent 50%),
+          radial-gradient(circle at 80% 20%, rgba(255, 119, 198, 0.3) 0%, transparent 50%),
+          radial-gradient(circle at 40% 40%, rgba(120, 219, 255, 0.2) 0%, transparent 50%);
+        animation: techPulse 8s ease-in-out infinite;
+        z-index: -1;
+      }
+    }
+
+    /* Light mode tech theme */
+    @media (prefers-color-scheme: light) {
+      body{
+        margin:0; font:16px/1.5 system-ui,-apple-system,Segoe UI,Roboto,sans-serif;
+        background: linear-gradient(135deg, #f8fafc 0%, #e2e8f0 50%, #cbd5e1 100%);
+        color:#1e293b;
+        position: relative;
+        overflow-x: hidden;
+      }
+
+      body::before {
+        content: '';
+        position: fixed;
+        top: 0;
+        left: 0;
+        right: 0;
+        bottom: 0;
+        background:
+          radial-gradient(circle at 20% 80%, rgba(0, 212, 255, 0.1) 0%, transparent 50%),
+          radial-gradient(circle at 80% 20%, rgba(0, 102, 255, 0.1) 0%, transparent 50%),
+          radial-gradient(circle at 40% 40%, rgba(0, 255, 204, 0.08) 0%, transparent 50%);
+        animation: techPulse 8s ease-in-out infinite;
+        z-index: -1;
+      }
+    }
+
+    @keyframes techPulse {
+      0%, 100% { opacity: 0.5; }
+      50% { opacity: 0.8; }
+    }
+
+    @keyframes techGradient {
+      0% { background-position: 0% 50%; }
+      50% { background-position: 100% 50%; }
+      100% { background-position: 0% 50%; }
+    }
+
+    /* Dark mode header */
+    @media (prefers-color-scheme: dark) {
+      header{
+        padding:20px 24px; position:sticky; top:0;
+        background:linear-gradient(180deg,rgba(10,10,10,0.9) 60%,rgba(10,10,10,0));
+        z-index:2; border-bottom:1px solid rgba(0, 212, 255, 0.2);
+        backdrop-filter: blur(10px);
+      }
+
+      h1{
+        margin:0; font-size:22px;
+        background: linear-gradient(45deg, var(--tech-blue), var(--tech-cyan), var(--tech-purple));
+        background-size: 400% 400%;
+        -webkit-background-clip: text;
+        background-clip: text;
+        color: transparent;
+        animation: techGradient 3s ease infinite;
+        text-shadow: 0 0 30px rgba(0, 212, 255, 0.5);
+      }
+
+      .muted{ opacity:.7; font-size:13px; color: #a0a0a0; }
+
+      #backBtn{
+        display:none; padding:6px 10px; border-radius:10px;
+        border:1px solid rgba(0, 212, 255, 0.3);
+        background:rgba(0, 0, 0, 0.3);
+        color:#e8eaed; cursor:pointer;
+        transition: all 0.3s ease;
+      }
+
+      #backBtn:hover {
+        border-color: var(--tech-blue);
+        box-shadow: 0 0 10px rgba(0, 212, 255, 0.3);
+      }
+
+      #search{
+        flex:1 1 260px; min-width:240px; max-width:520px;
+        padding:10px 14px; border-radius:12px;
+        border:1px solid rgba(0, 212, 255, 0.3);
+        background:rgba(0, 0, 0, 0.3);
+        color:#e8eaed; outline:none;
+        transition: all 0.3s ease;
+      }
+
+      #search:focus {
+        border-color: var(--tech-blue);
+        box-shadow: 0 0 10px rgba(0, 212, 255, 0.3);
+      }
+    }
+
+    /* Light mode header */
+    @media (prefers-color-scheme: light) {
+      header{
+        padding:20px 24px; position:sticky; top:0;
+        background:linear-gradient(180deg,rgba(248,250,252,0.9) 60%,rgba(248,250,252,0));
+        z-index:2; border-bottom:1px solid rgba(0, 212, 255, 0.3);
+        backdrop-filter: blur(10px);
+      }
+
+      h1{
+        margin:0; font-size:22px;
+        background: linear-gradient(45deg, #0066ff, #00d4ff, #00ffcc);
+        background-size: 400% 400%;
+        -webkit-background-clip: text;
+        background-clip: text;
+        color: transparent;
+        animation: techGradient 3s ease infinite;
+        text-shadow: 0 0 20px rgba(0, 102, 255, 0.3);
+      }
+
+      .muted{ opacity:.7; font-size:13px; color: #64748b; }
+
+      #backBtn{
+        display:none; padding:6px 10px; border-radius:10px;
+        border:1px solid rgba(0, 212, 255, 0.4);
+        background:rgba(255, 255, 255, 0.8);
+        color:#1e293b; cursor:pointer;
+        transition: all 0.3s ease;
+      }
+
+      #backBtn:hover {
+        border-color: #0066ff;
+        box-shadow: 0 0 10px rgba(0, 102, 255, 0.3);
+      }
+
+      #search{
+        flex:1 1 260px; min-width:240px; max-width:520px;
+        padding:10px 14px; border-radius:12px;
+        border:1px solid rgba(0, 212, 255, 0.4);
+        background:rgba(255, 255, 255, 0.8);
+        color:#1e293b; outline:none;
+        transition: all 0.3s ease;
+      }
+
+      #search:focus {
+        border-color: #0066ff;
+        box-shadow: 0 0 10px rgba(0, 102, 255, 0.3);
+      }
+    }
+
+    .row{ display:flex; gap:12px; align-items:center; flex-wrap:wrap; justify-content:center; }
+
+    main{ padding:16px 24px 24px; display:grid; place-items:center; }
+
+    .group-wrap{ width:min(900px,100%); }
+    .group-list{ list-style:none; margin:0; padding:0; display:grid; gap:10px; }
+
+    /* Dark mode cards */
+    @media (prefers-color-scheme: dark) {
+      .group-item{
+        display:flex; align-items:center; gap:12px; padding:12px 14px;
+        background:rgba(0, 0, 0, 0.3); border:1px solid rgba(0, 212, 255, 0.2); border-radius:14px; cursor:pointer;
+        transition: all 0.3s ease;
+        backdrop-filter: blur(10px);
+      }
+      .group-item:hover{
+        transform: translateY(-1px);
+        border-color:var(--tech-blue);
+        box-shadow: 0 4px 15px rgba(0, 212, 255, 0.2);
+      }
+
+      .card{
+        background:rgba(0, 0, 0, 0.3); border:1px solid rgba(0, 212, 255, 0.2); border-radius:var(--card-radius);
+        overflow:hidden; box-shadow:var(--shadow);
+        transition:all 0.3s ease; cursor:pointer; display:flex; flex-direction:column; max-width:var(--maxW);
+        backdrop-filter: blur(10px);
+      }
+      .card:hover{
+        transform:translateY(-2px);
+        border-color:var(--tech-blue);
+        box-shadow: 0 8px 25px rgba(0, 212, 255, 0.2);
+      }
+      .thumb-box{
+        position:relative; width:100%; aspect-ratio:2/1;
+        background:linear-gradient(135deg, #0e121b 0%, #1a1a2e 100%);
+        display:grid; place-items:center; overflow:hidden;
+        border-bottom: 1px solid rgba(0, 212, 255, 0.1);
+      }
+      .open{
+        font-size:12px; opacity:.7; padding:6px 8px;
+        border:1px solid rgba(0, 212, 255, 0.3);
+        border-radius:10px;
+        background:rgba(0, 212, 255, 0.1);
+        transition: all 0.3s ease;
+      }
+      .open:hover {
+        background:rgba(0, 212, 255, 0.2);
+        border-color: var(--tech-blue);
+      }
+    }
+
+    /* Light mode cards */
+    @media (prefers-color-scheme: light) {
+      .group-item{
+        display:flex; align-items:center; gap:12px; padding:12px 14px;
+        background:rgba(255, 255, 255, 0.8); border:1px solid rgba(0, 212, 255, 0.3); border-radius:14px; cursor:pointer;
+        transition: all 0.3s ease;
+        backdrop-filter: blur(10px);
+        box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
+      }
+      .group-item:hover{
+        transform: translateY(-1px);
+        border-color:#0066ff;
+        box-shadow: 0 4px 15px rgba(0, 102, 255, 0.2);
+      }
+
+      .card{
+        background:rgba(255, 255, 255, 0.8); border:1px solid rgba(0, 212, 255, 0.3); border-radius:var(--card-radius);
+        overflow:hidden; box-shadow:0 4px 6px rgba(0, 0, 0, 0.1);
+        transition:all 0.3s ease; cursor:pointer; display:flex; flex-direction:column; max-width:var(--maxW);
+        backdrop-filter: blur(10px);
+      }
+      .card:hover{
+        transform:translateY(-2px);
+        border-color:#0066ff;
+        box-shadow: 0 8px 25px rgba(0, 102, 255, 0.2);
+      }
+      .thumb-box{
+        position:relative; width:100%; aspect-ratio:2/1;
+        background:linear-gradient(135deg, #f8fafc 0%, #e2e8f0 100%);
+        display:grid; place-items:center; overflow:hidden;
+        border-bottom: 1px solid rgba(0, 212, 255, 0.2);
+      }
+      .open{
+        font-size:12px; opacity:.7; padding:6px 8px;
+        border:1px solid rgba(0, 212, 255, 0.4);
+        border-radius:10px;
+        background:rgba(0, 212, 255, 0.1);
+        transition: all 0.3s ease;
+      }
+      .open:hover {
+        background:rgba(0, 212, 255, 0.2);
+        border-color: #0066ff;
+      }
+    }
+
+    .gname{ font-weight:600; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; width:100%; }
+    .grid{
+      width:min(1200px,100%);
+      display:grid;
+      grid-template-columns:repeat(auto-fill,minmax(260px,1fr));
+      gap:var(--gap);
+      align-items:start;
+      justify-items:stretch;
+      margin: 0 auto;
+      padding: 0 20px;
+    }
+    .thumb{ max-width:100%; max-height:100%; object-fit:contain; display:block; }
+    .meta{ padding:12px 14px; display:flex; justify-content:space-between; align-items:center; gap:8px; }
+    .title{ font-weight:600; font-size:14px; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }
+    .empty{ opacity:.6; padding:40px 0; text-align:center; }
+    .crumb{ font-size:13px; opacity:.8; }
+
+    .overlay{ position:fixed; inset:0; background:rgba(0,0,0,.6); display:none; place-items:center; padding:20px; z-index:10; }
+    .overlay.show{ display:grid; }
+
+    /* Dark mode viewer */
+    @media (prefers-color-scheme: dark) {
+      .viewer{
+        inline-size:min(92vw,var(--maxW));
+        block-size:min(82vh,var(--maxH));
+        background:#0e121b; border:1px solid rgba(0, 212, 255, 0.3); border-radius:18px; overflow:hidden; position:relative; box-shadow:0 12px 36px rgba(0,0,0,.35);
+        display:grid;
+      }
+      .chip{ background:rgba(0,0,0,.45); border:1px solid rgba(0, 212, 255, 0.3); color:#e8eaed; padding:6px 10px; border-radius:12px; font-size:12px; max-width:60%; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }
+      .btn{ margin-left:auto; background:rgba(0, 0, 0, 0.3); color:#e8eaed; border:1px solid rgba(0, 212, 255, 0.3); border-radius:10px; padding:6px 10px; cursor:pointer; transition: all 0.3s ease; }
+      .btn:hover { border-color: var(--tech-blue); box-shadow: 0 0 10px rgba(0, 212, 255, 0.3); }
+      .mv-box{ width:100%; aspect-ratio:1036/518; background:#0b0d12; border:1px solid rgba(0, 212, 255, 0.2); border-radius:12px; overflow:hidden; }
+      .mv-box model-viewer{ width:100%; height:100%; background:#0b0d12; }
+      .res-cell{ position:relative; width:100%; aspect-ratio:2/1; background:#0e121b; border:1px solid rgba(0, 212, 255, 0.2); border-radius:12px; overflow:hidden; display:grid; place-items:center; }
+      .res-empty{ position:absolute; inset:0; display:grid; place-items:center; opacity:.55; font-size:12px; color:#9aa0a6; }
+      .download-icon{ background:rgba(0, 0, 0, 0.6); border:1px solid rgba(0, 212, 255, 0.3); color:#e8eaed; box-shadow:0 4px 12px rgba(0,0,0,0.3); }
+      .download-icon:hover{ background:rgba(0, 212, 255, 0.2); border-color:var(--tech-blue); box-shadow:0 0 20px rgba(0, 212, 255, 0.4); transform:scale(1.05); }
+    }
+
+    /* Light mode viewer */
+    @media (prefers-color-scheme: light) {
+      .viewer{
+        inline-size:min(92vw,var(--maxW));
+        block-size:min(82vh,var(--maxH));
+        background:#f8fafc; border:1px solid rgba(0, 212, 255, 0.4); border-radius:18px; overflow:hidden; position:relative; box-shadow:0 12px 36px rgba(0,0,0,.15);
+        display:grid;
+      }
+      .chip{ background:rgba(255,255,255,0.8); border:1px solid rgba(0, 212, 255, 0.4); color:#1e293b; padding:6px 10px; border-radius:12px; font-size:12px; max-width:60%; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }
+      .btn{ margin-left:auto; background:rgba(255, 255, 255, 0.8); color:#1e293b; border:1px solid rgba(0, 212, 255, 0.4); border-radius:10px; padding:6px 10px; cursor:pointer; transition: all 0.3s ease; }
+      .btn:hover { border-color: #0066ff; box-shadow: 0 0 10px rgba(0, 102, 255, 0.3); }
+      .mv-box{ width:100%; aspect-ratio:1036/518; background:#f8fafc; border:1px solid rgba(0, 212, 255, 0.3); border-radius:12px; overflow:hidden; }
+      .mv-box model-viewer{ width:100%; height:100%; background:#f8fafc; }
+      .res-cell{ position:relative; width:100%; aspect-ratio:2/1; background:#f8fafc; border:1px solid rgba(0, 212, 255, 0.3); border-radius:12px; overflow:hidden; display:grid; place-items:center; }
+      .res-empty{ position:absolute; inset:0; display:grid; place-items:center; opacity:.55; font-size:12px; color:#64748b; }
+      .download-icon{ background:rgba(255, 255, 255, 0.9); border:1px solid rgba(0, 212, 255, 0.4); color:#1e293b; box-shadow:0 4px 12px rgba(0,0,0,0.15); }
+      .download-icon:hover{ background:rgba(0, 212, 255, 0.2); border-color:#0066ff; box-shadow:0 0 20px rgba(0, 102, 255, 0.4); transform:scale(1.05); }
+    }
+
+    .viewer-header{ position:absolute; top:8px; left:8px; right:8px; display:flex; gap:8px; align-items:center; z-index:2; }
+    .viewer-body{ height:100%; display:grid; grid-template-rows:auto auto; gap:12px; padding:36px 8px 8px 8px; overflow:auto; }
+    .res-grid{ display:grid; grid-template-columns:1fr 1fr; gap:8px; }
+    .res-img{ max-width:100%; max-height:100%; object-fit:contain; display:block; }
+    .download-icon{ position:absolute; bottom:16px; right:16px; width:44px; height:44px; border-radius:50%; display:grid; place-items:center; font-size:20px; cursor:pointer; z-index:3; transition:all 0.3s ease; }
+
+    /* Pagination controls */
+    .pager {
+      grid-column: 1 / -1;
+      justify-content: center;
+      align-items: center;
+      display: flex;
+      gap: 16px;
+      margin-top: 8px;
+      font-size: 13px;
+      text-align: center;
+    }
+
+    /* Dark mode pagination */
+    @media (prefers-color-scheme: dark) {
+      .pager {
+        color: #ccc;
+      }
+      .pager button {
+        padding: 4px 10px;
+        border-radius: 8px;
+        border: 1px solid rgba(0, 212, 255, 0.3);
+        background: rgba(0, 0, 0, 0.3);
+        color: #e8eaed;
+        cursor: pointer;
+        transition: all 0.3s ease;
+      }
+      .pager button:hover:not(:disabled) {
+        border-color: var(--tech-blue);
+        box-shadow: 0 0 8px rgba(0, 212, 255, 0.2);
+      }
+      .pager button:disabled {
+        opacity: 0.4;
+        cursor: not-allowed;
+      }
+    }
+
+    /* Light mode pagination */
+    @media (prefers-color-scheme: light) {
+      .pager {
+        color: #64748b;
+      }
+      .pager button {
+        padding: 4px 10px;
+        border-radius: 8px;
+        border: 1px solid rgba(0, 212, 255, 0.4);
+        background: rgba(255, 255, 255, 0.8);
+        color: #1e293b;
+        cursor: pointer;
+        transition: all 0.3s ease;
+      }
+      .pager button:hover:not(:disabled) {
+        border-color: #0066ff;
+        box-shadow: 0 0 8px rgba(0, 102, 255, 0.2);
+      }
+      .pager button:disabled {
+        opacity: 0.4;
+        cursor: not-allowed;
+      }
+    }
+
+    /* Intro card styles */
+    @media (prefers-color-scheme: dark) {
+      .intro-card {
+        background: linear-gradient(135deg, rgba(0, 212, 255, 0.1) 0%, rgba(0, 102, 255, 0.1) 100%);
+        border: 1px solid rgba(0, 212, 255, 0.2);
+        backdrop-filter: blur(10px);
+      }
+      .intro-title {
+        background: linear-gradient(45deg, var(--tech-blue), var(--tech-cyan), var(--tech-purple));
+        background-size: 400% 400%;
+        -webkit-background-clip: text;
+        background-clip: text;
+        color: transparent;
+        animation: techGradient 3s ease infinite;
+        text-shadow: 0 0 20px rgba(0, 212, 255, 0.3);
+      }
+      .intro-description {
+        color: #e0e0e0;
+      }
+    }
+
+    @media (prefers-color-scheme: light) {
+      .intro-card {
+        background: linear-gradient(135deg, rgba(0, 212, 255, 0.05) 0%, rgba(0, 102, 255, 0.05) 100%);
+        border: 1px solid rgba(0, 212, 255, 0.3);
+        backdrop-filter: blur(10px);
+        box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+      }
+      .intro-title {
+        background: linear-gradient(45deg, #0066ff, #00d4ff, #00ffcc);
+        background-size: 400% 400%;
+        -webkit-background-clip: text;
+        background-clip: text;
+        color: transparent;
+        animation: techGradient 3s ease infinite;
+        text-shadow: 0 0 15px rgba(0, 102, 255, 0.2);
+      }
+      .intro-description {
+        color: #334155;
+      }
+    }
+
+    footer{
+      opacity:.55;
+      font-size:12px;
+      padding:12px 24px 24px;
+      text-align:center;
+      display:flex;
+      justify-content:center;
+      align-items:center;
+      width:100%;
+    }
+  </style>
+</head>
+<body>
+  <header>
+    <div class="row">
+      <button id="backBtn">← Back</button>
+      <h1 id="pageTitle">Depth Anything 3 Gallery</h1>
+      <span id="crumb" class="crumb"></span>
+      <input id="search" placeholder="Search…" />
+    </div>
+    <div class="muted" id="hint" style="text-align: center;">Level 1 shows groups only; click a group to browse scenes and previews.</div>
+  </header>
+
+  <main>
+    <!-- Tech intro card -->
+    <div class="intro-card" style="margin-bottom: 30px; padding: 25px; border-radius: 15px; text-align: center; max-width: 800px;">
+      <h2 class="intro-title" style="margin: 0 0 15px 0; font-size: 1.8em; font-weight: 700;">
+        🎯 Depth Anything 3 Gallery
+      </h2>
+      <p class="intro-description" style="margin: 0; font-size: 1.1em; line-height: 1.6;">
+        Explore 3D reconstructions and depth visualizations from Depth Anything 3.
+        Browse through groups of scenes, preview 3D models, and examine depth maps interactively.
+      </p>
+    </div>
+
+    <div id="level1" class="group-wrap" aria-live="polite">
+      <ul id="groupList" class="group-list"></ul>
+      <div id="groupEmpty" class="empty" style="display:none;">No available groups</div>
+    </div>
+
+    <div id="level2" style="display:none; width:100%;" aria-live="polite">
+      <div id="topPager" class="pager" style="margin-bottom: 16px;"></div>
+      <div id="grid" class="grid"></div>
+      <div id="sceneEmpty" class="empty" style="display:none;">No available scenes in this group</div>
+    </div>
+  </main>
+
+  <div id="overlay" class="overlay" role="dialog" aria-modal="true" aria-label="3D Preview">
+    <div class="viewer" id="viewer">
+      <div class="viewer-header">
+        <div id="viewerTitle" class="chip">Loading…</div>
+        <button id="toggleView" class="btn" title="Toggle between 3D-only and resource view">Resource View</button>
+        <button id="closeBtn" class="btn">Close</button>
+      </div>
+      <div id="downloadBtn" class="download-icon" title="Download GLB model">⬇</div>
+      <div class="viewer-body">
+        <div class="mv-box"><model-viewer id="mv"
+          src=""
+          ar
+          camera-controls
+          auto-rotate
+          interaction-prompt="auto"
+          shadow-intensity="0.7"
+          exposure="1.0"
+          alt="GLB Preview"></model-viewer></div>
+        <div class="res-grid" id="resGrid" hidden></div>
+      </div>
+    </div>
+  </div>
+
+  <footer>Depth Anything 3 Gallery. Copyright 2025 Depth Anything 3 authors.</footer>
+
+<script>
+const level1=document.getElementById('level1'),level2=document.getElementById('level2'),pageTitle=document.getElementById('pageTitle'),crumb=document.getElementById('crumb'),backBtn=document.getElementById('backBtn'),hint=document.getElementById('hint'),searchInput=document.getElementById('search'),groupList=document.getElementById('groupList'),groupEmpty=document.getElementById('groupEmpty'),topPager=document.getElementById('topPager'),grid=document.getElementById('grid'),sceneEmpty=document.getElementById('sceneEmpty'),overlay=document.getElementById('overlay'),viewer=document.getElementById('viewer'),mv=document.getElementById('mv'),viewerTitle=document.getElementById('viewerTitle'),downloadBtn=document.getElementById('downloadBtn'),toggleViewBtn=document.getElementById('toggleView'),closeBtn=document.getElementById('closeBtn'),resGrid=document.getElementById('resGrid');
+let GROUPS=[],SCENES=[],currentGroup=null,currentScene=null,currentPage=1,currentScenePage=1;
+
+const qs=()=>new URLSearchParams(location.search);
+async function loadGroups(){const r=await fetch('/manifest.json',{cache:'no-store'});if(!r.ok)throw new Error(r.status+' '+r.statusText);const j=await r.json();GROUPS=j.groups||[];renderGroups(GROUPS);}
+async function loadScenes(g){const r=await fetch('/manifest/'+encodeURIComponent(g)+'.json',{cache:'no-store'});if(!r.ok)throw new Error(r.status+' '+r.statusText);const j=await r.json();SCENES=j.items||[];const p=parseInt(qs().get('page'))||1;renderScenes(SCENES,p);}
+function renderGroups(list){groupList.innerHTML='';const q=searchInput.value.trim().toLowerCase();const f=list.filter(g=>(g.title||g.id||'').toLowerCase().includes(q));if(!f.length){groupEmpty.style.display='';return;}groupEmpty.style.display='none';for(const g of f){const li=document.createElement('li');li.className='group-item';li.title=g.title||g.id;li.onclick=()=>enterLevel2(g.id,{push:true});const name=document.createElement('div');name.className='gname';name.textContent=g.title||g.id;li.appendChild(name);groupList.appendChild(li);}}
+function renderScenes(list,page=1){topPager.innerHTML='';grid.innerHTML='';const q=searchInput.value.trim().toLowerCase();const f=list.filter(x=>(x.title||'').toLowerCase().includes(q)||(x.id||'').toLowerCase().includes(q));if(!f.length){sceneEmpty.style.display='';topPager.style.display='none';return;}sceneEmpty.style.display='none';topPager.style.display='flex';const perPage=16;const total=f.length;const totalPages=Math.max(1,Math.ceil(total/perPage));currentScenePage=page;const u=new URL(location.href);u.searchParams.set('page',page);history.replaceState(null,'',u);const subset=f.slice((page-1)*perPage,page*perPage);for(const i of subset){const c=document.createElement('div');c.className='card';c.title=i.title;const b=document.createElement('div');b.className='thumb-box';const img=document.createElement('img');img.className='thumb';img.loading='lazy';img.alt=i.title;img.src=i.thumbnail;b.appendChild(img);const m=document.createElement('div');m.className='meta';const t=document.createElement('div');t.className='title';t.textContent=i.title;const o=document.createElement('div');o.className='open';o.textContent='Preview';m.appendChild(t);m.appendChild(o);c.appendChild(b);c.appendChild(m);c.onclick=()=>openViewer(i,{push:true});grid.appendChild(c);}function buildPager(){const pg=document.createElement('div');pg.className='pager';const prev=document.createElement('button');prev.textContent='← Prev';prev.disabled=page<=1;prev.onclick=()=>renderScenes(list,page-1);const info=document.createElement('span');info.textContent=`${page} / ${totalPages}`;const next=document.createElement('button');next.textContent='Next →';next.disabled=page>=totalPages;next.onclick=()=>renderScenes(list,page+1);pg.appendChild(prev);pg.appendChild(info);pg.appendChild(next);return pg;}topPager.innerHTML='';topPager.appendChild(buildPager());grid.appendChild(buildPager());}
+function enterLevel1({push=false}={}){currentGroup=null;pageTitle.textContent='Depth Anything 3 Gallery';crumb.textContent='';backBtn.style.display='none';hint.style.display='';level1.style.display='';level2.style.display='none';overlay.classList.remove('show');mv.src='';const u=new URL(location.href);u.searchParams.delete('group');u.searchParams.delete('id');u.searchParams.delete('page');push?history.pushState(null,'',u):history.replaceState(null,'',u);searchInput.value='';loadGroups().catch(e=>{groupList.innerHTML='';groupEmpty.style.display='';groupEmpty.textContent='Failed to load groups: '+e;});}
+async function enterLevel2(g,{push=false}={}){currentGroup=g;pageTitle.textContent=g;crumb.textContent='(group)';backBtn.style.display='';hint.style.display='none';level1.style.display='none';level2.style.display='';overlay.classList.remove('show');mv.src='';const u=new URL(location.href);u.searchParams.set('group',g);u.searchParams.delete('id');push?history.pushState(null,'',u):history.replaceState(null,'',u);searchInput.value='';try{await loadScenes(g);const id=qs().get('id');if(id){const hit=SCENES.find(x=>x.id===id);if(hit)openViewer(hit,{push:false});}}catch(e){grid.innerHTML='';sceneEmpty.style.display='';sceneEmpty.textContent='Failed to load scenes: '+e;}}
+function buildResGrid(i,page=1){
+  resGrid.innerHTML='';
+  const imgs=i.depth_images||[];
+  const perPage=4;
+  const total=imgs.length;
+  const totalPages=Math.max(1, Math.ceil(total/perPage));
+  currentPage=page;
+
+  const subset=imgs.slice((page-1)*perPage,(page-1)*perPage+perPage);
+  for(let k=0;k<4;k++){
+    const cell=document.createElement('div');
+    cell.className='res-cell';
+    if(subset[k]){
+      const im=document.createElement('img');
+      im.className='res-img';
+      im.src=subset[k];
+      im.alt=(i.title||'scene')+' depth '+(k+1+(page-1)*perPage);
+      im.loading='lazy';
+      cell.appendChild(im);
+    } else {
+      const ph=document.createElement('div');
+      ph.className='res-empty';
+      ph.textContent='N/A';
+      cell.appendChild(ph);
+    }
+    resGrid.appendChild(cell);
+  }
+
+  // pagination bar (always rebuilt)
+  const pager=document.createElement('div');
+  pager.className='pager';
+
+  const prev=document.createElement('button');
+  prev.textContent='← Prev';
+  prev.disabled=page<=1;
+  prev.onclick=()=>buildResGrid(i,page-1);
+
+  const info=document.createElement('span');
+  info.textContent=`${page} / ${totalPages}`;
+
+  const next=document.createElement('button');
+  next.textContent='Next →';
+  next.disabled=page>=totalPages;
+  next.onclick=()=>buildResGrid(i,page+1);
+
+  pager.appendChild(prev);
+  pager.appendChild(info);
+  pager.appendChild(next);
+  resGrid.appendChild(pager);
+}
+function openViewer(i,{push=false}={}){currentScene=i;viewerTitle.textContent=i.title;mv.src=i.model;overlay.classList.add('show');resGrid.hidden=true;toggleViewBtn.textContent='Resource View';viewer.style.blockSize='min(82vh,var(--maxH))';buildResGrid(i,1);downloadBtn.onclick=()=>{const a=document.createElement('a');a.href=i.model;a.download=i.title+'.glb';a.click();};if(push){const u=new URL(location.href);if(!u.searchParams.get('group'))u.searchParams.set('group',currentGroup||'');u.searchParams.set('id',i.id);history.pushState(null,'',u);}}
+function toggleView(){const hidden=!resGrid.hidden;resGrid.hidden=hidden;toggleViewBtn.textContent=hidden?'Resource View':'3D Only';viewer.style.blockSize=hidden?'min(82vh,var(--maxH))':'min(92vh,900px)';}
+function closeViewer(){const hasId=!!qs().get('id');if(hasId&&history.length>1){history.back();return;}const u=new URL(location.href);u.searchParams.delete('id');history.replaceState(null,'',u);overlay.classList.remove('show');mv.src='';}
+overlay.onclick=e=>{if(e.target===overlay)closeViewer();};closeBtn.onclick=closeViewer;toggleViewBtn.onclick=toggleView;backBtn.onclick=()=>history.back();
+searchInput.oninput=()=>{!qs().get('group')?renderGroups(GROUPS):renderScenes(SCENES,1);};
+window.onpopstate=()=>routeFromURL();
+async function routeFromURL(){if(location.pathname!="/")history.replaceState(null,'','/'+location.search);const g=qs().get('group');const id=qs().get('id');if(!g){enterLevel1({push:false});return;}await enterLevel2(g,{push:false});if(id){const hit=SCENES.find(x=>x.id===id);if(hit)openViewer(hit,{push:false});else{overlay.classList.remove('show');mv.src='';}}else{overlay.classList.remove('show');mv.src='';}}
+routeFromURL();
+</script>
+</body>
+</html>
+"""
+
+# ------------------------------ Utilities ------------------------------ #
+
+IMAGE_EXTS = (".png", ".jpg", ".jpeg", ".webp", ".bmp")
+
+
+def _url_join(*parts: str) -> str:
+    norm = posixpath.join(*[p.replace("\\", "/") for p in parts])
+    segs = [s for s in norm.split("/") if s not in ("", ".")]
+    return "/".join(quote(s) for s in segs)
+
+
+def _is_plain_name(name: str) -> bool:
+    return all(c not in name for c in ("/", "\\")) and name not in (".", "..")
+
+
+def build_group_list(root_dir: str) -> dict:
+    groups = []
+    try:
+        for gname in sorted(os.listdir(root_dir)):
+            gpath = os.path.join(root_dir, gname)
+            if not os.path.isdir(gpath):
+                continue
+            has_scene = False
+            try:
+                for sname in os.listdir(gpath):
+                    spath = os.path.join(gpath, sname)
+                    if not os.path.isdir(spath):
+                        continue
+                    if os.path.exists(os.path.join(spath, "scene.glb")) and os.path.exists(
+                        os.path.join(spath, "scene.jpg")
+                    ):
+                        has_scene = True
+                        break
+            except Exception:
+                pass
+            if has_scene:
+                groups.append({"id": gname, "title": gname})
+    except Exception as e:
+        print(f"[warn] build_group_list failed: {e}", file=sys.stderr)
+    return {"groups": groups}
+
+
+def build_group_manifest(root_dir: str, group: str) -> dict:
+    items = []
+    gpath = os.path.join(root_dir, group)
+    try:
+        if not os.path.isdir(gpath):
+            return {"group": group, "items": []}
+        for sname in sorted(os.listdir(gpath)):
+            spath = os.path.join(gpath, sname)
+            if not os.path.isdir(spath):
+                continue
+            glb_fs = os.path.join(spath, "scene.glb")
+            jpg_fs = os.path.join(spath, "scene.jpg")
+            if not (os.path.exists(glb_fs) and os.path.exists(jpg_fs)):
+                continue
+            depth_images = []
+            dpath = os.path.join(spath, "depth_vis")
+            if os.path.isdir(dpath):
+                files = [
+                    f for f in os.listdir(dpath) if os.path.splitext(f)[1].lower() in IMAGE_EXTS
+                ]
+                for fn in sorted(files):
+                    depth_images.append("/" + _url_join(group, sname, "depth_vis", fn))
+            items.append(
+                {
+                    "id": sname,
+                    "title": sname,
+                    "model": "/" + _url_join(group, sname, "scene.glb"),
+                    "thumbnail": "/" + _url_join(group, sname, "scene.jpg"),
+                    "depth_images": depth_images,
+                }
+            )
+    except Exception as e:
+        print(f"[warn] build_group_manifest failed for {group}: {e}", file=sys.stderr)
+    return {"group": group, "items": items}
+
+
+class GalleryHandler(SimpleHTTPRequestHandler):
+    def __init__(self, *args, directory=None, **kwargs):
+        super().__init__(*args, directory=directory, **kwargs)
+
+    def do_GET(self):
+        if self.path in ("/", "/index.html") or self.path.startswith("/?"):
+            content = HTML_PAGE.encode("utf-8")
+            self.send_response(HTTPStatus.OK)
+            self.send_header("Content-Type", "text/html; charset=utf-8")
+            self.send_header("Content-Length", str(len(content)))
+            self.send_header("Cache-Control", "no-store")
+            self.end_headers()
+            self.wfile.write(content)
+            return
+        if self.path == "/manifest.json":
+            data = json.dumps(
+                build_group_list(self.directory), ensure_ascii=False, indent=2
+            ).encode("utf-8")
+            self.send_response(HTTPStatus.OK)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.send_header("Content-Length", str(len(data)))
+            self.send_header("Cache-Control", "no-store")
+            self.end_headers()
+            self.wfile.write(data)
+            return
+        if self.path.startswith("/manifest/") and self.path.endswith(".json"):
+            group_enc = self.path[len("/manifest/") : -len(".json")]
+            try:
+                group = unquote(group_enc)
+            except Exception:
+                group = group_enc
+            if not _is_plain_name(group):
+                self.send_error(HTTPStatus.BAD_REQUEST, "Invalid group name")
+                return
+            data = json.dumps(
+                build_group_manifest(self.directory, group), ensure_ascii=False, indent=2
+            ).encode("utf-8")
+            self.send_response(HTTPStatus.OK)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.send_header("Content-Length", str(len(data)))
+            self.send_header("Cache-Control", "no-store")
+            self.end_headers()
+            self.wfile.write(data)
+            return
+        if self.path == "/favicon.ico":
+            self.send_response(HTTPStatus.NO_CONTENT)
+            self.end_headers()
+            return
+        return super().do_GET()
+
+    def list_directory(self, path):
+        self.send_error(HTTPStatus.NOT_FOUND, "Directory listing disabled")
+        return None
+
+
+def gallery():
+    parser = argparse.ArgumentParser(
+        description="Depth Anything 3 Gallery Server (two-level, with pagination)"
+    )
+    parser.add_argument(
+        "-d", "--dir", required=True, help="Gallery root directory (two-level: group/scene)"
+    )
+    parser.add_argument("-p", "--port", type=int, default=8000, help="Port (default 8000)")
+    parser.add_argument("--host", default="127.0.0.1", help="Host address (default 127.0.0.1)")
+    parser.add_argument("--open", action="store_true", help="Open browser after launch")
+    args = parser.parse_args()
+
+    root_dir = os.path.abspath(args.dir)
+    if not os.path.isdir(root_dir):
+        print(f"[error] Directory not found: {root_dir}", file=sys.stderr)
+        sys.exit(1)
+
+    Handler = partial(GalleryHandler, directory=root_dir)
+    server = ThreadingHTTPServer((args.host, args.port), Handler)
+
+    addr = f"http://{args.host}:{args.port}/"
+    print(f"[info] Serving gallery from: {root_dir}")
+    print(f"[info] Open: {addr}")
+
+    if args.open:
+        try:
+            import webbrowser
+
+            webbrowser.open(addr)
+        except Exception as e:
+            print(f"[warn] Failed to open browser: {e}", file=sys.stderr)
+
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        print("\n[info] Shutting down...")
+    finally:
+        server.server_close()
+
+
+def main():
+    """Main entry point for gallery server."""
+    mimetypes.add_type("model/gltf-binary", ".glb")
+    gallery()
+
+
+if __name__ == "__main__":
+    main()

multivue/cameras/source/src/depth_anything_3/services/inference_service.py

@@ -0,0 +1,247 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Unified Inference Service
+Provides unified interface for local and remote inference
+"""
+
+import os
+
+from typing import Any, Dict, List, Optional, Union
+import numpy as np
+import requests
+import typer
+
+from ..api import DepthAnything3
+
+
+class InferenceService:
+    """Unified inference service class"""
+
+    def __init__(self, model_dir: str, device: str = "cuda"):
+        self.model_dir = model_dir
+        self.device = device
+        self.model = None
+
+    def load_model(self):
+        """Load model"""
+        if self.model is None:
+            typer.echo(f"Loading model from {self.model_dir}...")
+            self.model = DepthAnything3.from_pretrained(self.model_dir).to(self.device)
+        return self.model
+
+    def run_local_inference(
+        self,
+        image_paths: List[str],
+        export_dir: str,
+        export_format: str = "mini_npz-glb",
+        process_res: int = 504,
+        process_res_method: str = "upper_bound_resize",
+        export_feat_layers: List[int] = None,
+        extrinsics: Optional[np.ndarray] = None,
+        intrinsics: Optional[np.ndarray] = None,
+        align_to_input_ext_scale: bool = True,
+        use_ray_pose: bool = False,
+        ref_view_strategy: str = "saddle_balanced",
+        conf_thresh_percentile: float = 40.0,
+        num_max_points: int = 1_000_000,
+        show_cameras: bool = True,
+        feat_vis_fps: int = 15,
+    ) -> Any:
+        """Run local inference"""
+        if export_feat_layers is None:
+            export_feat_layers = []
+
+        model = self.load_model()
+
+        # Prepare inference parameters
+        inference_kwargs = {
+            "image": image_paths,
+            "export_dir": export_dir,
+            "export_format": export_format,
+            "process_res": process_res,
+            "process_res_method": process_res_method,
+            "export_feat_layers": export_feat_layers,
+            "align_to_input_ext_scale": align_to_input_ext_scale,
+            "use_ray_pose": use_ray_pose,
+            "ref_view_strategy": ref_view_strategy,
+            "conf_thresh_percentile": conf_thresh_percentile,
+            "num_max_points": num_max_points,
+            "show_cameras": show_cameras,
+            "feat_vis_fps": feat_vis_fps,
+        }
+
+        # Add pose data (if exists)
+        if extrinsics is not None:
+            inference_kwargs["extrinsics"] = extrinsics
+        if intrinsics is not None:
+            inference_kwargs["intrinsics"] = intrinsics
+
+        # Run inference
+        typer.echo(f"Running inference on {len(image_paths)} images...")
+        prediction = model.inference(**inference_kwargs)
+
+        typer.echo(f"Results saved to {export_dir}")
+        typer.echo(f"Export format: {export_format}")
+
+        return prediction
+
+    def run_backend_inference(
+        self,
+        image_paths: List[str],
+        export_dir: str,
+        backend_url: str,
+        export_format: str = "mini_npz-glb",
+        process_res: int = 504,
+        process_res_method: str = "upper_bound_resize",
+        export_feat_layers: List[int] = None,
+        extrinsics: Optional[np.ndarray] = None,
+        intrinsics: Optional[np.ndarray] = None,
+        align_to_input_ext_scale: bool = True,
+        use_ray_pose: bool = False,
+        ref_view_strategy: str = "saddle_balanced",
+        conf_thresh_percentile: float = 40.0,
+        num_max_points: int = 1_000_000,
+        show_cameras: bool = True,
+        feat_vis_fps: int = 15,
+    ) -> Dict[str, Any]:
+        """Run backend inference"""
+        if export_feat_layers is None:
+            export_feat_layers = []
+
+        # Check backend status
+        if not self._check_backend_status(backend_url):
+            raise typer.BadParameter(f"Backend service is not running at {backend_url}")
+
+        # Prepare payload
+        payload = {
+            "image_paths": image_paths,
+            "export_dir": export_dir,
+            "export_format": export_format,
+            "process_res": process_res,
+            "process_res_method": process_res_method,
+            "export_feat_layers": export_feat_layers,
+            "align_to_input_ext_scale": align_to_input_ext_scale,
+            "use_ray_pose": use_ray_pose,
+            "ref_view_strategy": ref_view_strategy,
+            "conf_thresh_percentile": conf_thresh_percentile,
+            "num_max_points": num_max_points,
+            "show_cameras": show_cameras,
+            "feat_vis_fps": feat_vis_fps,
+        }
+
+        # Add pose data (if exists)
+        if extrinsics is not None:
+            payload["extrinsics"] = [ext.astype(np.float64).tolist() for ext in extrinsics]
+        if intrinsics is not None:
+            payload["intrinsics"] = [intr.astype(np.float64).tolist() for intr in intrinsics]
+
+        # Submit task
+        typer.echo("Submitting inference task to backend...")
+        # Sent automatically when the backend was started with a matching --api-key /
+        # DA3_BACKEND_API_KEY, so client and server share the same env-var convention.
+        api_key = os.environ.get("DA3_BACKEND_API_KEY")
+        headers = {"X-API-Key": api_key} if api_key else None
+        try:
+            response = requests.post(
+                f"{backend_url}/inference", json=payload, headers=headers, timeout=30
+            )
+            response.raise_for_status()
+            result = response.json()
+
+            if result["success"]:
+                task_id = result["task_id"]
+                typer.echo("Task submitted successfully!")
+                typer.echo(f"Task ID: {task_id}")
+                typer.echo(f"Results will be saved to: {export_dir}")
+                typer.echo(f"Check backend logs for progress updates with task ID: {task_id}")
+                return result
+            else:
+                raise typer.BadParameter(
+                    f"Backend inference submission failed: {result['message']}"
+                )
+        except requests.exceptions.RequestException as e:
+            raise typer.BadParameter(f"Backend inference submission failed: {e}")
+
+    def _check_backend_status(self, backend_url: str) -> bool:
+        """Check backend status"""
+        try:
+            response = requests.get(f"{backend_url}/status", timeout=5)
+            return response.status_code == 200
+        except Exception:
+            return False
+
+
+def run_inference(
+    image_paths: List[str],
+    export_dir: str,
+    model_dir: str,
+    device: str = "cuda",
+    backend_url: Optional[str] = None,
+    export_format: str = "mini_npz-glb",
+    process_res: int = 504,
+    process_res_method: str = "upper_bound_resize",
+    export_feat_layers: List[int] = None,
+    extrinsics: Optional[np.ndarray] = None,
+    intrinsics: Optional[np.ndarray] = None,
+    align_to_input_ext_scale: bool = True,
+    use_ray_pose: bool = False,
+    ref_view_strategy: str = "saddle_balanced",
+    conf_thresh_percentile: float = 40.0,
+    num_max_points: int = 1_000_000,
+    show_cameras: bool = True,
+    feat_vis_fps: int = 15,
+) -> Union[Any, Dict[str, Any]]:
+    """Unified inference interface"""
+
+    service = InferenceService(model_dir, device)
+
+    if backend_url:
+        return service.run_backend_inference(
+            image_paths=image_paths,
+            export_dir=export_dir,
+            backend_url=backend_url,
+            export_format=export_format,
+            process_res=process_res,
+            process_res_method=process_res_method,
+            export_feat_layers=export_feat_layers,
+            extrinsics=extrinsics,
+            intrinsics=intrinsics,
+            align_to_input_ext_scale=align_to_input_ext_scale,
+            use_ray_pose=use_ray_pose,
+            ref_view_strategy=ref_view_strategy,
+            conf_thresh_percentile=conf_thresh_percentile,
+            num_max_points=num_max_points,
+            show_cameras=show_cameras,
+            feat_vis_fps=feat_vis_fps,
+        )
+    else:
+        return service.run_local_inference(
+            image_paths=image_paths,
+            export_dir=export_dir,
+            export_format=export_format,
+            process_res=process_res,
+            process_res_method=process_res_method,
+            export_feat_layers=export_feat_layers,
+            extrinsics=extrinsics,
+            intrinsics=intrinsics,
+            align_to_input_ext_scale=align_to_input_ext_scale,
+            use_ray_pose=use_ray_pose,
+            ref_view_strategy=ref_view_strategy,
+            conf_thresh_percentile=conf_thresh_percentile,
+            num_max_points=num_max_points,
+            show_cameras=show_cameras,
+            feat_vis_fps=feat_vis_fps,
+        )

multivue/cameras/source/src/depth_anything_3/services/input_handlers.py

@@ -0,0 +1,266 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Input Processing Service
+Handles different types of inputs (image, images, colmap, video)
+"""
+
+import glob
+import os
+from typing import List, Tuple
+import cv2
+import numpy as np
+import typer
+
+from ..utils.read_write_model import read_model
+
+
+class InputHandler:
+    """Base input handler class"""
+
+    @staticmethod
+    def validate_path(path: str, path_type: str = "file") -> str:
+        """Validate path"""
+        if not os.path.exists(path):
+            raise typer.BadParameter(f"{path_type} not found: {path}")
+        return path
+
+    @staticmethod
+    def handle_export_dir(export_dir: str, auto_cleanup: bool = False) -> str:
+        """Handle export directory"""
+        if os.path.exists(export_dir):
+            if auto_cleanup:
+                typer.echo(f"Auto-cleaning existing export directory: {export_dir}")
+                import shutil
+
+                shutil.rmtree(export_dir)
+                os.makedirs(export_dir, exist_ok=True)
+            else:
+                typer.echo(f"Export directory '{export_dir}' already exists.")
+                if typer.confirm("Do you want to clean it and continue?"):
+                    import shutil
+
+                    shutil.rmtree(export_dir)
+                    os.makedirs(export_dir, exist_ok=True)
+                    typer.echo(f"Cleaned export directory: {export_dir}")
+                else:
+                    typer.echo("Operation cancelled.")
+                    raise typer.Exit(0)
+        else:
+            os.makedirs(export_dir, exist_ok=True)
+        return export_dir
+
+
+class ImageHandler(InputHandler):
+    """Single image handler"""
+
+    @staticmethod
+    def process(image_path: str) -> List[str]:
+        """Process single image"""
+        InputHandler.validate_path(image_path, "Image file")
+        return [image_path]
+
+
+class ImagesHandler(InputHandler):
+    """Image directory handler"""
+
+    @staticmethod
+    def process(images_dir: str, image_extensions: str = "png,jpg,jpeg") -> List[str]:
+        """Process image directory"""
+        InputHandler.validate_path(images_dir, "Images directory")
+
+        # Parse extensions
+        extensions = [ext.strip().lower() for ext in image_extensions.split(",")]
+        extensions = [ext if ext.startswith(".") else f".{ext}" for ext in extensions]
+
+        # Find image files
+        image_files = []
+        for ext in extensions:
+            pattern = f"*{ext}"
+            image_files.extend(glob.glob(os.path.join(images_dir, pattern)))
+            image_files.extend(glob.glob(os.path.join(images_dir, pattern.upper())))
+
+        image_files = sorted(list(set(image_files)))  # Remove duplicates and sort
+
+        if not image_files:
+            raise typer.BadParameter(
+                f"No image files found in {images_dir} with extensions: {extensions}"
+            )
+
+        typer.echo(f"Found {len(image_files)} images to process")
+        return image_files
+
+
+class ColmapHandler(InputHandler):
+    """COLMAP data handler"""
+
+    @staticmethod
+    def process(
+        colmap_dir: str, sparse_subdir: str = ""
+    ) -> Tuple[List[str], np.ndarray, np.ndarray]:
+        """Process COLMAP data"""
+        InputHandler.validate_path(colmap_dir, "COLMAP directory")
+
+        # Build paths
+        images_dir = os.path.join(colmap_dir, "images")
+        if sparse_subdir:
+            sparse_dir = os.path.join(colmap_dir, "sparse", sparse_subdir)
+        else:
+            sparse_dir = os.path.join(colmap_dir, "sparse")
+
+        InputHandler.validate_path(images_dir, "Images directory")
+        InputHandler.validate_path(sparse_dir, "Sparse reconstruction directory")
+
+        # Load COLMAP data
+        typer.echo("Loading COLMAP reconstruction data...")
+        try:
+            cameras, images, points3D = read_model(sparse_dir)
+
+            typer.echo(
+                f"Loaded COLMAP data: {len(cameras)} cameras, {len(images)} images, "
+                f"{len(points3D)} 3D points."
+            )
+
+            # Get image files and pose data
+            image_files = []
+            extrinsics = []
+            intrinsics = []
+
+            for image_id, image_data in images.items():
+                image_name = image_data.name
+                image_path = os.path.join(images_dir, image_name)
+
+                if os.path.exists(image_path):
+                    image_files.append(image_path)
+
+                    # Get camera parameters
+                    camera = cameras[image_data.camera_id]
+
+                    # Convert quaternion to rotation matrix
+                    R = image_data.qvec2rotmat()
+                    t = image_data.tvec
+
+                    # Create extrinsic matrix (world to camera)
+                    extrinsic = np.eye(4)
+                    extrinsic[:3, :3] = R
+                    extrinsic[:3, 3] = t
+                    extrinsics.append(extrinsic)
+
+                    # Create intrinsic matrix
+                    if camera.model == "PINHOLE":
+                        fx, fy, cx, cy = camera.params
+                    elif camera.model == "SIMPLE_PINHOLE":
+                        f, cx, cy = camera.params
+                        fx = fy = f
+                    else:
+                        # For other models, use basic pinhole approximation
+                        fx = fy = camera.params[0] if len(camera.params) > 0 else 1000
+                        cx = camera.width / 2
+                        cy = camera.height / 2
+
+                    intrinsic = np.array([[fx, 0, cx], [0, fy, cy], [0, 0, 1]])
+                    intrinsics.append(intrinsic)
+
+            if not image_files:
+                raise typer.BadParameter("No valid images found in COLMAP data")
+
+            typer.echo(f"Found {len(image_files)} valid images with pose data")
+
+            return image_files, np.array(extrinsics), np.array(intrinsics)
+
+        except Exception as e:
+            raise typer.BadParameter(f"Failed to load COLMAP data: {e}")
+
+
+class VideoHandler(InputHandler):
+    """Video handler"""
+
+    @staticmethod
+    def process(video_path: str, output_dir: str, fps: float = 1.0) -> List[str]:
+        """Process video, extract frames"""
+        InputHandler.validate_path(video_path, "Video file")
+
+        cap = cv2.VideoCapture(video_path)
+        if not cap.isOpened():
+            raise typer.BadParameter(f"Cannot open video: {video_path}")
+
+        # Get video properties
+        video_fps = cap.get(cv2.CAP_PROP_FPS)
+        total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+        duration = total_frames / video_fps
+
+        # Calculate frame interval (ensure at least 1)
+        frame_interval = max(1, int(video_fps / fps))
+        actual_fps = video_fps / frame_interval
+
+        typer.echo(f"Video FPS: {video_fps:.2f}, Duration: {duration:.2f}s")
+
+        # Warn if requested FPS is higher than video FPS
+        if fps > video_fps:
+            typer.echo(
+                f"⚠️  Warning: Requested sampling FPS ({fps:.2f}) exceeds video FPS ({video_fps:.2f})",  # noqa: E501
+                err=True,
+            )
+            typer.echo(
+                f"⚠️  Using maximum available FPS: {actual_fps:.2f} (extracting every frame)",
+                err=True,
+            )
+
+        typer.echo(f"Extracting frames at {actual_fps:.2f} FPS (every {frame_interval} frame(s))")
+
+        # Create output directory
+        frames_dir = os.path.join(output_dir, "input_images")
+        os.makedirs(frames_dir, exist_ok=True)
+
+        frame_count = 0
+        saved_count = 0
+
+        while True:
+            ret, frame = cap.read()
+            if not ret:
+                break
+
+            if frame_count % frame_interval == 0:
+                frame_path = os.path.join(frames_dir, f"{saved_count:06d}.png")
+                cv2.imwrite(frame_path, frame)
+                saved_count += 1
+
+            frame_count += 1
+
+        cap.release()
+        typer.echo(f"Extracted {saved_count} frames to {frames_dir}")
+
+        # Get frame file list
+        frame_files = sorted(
+            [f for f in os.listdir(frames_dir) if f.endswith((".png", ".jpg", ".jpeg"))]
+        )
+        if not frame_files:
+            raise typer.BadParameter("No frames extracted from video")
+
+        return [os.path.join(frames_dir, f) for f in frame_files]
+
+
+def parse_export_feat(export_feat_str: str) -> List[int]:
+    """Parse export_feat parameter"""
+    if not export_feat_str:
+        return []
+
+    try:
+        return [int(x.strip()) for x in export_feat_str.split(",") if x.strip()]
+    except ValueError:
+        raise typer.BadParameter(
+            f"Invalid export_feat format: {export_feat_str}. "
+            "Use comma-separated integers like '0,1,2'"
+        )

multivue/cameras/source/src/depth_anything_3/specs.py

@@ -0,0 +1,45 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Optional
+import numpy as np
+import torch
+
+
+@dataclass
+class Gaussians:
+    """3DGS parameters, all in world space"""
+
+    means: torch.Tensor  # world points, "batch gaussian dim"
+    scales: torch.Tensor  # scales_std, "batch gaussian 3"
+    rotations: torch.Tensor  # world_quat_wxyz, "batch gaussian 4"
+    harmonics: torch.Tensor  # world SH, "batch gaussian 3 d_sh"
+    opacities: torch.Tensor  # opacity | opacity SH, "batch gaussian" | "batch gaussian 1 d_sh"
+
+
+@dataclass
+class Prediction:
+    depth: np.ndarray  # N, H, W
+    is_metric: int
+    sky: np.ndarray | None = None  # N, H, W
+    conf: np.ndarray | None = None  # N, H, W
+    extrinsics: np.ndarray | None = None  # N, 4, 4
+    intrinsics: np.ndarray | None = None  # N, 3, 3
+    processed_images: np.ndarray | None = None  # N, H, W, 3 - processed images for visualization
+    gaussians: Gaussians | None = None  # 3D gaussians
+    aux: dict[str, Any] = None  #
+    scale_factor: Optional[float] = None  # metric scale

multivue/cameras/source/src/depth_anything_3/utils/alignment.py

@@ -0,0 +1,163 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Alignment utilities for depth estimation and metric scaling.
+"""
+
+from typing import Tuple
+import torch
+
+
+def least_squares_scale_scalar(
+    a: torch.Tensor, b: torch.Tensor, eps: float = 1e-12
+) -> torch.Tensor:
+    """
+    Compute least squares scale factor s such that a ≈ s * b.
+
+    Args:
+        a: First tensor
+        b: Second tensor
+        eps: Small epsilon for numerical stability
+
+    Returns:
+        Scalar tensor containing the scale factor
+
+    Raises:
+        ValueError: If tensors have mismatched shapes or devices
+        TypeError: If tensors are not floating point
+    """
+    if a.shape != b.shape:
+        raise ValueError(f"Shape mismatch: {a.shape} vs {b.shape}")
+    if a.device != b.device:
+        raise ValueError(f"Device mismatch: {a.device} vs {b.device}")
+    if not a.is_floating_point() or not b.is_floating_point():
+        raise TypeError("Tensors must be floating point type")
+
+    # Compute dot products for least squares solution
+    num = torch.dot(a.reshape(-1), b.reshape(-1))
+    den = torch.dot(b.reshape(-1), b.reshape(-1)).clamp_min(eps)
+    return num / den
+
+
+def compute_sky_mask(sky_prediction: torch.Tensor, threshold: float = 0.3) -> torch.Tensor:
+    """
+    Compute non-sky mask from sky prediction.
+
+    Args:
+        sky_prediction: Sky prediction tensor
+        threshold: Threshold for sky classification
+
+    Returns:
+        Boolean mask where True indicates non-sky regions
+    """
+    return sky_prediction < threshold
+
+
+def compute_alignment_mask(
+    depth_conf: torch.Tensor,
+    non_sky_mask: torch.Tensor,
+    depth: torch.Tensor,
+    metric_depth: torch.Tensor,
+    median_conf: torch.Tensor,
+    min_depth_threshold: float = 1e-3,
+    min_metric_depth_threshold: float = 1e-2,
+) -> torch.Tensor:
+    """
+    Compute mask for depth alignment based on confidence and depth thresholds.
+
+    Args:
+        depth_conf: Depth confidence tensor
+        non_sky_mask: Non-sky region mask
+        depth: Predicted depth tensor
+        metric_depth: Metric depth tensor
+        median_conf: Median confidence threshold
+        min_depth_threshold: Minimum depth threshold
+        min_metric_depth_threshold: Minimum metric depth threshold
+
+    Returns:
+        Boolean mask for valid alignment regions
+    """
+    return (
+        (depth_conf >= median_conf)
+        & non_sky_mask
+        & (metric_depth > min_metric_depth_threshold)
+        & (depth > min_depth_threshold)
+    )
+
+
+def sample_tensor_for_quantile(tensor: torch.Tensor, max_samples: int = 100000) -> torch.Tensor:
+    """
+    Sample tensor elements for quantile computation to reduce memory usage.
+
+    Args:
+        tensor: Input tensor to sample
+        max_samples: Maximum number of samples to take
+
+    Returns:
+        Sampled tensor
+    """
+    if tensor.numel() <= max_samples:
+        return tensor
+
+    idx = torch.randperm(tensor.numel(), device=tensor.device)[:max_samples]
+    return tensor.flatten()[idx]
+
+
+def apply_metric_scaling(
+    depth: torch.Tensor, intrinsics: torch.Tensor, scale_factor: float = 300.0
+) -> torch.Tensor:
+    """
+    Apply metric scaling to depth based on camera intrinsics.
+
+    Args:
+        depth: Input depth tensor
+        intrinsics: Camera intrinsics tensor
+        scale_factor: Scaling factor for metric conversion
+
+    Returns:
+        Scaled depth tensor
+    """
+    focal_length = (intrinsics[:, :, 0, 0] + intrinsics[:, :, 1, 1]) / 2
+    return depth * (focal_length[:, :, None, None] / scale_factor)
+
+
+def set_sky_regions_to_max_depth(
+    depth: torch.Tensor,
+    depth_conf: torch.Tensor,
+    non_sky_mask: torch.Tensor,
+    max_depth: float = 200.0,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    """
+    Set sky regions to maximum depth and high confidence.
+
+    Args:
+        depth: Depth tensor
+        depth_conf: Depth confidence tensor
+        non_sky_mask: Non-sky region mask
+        max_depth: Maximum depth value for sky regions
+
+    Returns:
+        Tuple of (updated_depth, updated_depth_conf)
+    """
+    depth = depth.clone()
+
+    # Set sky regions to max depth and high confidence
+    depth[~non_sky_mask] = max_depth
+    if depth_conf is not None:
+        depth_conf = depth_conf.clone()
+        depth_conf[~non_sky_mask] = 1.0
+        return depth, depth_conf
+    else:
+        return depth, None

multivue/cameras/source/src/depth_anything_3/utils/api_helpers.py

@@ -0,0 +1,58 @@
+import argparse
+
+
+def parse_scalar(s):
+    if not isinstance(s, str):
+        return s
+    t = s.strip()
+    l = t.lower()
+    if l == "true":
+        return True
+    if l == "false":
+        return False
+    if l in ("none", "null"):
+        return None
+    try:
+        return int(t, 10)
+    except Exception:
+        pass
+    try:
+        return float(t)
+    except Exception:
+        return s
+
+
+def fn_kv_csv(s: str) -> dict[str, dict[str, object]]:
+    """
+    Parse a string of comma-separated triplets: fn:key:value
+
+    Returns:
+        dict[fn_name] -> dict[key] = parsed_value
+
+    Example:
+        "fn1:width:1920,fn1:height:1080,fn2:quality:0.8"
+        -> {"fn1": {"width": 1920, "height": 1080}, "fn2": {"quality": 0.8}}
+    """
+    result: dict[str, dict[str, object]] = {}
+    if not s:
+        return result
+
+    for item in s.split(","):
+        if not item:
+            continue
+        parts = item.split(":", 2)  # allow value to contain ":" beyond first two separators
+        if len(parts) < 3:
+            raise argparse.ArgumentTypeError(f"Bad item '{item}', expected FN:KEY:VALUE")
+        fn, key, raw_val = parts[0], parts[1], parts[2]
+        # If you need to allow colons in values, join leftover parts:
+        # fn, key, raw_val = parts[0], parts[1], ":".join(parts[2:])
+
+        if not fn:
+            raise argparse.ArgumentTypeError(f"Bad item '{item}': empty function name")
+        if not key:
+            raise argparse.ArgumentTypeError(f"Bad item '{item}': empty key")
+
+        val = parse_scalar(raw_val)
+        bucket = result.setdefault(fn, {})
+        bucket[key] = val
+    return result

multivue/cameras/source/src/depth_anything_3/utils/camera_trj_helpers.py

@@ -0,0 +1,479 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import cv2
+import numpy as np
+import torch
+import torch.nn.functional as F
+from einops import einsum, rearrange, reduce
+
+try:
+    from scipy.spatial.transform import Rotation as R
+except ImportError:
+    from depth_anything_3.utils.logger import logger
+
+    logger.warn("Dependency 'scipy' not found. Required for interpolating camera trajectory.")
+
+from depth_anything_3.utils.geometry import as_homogeneous
+
+
+@torch.no_grad()
+def render_stabilization_path(poses, k_size=45):
+    """Rendering stabilized camera path.
+    poses: [batch, 4, 4] or [batch, 3, 4],
+    return:
+        smooth path: [batch 4 4]"""
+    num_frames = poses.shape[0]
+    device = poses.device
+    dtype = poses.dtype
+
+    # Early exit for trivial cases
+    if num_frames <= 1:
+        return as_homogeneous(poses)
+
+    # Make k_size safe: positive odd and not larger than num_frames
+    # 1) Ensure odd
+    if k_size < 1:
+        k_size = 1
+    if k_size % 2 == 0:
+        k_size += 1
+    # 2) Cap to num_frames (keep odd)
+    max_odd = num_frames if (num_frames % 2 == 1) else (num_frames - 1)
+    if max_odd < 1:
+        max_odd = 1  # covers num_frames == 0 theoretically
+    k_size = min(k_size, max_odd)
+    # 3) enforce a minimum of 3 when possible (for better smoothing)
+    if num_frames >= 3 and k_size < 3:
+        k_size = 3
+
+    input_poses = []
+    for i in range(num_frames):
+        input_poses.append(
+            torch.cat([poses[i, :3, 0:1], poses[i, :3, 1:2], poses[i, :3, 3:4]], dim=-1)
+        )
+    input_poses = torch.stack(input_poses)  # (num_frames, 3, 3)
+
+    # Prepare Gaussian kernel
+    gaussian_kernel = cv2.getGaussianKernel(ksize=k_size, sigma=-1).astype(np.float32).squeeze()
+    gaussian_kernel = torch.tensor(gaussian_kernel, dtype=dtype, device=device).view(1, 1, -1)
+    pad = k_size // 2
+
+    output_vectors = []
+    for idx in range(3):  # For r1, r2, t
+        vec = (
+            input_poses[:, :, idx].T.unsqueeze(0).unsqueeze(0)
+        )  # (1, 1, 3, num_frames) -> (1, 1, 3, num_frames)
+        # But actually, we want (batch=3, channel=1, width=num_frames)
+        # So:
+        vec = input_poses[:, :, idx].T.unsqueeze(1)  # (3, 1, num_frames)
+        vec_padded = F.pad(vec, (pad, pad), mode="reflect")
+        filtered = F.conv1d(vec_padded, gaussian_kernel)
+        output_vectors.append(filtered.squeeze(1).T)  # (num_frames, 3)
+
+    output_r1, output_r2, output_t = output_vectors  # Each is (num_frames, 3)
+
+    # Normalize r1 and r2
+    output_r1 = output_r1 / output_r1.norm(dim=-1, keepdim=True)
+    output_r2 = output_r2 / output_r2.norm(dim=-1, keepdim=True)
+
+    output_poses = []
+    for i in range(num_frames):
+        output_r3 = torch.linalg.cross(output_r1[i], output_r2[i])
+        render_pose = torch.cat(
+            [
+                output_r1[i].unsqueeze(-1),
+                output_r2[i].unsqueeze(-1),
+                output_r3.unsqueeze(-1),
+                output_t[i].unsqueeze(-1),
+            ],
+            dim=-1,
+        )
+        output_poses.append(render_pose[:3, :])
+    output_poses = as_homogeneous(torch.stack(output_poses, dim=0))
+
+    return output_poses
+
+
+@torch.no_grad()
+def render_wander_path(
+    cam2world: torch.Tensor,
+    intrinsic: torch.Tensor,
+    h: int,
+    w: int,
+    num_frames: int = 120,
+    max_disp: float = 48.0,
+):
+    device, dtype = cam2world.device, cam2world.dtype
+    fx = intrinsic[0, 0] * w
+    r = max_disp / fx
+    th = torch.linspace(0, 2.0 * torch.pi, steps=num_frames, device=device, dtype=dtype)
+    x = r * torch.sin(th)
+    yz = r * torch.cos(th) / 3.0
+    T = torch.eye(4, device=device, dtype=dtype).unsqueeze(0).repeat(num_frames, 1, 1)
+    T[:, :3, 3] = torch.stack([x, yz, yz], dim=-1) * -1.0
+    c2ws = cam2world.unsqueeze(0) @ T
+    # Start at reference pose and end back at reference pose
+    c2ws = torch.cat([cam2world.unsqueeze(0), c2ws, cam2world.unsqueeze(0)], dim=0)
+    Ks = intrinsic.unsqueeze(0).repeat(c2ws.shape[0], 1, 1)
+    return c2ws, Ks
+
+
+@torch.no_grad()
+def render_dolly_zoom_path(
+    cam2world: torch.Tensor,
+    intrinsic: torch.Tensor,
+    h: int,
+    w: int,
+    num_frames: int = 120,
+    max_disp: float = 0.1,
+    D_focus: float = 10.0,
+):
+    device, dtype = cam2world.device, cam2world.dtype
+    fx0, fy0 = intrinsic[0, 0] * w, intrinsic[1, 1] * h
+    t = torch.linspace(0.0, 2.0, steps=num_frames, device=device, dtype=dtype)
+    z = 0.5 * (1.0 - torch.cos(torch.pi * t)) * max_disp
+    T = torch.eye(4, device=device, dtype=dtype).unsqueeze(0).repeat(num_frames, 1, 1)
+    T[:, 2, 3] = -z
+    c2ws = cam2world.unsqueeze(0) @ T
+    Df = torch.as_tensor(D_focus, device=device, dtype=dtype)
+    scale = (Df / (Df + z)).clamp(min=1e-6)
+    Ks = intrinsic.unsqueeze(0).repeat(num_frames, 1, 1)
+    Ks[:, 0, 0] = (fx0 * scale) / w
+    Ks[:, 1, 1] = (fy0 * scale) / h
+    return c2ws, Ks
+
+
+@torch.no_grad()
+def interpolate_intrinsics(
+    initial: torch.Tensor,  # "*#batch 3 3"
+    final: torch.Tensor,  # "*#batch 3 3"
+    t: torch.Tensor,  # " time_step"
+) -> torch.Tensor:  # "*batch time_step 3 3"
+    initial = rearrange(initial, "... i j -> ... () i j")
+    final = rearrange(final, "... i j -> ... () i j")
+    t = rearrange(t, "t -> t () ()")
+    return initial + (final - initial) * t
+
+
+def intersect_rays(
+    a_origins: torch.Tensor,  # "*#batch dim"
+    a_directions: torch.Tensor,  # "*#batch dim"
+    b_origins: torch.Tensor,  # "*#batch dim"
+    b_directions: torch.Tensor,  # "*#batch dim"
+) -> torch.Tensor:  # "*batch dim"
+    """Compute the least-squares intersection of rays. Uses the math from here:
+    https://math.stackexchange.com/a/1762491/286022
+    """
+
+    # Broadcast and stack the tensors.
+    a_origins, a_directions, b_origins, b_directions = torch.broadcast_tensors(
+        a_origins, a_directions, b_origins, b_directions
+    )
+    origins = torch.stack((a_origins, b_origins), dim=-2)
+    directions = torch.stack((a_directions, b_directions), dim=-2)
+
+    # Compute n_i * n_i^T - eye(3) from the equation.
+    n = einsum(directions, directions, "... n i, ... n j -> ... n i j")
+    n = n - torch.eye(3, dtype=origins.dtype, device=origins.device)
+
+    # Compute the left-hand side of the equation.
+    lhs = reduce(n, "... n i j -> ... i j", "sum")
+
+    # Compute the right-hand side of the equation.
+    rhs = einsum(n, origins, "... n i j, ... n j -> ... n i")
+    rhs = reduce(rhs, "... n i -> ... i", "sum")
+
+    # Left-matrix-multiply both sides by the inverse of lhs to find p.
+    return torch.linalg.lstsq(lhs, rhs).solution
+
+
+def normalize(a: torch.Tensor) -> torch.Tensor:  # "*#batch dim" -> "*#batch dim"
+    return a / a.norm(dim=-1, keepdim=True)
+
+
+def generate_coordinate_frame(
+    y: torch.Tensor,  # "*#batch 3"
+    z: torch.Tensor,  # "*#batch 3"
+) -> torch.Tensor:  # "*batch 3 3"
+    """Generate a coordinate frame given perpendicular, unit-length Y and Z vectors."""
+    y, z = torch.broadcast_tensors(y, z)
+    return torch.stack([y.cross(z, dim=-1), y, z], dim=-1)
+
+
+def generate_rotation_coordinate_frame(
+    a: torch.Tensor,  # "*#batch 3"
+    b: torch.Tensor,  # "*#batch 3"
+    eps: float = 1e-4,
+) -> torch.Tensor:  # "*batch 3 3"
+    """Generate a coordinate frame where the Y direction is normal to the plane defined
+    by unit vectors a and b. The other axes are arbitrary."""
+    device = a.device
+
+    # Replace every entry in b that's parallel to the corresponding entry in a with an
+    # arbitrary vector.
+    b = b.detach().clone()
+    parallel = (einsum(a, b, "... i, ... i -> ...").abs() - 1).abs() < eps
+    b[parallel] = torch.tensor([0, 0, 1], dtype=b.dtype, device=device)
+    parallel = (einsum(a, b, "... i, ... i -> ...").abs() - 1).abs() < eps
+    b[parallel] = torch.tensor([0, 1, 0], dtype=b.dtype, device=device)
+
+    # Generate the coordinate frame. The initial cross product defines the plane.
+    return generate_coordinate_frame(normalize(torch.linalg.cross(a, b)), a)
+
+
+def matrix_to_euler(
+    rotations: torch.Tensor,  # "*batch 3 3"
+    pattern: str,
+) -> torch.Tensor:  # "*batch 3"
+    *batch, _, _ = rotations.shape
+    rotations = rotations.reshape(-1, 3, 3)
+    angles_np = R.from_matrix(rotations.detach().cpu().numpy()).as_euler(pattern)
+    rotations = torch.tensor(angles_np, dtype=rotations.dtype, device=rotations.device)
+    return rotations.reshape(*batch, 3)
+
+
+def euler_to_matrix(
+    rotations: torch.Tensor,  # "*batch 3"
+    pattern: str,
+) -> torch.Tensor:  # "*batch 3 3"
+    *batch, _ = rotations.shape
+    rotations = rotations.reshape(-1, 3)
+    matrix_np = R.from_euler(pattern, rotations.detach().cpu().numpy()).as_matrix()
+    rotations = torch.tensor(matrix_np, dtype=rotations.dtype, device=rotations.device)
+    return rotations.reshape(*batch, 3, 3)
+
+
+def extrinsics_to_pivot_parameters(
+    extrinsics: torch.Tensor,  # "*#batch 4 4"
+    pivot_coordinate_frame: torch.Tensor,  # "*#batch 3 3"
+    pivot_point: torch.Tensor,  # "*#batch 3"
+) -> torch.Tensor:  # "*batch 5"
+    """Convert the extrinsics to a representation with 5 degrees of freedom:
+    1. Distance from pivot point in the "X" (look cross pivot axis) direction.
+    2. Distance from pivot point in the "Y" (pivot axis) direction.
+    3. Distance from pivot point in the Z (look) direction
+    4. Angle in plane
+    5. Twist (rotation not in plane)
+    """
+
+    # The pivot coordinate frame's Z axis is normal to the plane.
+    pivot_axis = pivot_coordinate_frame[..., :, 1]
+
+    # Compute the translation elements of the pivot parametrization.
+    translation_frame = generate_coordinate_frame(pivot_axis, extrinsics[..., :3, 2])
+    origin = extrinsics[..., :3, 3]
+    delta = pivot_point - origin
+    translation = einsum(translation_frame, delta, "... i j, ... i -> ... j")
+
+    # Add the rotation elements of the pivot parametrization.
+    inverted = pivot_coordinate_frame.inverse() @ extrinsics[..., :3, :3]
+    y, _, z = matrix_to_euler(inverted, "YXZ").unbind(dim=-1)
+
+    return torch.cat([translation, y[..., None], z[..., None]], dim=-1)
+
+
+def pivot_parameters_to_extrinsics(
+    parameters: torch.Tensor,  # "*#batch 5"
+    pivot_coordinate_frame: torch.Tensor,  # "*#batch 3 3"
+    pivot_point: torch.Tensor,  # "*#batch 3"
+) -> torch.Tensor:  # "*batch 4 4"
+    translation, y, z = parameters.split((3, 1, 1), dim=-1)
+
+    euler = torch.cat((y, torch.zeros_like(y), z), dim=-1)
+    rotation = pivot_coordinate_frame @ euler_to_matrix(euler, "YXZ")
+
+    # The pivot coordinate frame's Z axis is normal to the plane.
+    pivot_axis = pivot_coordinate_frame[..., :, 1]
+
+    translation_frame = generate_coordinate_frame(pivot_axis, rotation[..., :3, 2])
+    delta = einsum(translation_frame, translation, "... i j, ... j -> ... i")
+    origin = pivot_point - delta
+
+    *batch, _ = origin.shape
+    extrinsics = torch.eye(4, dtype=parameters.dtype, device=parameters.device)
+    extrinsics = extrinsics.broadcast_to((*batch, 4, 4)).clone()
+    extrinsics[..., 3, 3] = 1
+    extrinsics[..., :3, :3] = rotation
+    extrinsics[..., :3, 3] = origin
+    return extrinsics
+
+
+def interpolate_circular(
+    a: torch.Tensor,  # "*#batch"
+    b: torch.Tensor,  # "*#batch"
+    t: torch.Tensor,  # "*#batch"
+) -> torch.Tensor:  # " *batch"
+    a, b, t = torch.broadcast_tensors(a, b, t)
+
+    tau = 2 * torch.pi
+    a = a % tau
+    b = b % tau
+
+    # Consider piecewise edge cases.
+    d = (b - a).abs()
+    a_left = a - tau
+    d_left = (b - a_left).abs()
+    a_right = a + tau
+    d_right = (b - a_right).abs()
+    use_d = (d < d_left) & (d < d_right)
+    use_d_left = (d_left < d_right) & (~use_d)
+    use_d_right = (~use_d) & (~use_d_left)
+
+    result = a + (b - a) * t
+    result[use_d_left] = (a_left + (b - a_left) * t)[use_d_left]
+    result[use_d_right] = (a_right + (b - a_right) * t)[use_d_right]
+
+    return result
+
+
+def interpolate_pivot_parameters(
+    initial: torch.Tensor,  # "*#batch 5"
+    final: torch.Tensor,  # "*#batch 5"
+    t: torch.Tensor,  # " time_step"
+) -> torch.Tensor:  # "*batch time_step 5"
+    initial = rearrange(initial, "... d -> ... () d")
+    final = rearrange(final, "... d -> ... () d")
+    t = rearrange(t, "t -> t ()")
+    ti, ri = initial.split((3, 2), dim=-1)
+    tf, rf = final.split((3, 2), dim=-1)
+
+    t_lerp = ti + (tf - ti) * t
+    r_lerp = interpolate_circular(ri, rf, t)
+
+    return torch.cat((t_lerp, r_lerp), dim=-1)
+
+
+@torch.no_grad()
+def interpolate_extrinsics(
+    initial: torch.Tensor,  # "*#batch 4 4"
+    final: torch.Tensor,  # "*#batch 4 4"
+    t: torch.Tensor,  # " time_step"
+    eps: float = 1e-4,
+) -> torch.Tensor:  # "*batch time_step 4 4"
+    """Interpolate extrinsics by rotating around their "focus point," which is the
+    least-squares intersection between the look vectors of the initial and final
+    extrinsics.
+    """
+
+    initial = initial.type(torch.float64)
+    final = final.type(torch.float64)
+    t = t.type(torch.float64)
+
+    # Based on the dot product between the look vectors, pick from one of two cases:
+    # 1. Look vectors are parallel: interpolate about their origins' midpoint.
+    # 3. Look vectors aren't parallel: interpolate about their focus point.
+    initial_look = initial[..., :3, 2]
+    final_look = final[..., :3, 2]
+    dot_products = einsum(initial_look, final_look, "... i, ... i -> ...")
+    parallel_mask = (dot_products.abs() - 1).abs() < eps
+
+    # Pick focus points.
+    initial_origin = initial[..., :3, 3]
+    final_origin = final[..., :3, 3]
+    pivot_point = 0.5 * (initial_origin + final_origin)
+    pivot_point[~parallel_mask] = intersect_rays(
+        initial_origin[~parallel_mask],
+        initial_look[~parallel_mask],
+        final_origin[~parallel_mask],
+        final_look[~parallel_mask],
+    )
+
+    # Convert to pivot parameters.
+    pivot_frame = generate_rotation_coordinate_frame(initial_look, final_look, eps=eps)
+    initial_params = extrinsics_to_pivot_parameters(initial, pivot_frame, pivot_point)
+    final_params = extrinsics_to_pivot_parameters(final, pivot_frame, pivot_point)
+
+    # Interpolate the pivot parameters.
+    interpolated_params = interpolate_pivot_parameters(initial_params, final_params, t)
+
+    # Convert back.
+    return pivot_parameters_to_extrinsics(
+        interpolated_params.type(torch.float32),
+        rearrange(pivot_frame, "... i j -> ... () i j").type(torch.float32),
+        rearrange(pivot_point, "... xyz -> ... () xyz").type(torch.float32),
+    )
+
+
+@torch.no_grad()
+def generate_wobble_transformation(
+    radius: torch.Tensor,  # "*#batch"
+    t: torch.Tensor,  # " time_step"
+    num_rotations: int = 1,
+    scale_radius_with_t: bool = True,
+) -> torch.Tensor:  # "*batch time_step 4 4"]:
+    # Generate a translation in the image plane.
+    tf = torch.eye(4, dtype=torch.float32, device=t.device)
+    tf = tf.broadcast_to((*radius.shape, t.shape[0], 4, 4)).clone()
+    radius = radius[..., None]
+    if scale_radius_with_t:
+        radius = radius * t
+    tf[..., 0, 3] = torch.sin(2 * torch.pi * num_rotations * t) * radius
+    tf[..., 1, 3] = -torch.cos(2 * torch.pi * num_rotations * t) * radius
+    return tf
+
+
+@torch.no_grad()
+def render_wobble_inter_path(
+    cam2world: torch.Tensor, intr_normed: torch.Tensor, inter_len: int, n_skip: int = 3
+):
+    """
+    cam2world: [batch, 4, 4],
+    intr_normed: [batch, 3, 3]
+    """
+    frame_per_round = n_skip * inter_len
+    num_rotations = 1
+
+    t = torch.linspace(0, 1, frame_per_round, dtype=torch.float32, device=cam2world.device)
+    # t = (torch.cos(torch.pi * (t + 1)) + 1) / 2
+    tgt_c2w_b = []
+    tgt_intr_b = []
+    for b_idx in range(cam2world.shape[0]):
+        tgt_c2w = []
+        tgt_intr = []
+        for cur_idx in range(0, cam2world.shape[1] - n_skip, n_skip):
+            origin_a = cam2world[b_idx, cur_idx, :3, 3]
+            origin_b = cam2world[b_idx, cur_idx + n_skip, :3, 3]
+            delta = (origin_a - origin_b).norm(dim=-1)
+            if cur_idx == 0:
+                delta_prev = delta
+            else:
+                delta = (delta_prev + delta) / 2
+                delta_prev = delta
+            tf = generate_wobble_transformation(
+                radius=delta * 0.5,
+                t=t,
+                num_rotations=num_rotations,
+                scale_radius_with_t=False,
+            )
+            cur_extrs = (
+                interpolate_extrinsics(
+                    cam2world[b_idx, cur_idx],
+                    cam2world[b_idx, cur_idx + n_skip],
+                    t,
+                )
+                @ tf
+            )
+            tgt_c2w.append(cur_extrs[(0 if cur_idx == 0 else 1) :])
+            tgt_intr.append(
+                interpolate_intrinsics(
+                    intr_normed[b_idx, cur_idx],
+                    intr_normed[b_idx, cur_idx + n_skip],
+                    t,
+                )[(0 if cur_idx == 0 else 1) :]
+            )
+        tgt_c2w_b.append(torch.cat(tgt_c2w))
+        tgt_intr_b.append(torch.cat(tgt_intr))
+    tgt_c2w = torch.stack(tgt_c2w_b)  # b v 4 4
+    tgt_intr = torch.stack(tgt_intr_b)  # b v 3 3
+    return tgt_c2w, tgt_intr

multivue/cameras/source/src/depth_anything_3/utils/constants.py

@@ -0,0 +1,270 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DEFAULT_MODEL = "depth-anything/DA3NESTED-GIANT-LARGE-1.1"
+DEFAULT_EXPORT_DIR = "workspace/gallery/scene"
+DEFAULT_GALLERY_DIR = "workspace/gallery"
+DEFAULT_GRADIO_DIR = "workspace/gradio"
+THRESH_FOR_REF_SELECTION = 3
+
+# =============================================================================
+# Benchmark Evaluation Constants
+# =============================================================================
+
+# Default evaluation workspace directory
+DEFAULT_EVAL_WORKSPACE = "workspace/evaluation"
+
+# Default reference view selection strategy for evaluation
+# Use "first" for consistent and reproducible evaluation results
+# Other options: "saddle_balanced", "auto", "mid"
+EVAL_REF_VIEW_STRATEGY = "first"
+
+# -----------------------------------------------------------------------------
+# DTU Dataset Configuration
+# Reference: https://roboimagedata.compute.dtu.dk/
+# Note: DepthAnything3 was never trained on any images from DTU.
+# -----------------------------------------------------------------------------
+
+# Root directory for DTU evaluation data (MVSNet format)
+# Download from: https://drive.google.com/file/d/1rX0EXlUL4prRxrRu2DgLJv2j7-tpUD4D/view
+DTU_EVAL_DATA_ROOT = "workspace/benchmark_dataset/dtu"
+
+# List of DTU evaluation scenes
+DTU_SCENES = [
+    "scan1",
+    "scan4",
+    "scan9",
+    "scan10",
+    "scan11",
+    "scan12",
+    "scan13",
+    "scan15",
+    "scan23",
+    "scan24",
+    "scan29",
+    "scan32",
+    "scan33",
+    "scan34",
+    "scan48",
+    "scan49",
+    "scan62",
+    "scan75",
+    "scan77",
+    "scan110",
+    "scan114",
+    "scan118",
+]
+
+# Point cloud fusion hyperparameters
+DTU_DIST_THRESH = 0.2  # Distance threshold for geometric consistency (mm)
+DTU_NUM_CONSIST = 4  # Minimum number of consistent views for a point
+DTU_MAX_POINTS = 4_000_000  # Maximum points in fused point cloud
+
+# 3D reconstruction evaluation hyperparameters
+DTU_DOWN_DENSE = 0.2  # Downsample density for evaluation (mm)
+DTU_PATCH_SIZE = 60  # Patch size for boundary handling
+DTU_MAX_DIST = 20  # Outlier threshold for accuracy/completeness (mm)
+
+# -----------------------------------------------------------------------------
+# DTU-64 Dataset Configuration (Pose Evaluation Only)
+# This is a subset of DTU with 64 images per scene for pose evaluation.
+# Note: This dataset is ONLY for pose evaluation, not 3D reconstruction.
+# -----------------------------------------------------------------------------
+
+# Root directory for DTU-64 evaluation data
+DTU64_EVAL_DATA_ROOT = "workspace/benchmark_dataset/dtu64"
+DTU64_CAMERA_ROOT = "workspace/benchmark_dataset/dtu64/Cameras"
+
+# List of DTU-64 evaluation scenes (13 scenes)
+DTU64_SCENES = [
+    "scan105",
+    "scan114",
+    "scan118",
+    "scan122",
+    "scan24",
+    "scan37",
+    "scan40",
+    "scan55",
+    "scan63",
+    "scan65",
+    "scan69",
+    "scan83",
+    "scan97",
+]
+
+# -----------------------------------------------------------------------------
+# ETH3D Dataset Configuration
+# Reference: https://www.eth3d.net/
+# High-resolution multi-view stereo benchmark with laser-scanned ground truth.
+# Note: DepthAnything3 was never trained on any images from ETH3D.
+# -----------------------------------------------------------------------------
+
+# Root directory for ETH3D evaluation data
+ETH3D_EVAL_DATA_ROOT = "workspace/benchmark_dataset/eth3d"
+
+# List of ETH3D evaluation scenes (indoor and outdoor)
+ETH3D_SCENES = [
+    "courtyard",
+    "electro",
+    "kicker",
+    "pipes",
+    "relief",
+    # "terrace",  # Excluded: known issues
+    "delivery_area",
+    "facade",
+    # "meadow",   # Excluded: known issues
+    "office",
+    "playground",
+    "relief_2",
+    "terrains",
+]
+
+# Images to filter out (known problematic views per scene)
+ETH3D_FILTER_KEYS = {
+    "delivery_area": ["711.JPG", "712.JPG", "713.JPG", "714.JPG"],
+    "electro": ["9289.JPG", "9290.JPG", "9291.JPG", "9292.JPG", "9293.JPG", "9298.JPG"],
+    "playground": ["587.JPG", "588.JPG", "589.JPG", "590.JPG", "591.JPG", "592.JPG"],
+    "relief": [
+        "427.JPG", "428.JPG", "429.JPG", "430.JPG", "431.JPG", "432.JPG",
+        "433.JPG", "434.JPG", "435.JPG", "436.JPG", "437.JPG", "438.JPG",
+    ],
+    "relief_2": [
+        "458.JPG", "459.JPG", "460.JPG", "461.JPG", "462.JPG", "463.JPG",
+        "464.JPG", "465.JPG", "466.JPG", "467.JPG", "468.JPG",
+    ],
+}
+
+# TSDF fusion hyperparameters (scaled for outdoor scenes)
+ETH3D_VOXEL_LENGTH = 4.0 / 512.0 * 5  # Voxel size for TSDF (meters)
+ETH3D_SDF_TRUNC = 0.04 * 5  # SDF truncation distance (meters)
+ETH3D_MAX_DEPTH = 100000.0  # Maximum depth for integration (effectively no truncation)
+
+# Point cloud sampling
+ETH3D_SAMPLING_NUMBER = 1_000_000  # Number of points to sample from mesh
+
+# 3D reconstruction evaluation hyperparameters
+ETH3D_EVAL_THRESHOLD = 0.05 * 5  # Distance threshold for precision/recall (meters)
+ETH3D_DOWN_SAMPLE = 4.0 / 512.0 * 5  # Voxel size for evaluation downsampling (meters)
+
+
+# ==============================================================================
+# 7Scenes Dataset Configuration
+# ==============================================================================
+# Reference: https://www.microsoft.com/en-us/research/project/rgb-d-dataset-7-scenes/
+# Note: Indoor RGB-D dataset with ground truth poses and meshes.
+
+# Root directory for 7Scenes evaluation data
+SEVENSCENES_EVAL_DATA_ROOT = "workspace/benchmark_dataset/7scenes"
+
+# List of 7Scenes evaluation scenes
+SEVENSCENES_SCENES = [
+    "chess",
+    "fire",
+    "heads",
+    "office",
+    "pumpkin",
+    "redkitchen",
+    "stairs",
+]
+
+# Fixed camera intrinsics for 7Scenes (all images share same intrinsics)
+SEVENSCENES_FX = 585.0
+SEVENSCENES_FY = 585.0
+SEVENSCENES_CX = 320.0
+SEVENSCENES_CY = 240.0
+
+# TSDF fusion hyperparameters (indoor scenes, smaller voxels)
+SEVENSCENES_VOXEL_LENGTH = 4.0 / 512.0  # Voxel size for TSDF (meters)
+SEVENSCENES_SDF_TRUNC = 0.04  # SDF truncation distance (meters)
+SEVENSCENES_MAX_DEPTH = 1000000.0  # Maximum depth for integration (no truncation)
+
+# Point cloud sampling
+SEVENSCENES_SAMPLING_NUMBER = 1_000_000  # Number of points to sample from mesh
+
+# 3D reconstruction evaluation hyperparameters
+SEVENSCENES_EVAL_THRESHOLD = 0.05  # Distance threshold for precision/recall (meters)
+SEVENSCENES_DOWN_SAMPLE = 4.0 / 512.0  # Voxel size for evaluation downsampling (meters)
+
+
+# ==============================================================================
+# ScanNet++ Dataset Configuration
+# ==============================================================================
+# Reference: https://kaldir.vc.in.tum.de/scannetpp/
+# Note: High-quality indoor RGB-D dataset with iPhone and DSLR images.
+
+# Root directory for ScanNet++ evaluation data
+SCANNETPP_EVAL_DATA_ROOT = "workspace/benchmark_dataset/scannetpp"
+
+# List of ScanNet++ evaluation scenes
+SCANNETPP_SCENES = [
+    "09c1414f1b",
+    "1ada7a0617",
+    "40aec5fffa",
+    "3e8bba0176",
+    "acd95847c5",
+    "578511c8a9",
+    "5f99900f09",
+    "c4c04e6d6c",
+    "f3d64c30f8",
+    "7bc286c1b6",
+    "c5439f4607",
+    "286b55a2bf",
+    "fb5a96b1a2",
+    "7831862f02",
+    "38d58a7a31",
+    "bde1e479ad",
+    "9071e139d9",
+    "21d970d8de",
+    "bcd2436daf",
+    "cc5237fd77",
+]
+
+# Input resolution for ScanNet++ (after undistortion and resize)
+SCANNETPP_INPUT_H = 768
+SCANNETPP_INPUT_W = 1024
+
+# TSDF fusion hyperparameters (indoor scenes)
+SCANNETPP_VOXEL_LENGTH = 0.02  # Voxel size for TSDF (meters)
+SCANNETPP_SDF_TRUNC = 0.15  # SDF truncation distance (meters)
+SCANNETPP_MAX_DEPTH = 5.0  # Maximum depth for integration (meters)
+
+# Point cloud sampling
+SCANNETPP_SAMPLING_NUMBER = 1_000_000  # Number of points to sample from mesh
+
+# 3D reconstruction evaluation hyperparameters
+SCANNETPP_EVAL_THRESHOLD = 0.05  # Distance threshold for precision/recall (meters)
+SCANNETPP_DOWN_SAMPLE = 0.02  # Voxel size for evaluation downsampling (meters)
+
+
+# ==============================================================================
+# HiRoom Dataset Configuration
+# ==============================================================================
+# Note: Indoor RGB-D dataset.
+
+# Root directory for HiRoom evaluation data
+HIROOM_EVAL_DATA_ROOT = "workspace/benchmark_dataset/hiroom/data"
+HIROOM_GT_ROOT_PATH = "workspace/benchmark_dataset/hiroom/fused_pcd"
+HIROOM_SCENE_LIST_PATH = "workspace/benchmark_dataset/hiroom/selected_scene_list_val.txt"
+
+# TSDF fusion hyperparameters (indoor scenes)
+HIROOM_VOXEL_LENGTH = 4.0 / 512.0  # Voxel size for TSDF (meters)
+HIROOM_SDF_TRUNC = 0.04  # SDF truncation distance (meters)
+HIROOM_MAX_DEPTH = 10000.0  # Maximum depth for integration (no truncation)
+
+# Point cloud sampling
+HIROOM_SAMPLING_NUMBER = 1_000_000  # Number of points to sample from mesh
+
+# 3D reconstruction evaluation hyperparameters
+HIROOM_EVAL_THRESHOLD = 0.05  # Distance threshold for precision/recall (meters)
+HIROOM_DOWN_SAMPLE = 4.0 / 512.0  # Voxel size for evaluation downsampling (meters)

multivue/cameras/source/src/depth_anything_3/utils/export/__init__.py

@@ -0,0 +1,75 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from depth_anything_3.specs import Prediction
+from depth_anything_3.utils.export.gs import export_to_gs_ply, export_to_gs_video
+
+from .colmap import export_to_colmap
+from .depth_vis import export_to_depth_vis
+from .feat_vis import export_to_feat_vis
+from .glb import export_to_glb
+from .npz import export_to_mini_npz, export_to_npz
+
+# Canonical set of export formats this dispatcher understands. Treat this as the
+# server-trusted enum for any request-facing validation (e.g. the HTTP backend) instead
+# of letting a client-supplied export_format string reach the dispatcher unchecked.
+SUPPORTED_EXPORT_FORMATS = frozenset(
+    {
+        "glb",
+        "mini_npz",
+        "npz",
+        "feat_vis",
+        "depth_vis",
+        "gs_ply",
+        "gs_video",
+        "colmap",
+    }
+)
+
+
+def export(
+    prediction: Prediction,
+    export_format: str,
+    export_dir: str,
+    **kwargs,
+):
+    if "-" in export_format:
+        export_formats = export_format.split("-")
+        for export_format in export_formats:
+            export(prediction, export_format, export_dir, **kwargs)
+        return  # Prevent falling through to single-format handling
+
+    if export_format == "glb":
+        export_to_glb(prediction, export_dir, **kwargs.get(export_format, {}))
+    elif export_format == "mini_npz":
+        export_to_mini_npz(prediction, export_dir)
+    elif export_format == "npz":
+        export_to_npz(prediction, export_dir)
+    elif export_format == "feat_vis":
+        export_to_feat_vis(prediction, export_dir, **kwargs.get(export_format, {}))
+    elif export_format == "depth_vis":
+        export_to_depth_vis(prediction, export_dir)
+    elif export_format == "gs_ply":
+        export_to_gs_ply(prediction, export_dir, **kwargs.get(export_format, {}))
+    elif export_format == "gs_video":
+        export_to_gs_video(prediction, export_dir, **kwargs.get(export_format, {}))
+    elif export_format == "colmap":
+        export_to_colmap(prediction, export_dir, **kwargs.get(export_format, {}))
+    else:
+        raise ValueError(f"Unsupported export format: {export_format}")
+
+
+__all__ = [
+    export,
+]

multivue/cameras/source/src/depth_anything_3/utils/export/colmap.py

@@ -0,0 +1,150 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import pycolmap
+import cv2 as cv
+import numpy as np
+
+from PIL import Image
+
+from depth_anything_3.specs import Prediction
+from depth_anything_3.utils.logger import logger
+
+from .glb import _depths_to_world_points_with_colors
+
+
+def export_to_colmap(
+    prediction: Prediction,
+    export_dir: str,
+    image_paths: list[str],
+    conf_thresh_percentile: float = 40.0,
+    process_res_method: str = "upper_bound_resize",
+) -> None:
+    # 1. Data preparation
+    conf_thresh = np.percentile(prediction.conf, conf_thresh_percentile)
+    points, colors = _depths_to_world_points_with_colors(
+        prediction.depth,
+        prediction.intrinsics,
+        prediction.extrinsics,  # w2c
+        prediction.processed_images,
+        prediction.conf,
+        conf_thresh,
+    )
+    num_points = len(points)
+    logger.info(f"Exporting to COLMAP with {num_points} points")
+    num_frames = len(prediction.processed_images)
+    h, w = prediction.processed_images.shape[1:3]
+    points_xyf = _create_xyf(num_frames, h, w)
+    points_xyf = points_xyf[prediction.conf >= conf_thresh]
+
+    # 2. Set Reconstruction
+    reconstruction = pycolmap.Reconstruction()
+
+    point3d_ids = []
+    for vidx in range(num_points):
+        point3d_id = reconstruction.add_point3D(points[vidx], pycolmap.Track(), colors[vidx])
+        point3d_ids.append(point3d_id)
+
+    for fidx in range(num_frames):
+        orig_w, orig_h = Image.open(image_paths[fidx]).size
+
+        intrinsic = prediction.intrinsics[fidx]
+        if process_res_method.endswith("resize"):
+            intrinsic[:1] *= orig_w / w
+            intrinsic[1:2] *= orig_h / h
+        elif process_res_method == "crop":
+            raise NotImplementedError("COLMAP export for crop method is not implemented")
+        else:
+            raise ValueError(f"Unknown process_res_method: {process_res_method}")
+
+        pycolmap_intri = np.array(
+            [intrinsic[0, 0], intrinsic[1, 1], intrinsic[0, 2], intrinsic[1, 2]]
+        )
+
+        extrinsic = prediction.extrinsics[fidx]
+        cam_from_world = pycolmap.Rigid3d(pycolmap.Rotation3d(extrinsic[:3, :3]), extrinsic[:3, 3])
+
+        # set and add camera
+        camera = pycolmap.Camera()
+        camera.camera_id = fidx + 1
+        camera.model = pycolmap.CameraModelId.PINHOLE
+        camera.width = orig_w
+        camera.height = orig_h
+        camera.params = pycolmap_intri
+        reconstruction.add_camera(camera)
+
+        # set and add rig (from camera)
+        rig = pycolmap.Rig()
+        rig.rig_id = camera.camera_id
+        rig.add_ref_sensor(camera.sensor_id)
+        reconstruction.add_rig(rig)
+
+        # set image
+        image = pycolmap.Image()
+        image.image_id = fidx + 1
+        image.camera_id = camera.camera_id
+
+        # set and add frame (from image)
+        frame = pycolmap.Frame()
+        frame.frame_id = image.image_id
+        frame.rig_id = camera.camera_id
+        frame.add_data_id(image.data_id)
+        frame.rig_from_world = cam_from_world
+        reconstruction.add_frame(frame)
+
+        # set point2d and update track
+        point2d_list = []
+        points_in_frame = points_xyf[:, 2].astype(np.int32) == fidx
+        for vidx in np.where(points_in_frame)[0]:
+            point2d = points_xyf[vidx][:2]
+            point2d[0] *= orig_w / w
+            point2d[1] *= orig_h / h
+            point3d_id = point3d_ids[vidx]
+            point2d_list.append(pycolmap.Point2D(point2d, point3d_id))
+            reconstruction.point3D(point3d_id).track.add_element(
+                image.image_id, len(point2d_list) - 1
+            )
+
+        # set and add image
+        image.frame_id = image.image_id
+        image.name = os.path.basename(image_paths[fidx])
+        image.points2D = pycolmap.Point2DList(point2d_list)
+        reconstruction.add_image(image)
+
+    # 3. Export
+    reconstruction.write(export_dir)
+
+
+def _create_xyf(num_frames, height, width):
+    """
+    Creates a grid of pixel coordinates and frame indices (fidx) for all frames.
+    """
+    # Create coordinate grids for a single frame
+    y_grid, x_grid = np.indices((height, width), dtype=np.int32)
+    x_grid = x_grid[np.newaxis, :, :]
+    y_grid = y_grid[np.newaxis, :, :]
+
+    # Broadcast to all frames
+    x_coords = np.broadcast_to(x_grid, (num_frames, height, width))
+    y_coords = np.broadcast_to(y_grid, (num_frames, height, width))
+
+    # Create frame indices and broadcast
+    f_idx = np.arange(num_frames, dtype=np.int32)[:, np.newaxis, np.newaxis]
+    f_coords = np.broadcast_to(f_idx, (num_frames, height, width))
+
+    # Stack coordinates and frame indices
+    points_xyf = np.stack((x_coords, y_coords, f_coords), axis=-1)
+
+    return points_xyf

multivue/cameras/source/src/depth_anything_3/utils/export/depth_vis.py

@@ -0,0 +1,41 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import imageio
+import numpy as np
+
+from depth_anything_3.specs import Prediction
+from depth_anything_3.utils.visualize import visualize_depth
+
+
+def export_to_depth_vis(
+    prediction: Prediction,
+    export_dir: str,
+):
+    # Use prediction.processed_images, which is already processed image data
+    if prediction.processed_images is None:
+        raise ValueError("prediction.processed_images is required but not available")
+
+    images_u8 = prediction.processed_images  # (N,H,W,3) uint8
+
+    os.makedirs(os.path.join(export_dir, "depth_vis"), exist_ok=True)
+    for idx in range(prediction.depth.shape[0]):
+        depth_vis = visualize_depth(prediction.depth[idx])
+        image_vis = images_u8[idx]
+        depth_vis = depth_vis.astype(np.uint8)
+        image_vis = image_vis.astype(np.uint8)
+        vis_image = np.concatenate([image_vis, depth_vis], axis=1)
+        save_path = os.path.join(export_dir, f"depth_vis/{idx:04d}.jpg")
+        imageio.imwrite(save_path, vis_image, quality=95)

multivue/cameras/source/src/depth_anything_3/utils/export/feat_vis.py

@@ -0,0 +1,75 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import cv2
+import imageio
+import numpy as np
+from tqdm.auto import tqdm
+
+from depth_anything_3.utils.logger import logger
+from depth_anything_3.utils.parallel_utils import async_call
+from depth_anything_3.utils.pca_utils import PCARGBVisualizer
+
+
+@async_call
+def export_to_feat_vis(
+    prediction,
+    export_dir,
+    fps=15,
+):
+    """Export feature visualization with PCA.
+
+    Args:
+        prediction: Model prediction containing feature maps
+        export_dir: Directory to export results
+        fps: Frame rate for output video (default: 15)
+    """
+    out_dir = os.path.join(export_dir, "feat_vis")
+    os.makedirs(out_dir, exist_ok=True)
+
+    images = prediction.processed_images
+    for k, v in prediction.aux.items():
+        if not k.startswith("feat_layer_"):
+            continue
+        os.makedirs(os.path.join(out_dir, k), exist_ok=True)
+        viz = PCARGBVisualizer(basis_mode="fixed", percentile_mode="global", clip_percent=10.0)
+        viz.fit_reference(v)
+        feats_vis = viz.transform_video(v)
+        for idx in tqdm(range(len(feats_vis))):
+            img = images[idx]
+            feat_vis = (feats_vis[idx] * 255).astype(np.uint8)
+            feat_vis = cv2.resize(
+                feat_vis, (img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST
+            )
+            save_path = os.path.join(out_dir, f"{k}/{idx:06d}.jpg")
+            save = np.concatenate([img, feat_vis], axis=1)
+            imageio.imwrite(save_path, save, quality=95)
+        cmd = [
+            "ffmpeg",
+            "-loglevel", "error",
+            "-hide_banner",
+            "-y",
+            "-framerate", str(fps),
+            "-start_number", "0",
+            "-i", os.path.join(out_dir, k, "%06d.jpg"),
+            "-c:v", "libx264",
+            "-pix_fmt", "yuv420p",
+            os.path.join(out_dir, f"{k}.mp4"),
+        ]
+        try:
+            subprocess.run(cmd, shell=False, check=False)
+        except OSError as e:
+            logger.warn(f"Failed to run ffmpeg for feature visualization export: {e}")

multivue/cameras/source/src/depth_anything_3/utils/export/glb.py

@@ -0,0 +1,437 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import os
+import shutil
+import numpy as np
+import trimesh
+
+from depth_anything_3.specs import Prediction
+from depth_anything_3.utils.logger import logger
+
+from .depth_vis import export_to_depth_vis
+
+
+def set_sky_depth(prediction: Prediction, sky_mask: np.ndarray, sky_depth_def: float = 98.0):
+    non_sky_mask = ~sky_mask
+    valid_depth = prediction.depth[non_sky_mask]
+    if valid_depth.size > 0:
+        max_depth = np.percentile(valid_depth, sky_depth_def)
+        prediction.depth[sky_mask] = max_depth
+
+
+def get_conf_thresh(
+    prediction: Prediction,
+    sky_mask: np.ndarray,
+    conf_thresh: float,
+    conf_thresh_percentile: float = 10.0,
+    ensure_thresh_percentile: float = 90.0,
+):
+    if sky_mask is not None and (~sky_mask).sum() > 10:
+        conf_pixels = prediction.conf[~sky_mask]
+    else:
+        conf_pixels = prediction.conf
+    lower = np.percentile(conf_pixels, conf_thresh_percentile)
+    upper = np.percentile(conf_pixels, ensure_thresh_percentile)
+    conf_thresh = min(max(conf_thresh, lower), upper)
+    return conf_thresh
+
+
+def export_to_glb(
+    prediction: Prediction,
+    export_dir: str,
+    num_max_points: int = 1_000_000,
+    conf_thresh: float = 1.05,
+    filter_black_bg: bool = False,
+    filter_white_bg: bool = False,
+    conf_thresh_percentile: float = 40.0,
+    ensure_thresh_percentile: float = 90.0,
+    sky_depth_def: float = 98.0,
+    show_cameras: bool = True,
+    camera_size: float = 0.03,
+    export_depth_vis: bool = True,
+) -> str:
+    """Generate a 3D point cloud and camera wireframes and export them as a ``.glb`` file.
+
+    The function builds a point cloud from the predicted depth maps, aligns it to the
+    first camera in glTF coordinates (X-right, Y-up, Z-backward), optionally draws
+    camera wireframes, and writes the result to ``scene.glb``. Auxiliary assets such as
+    depth visualizations can also be generated alongside the main export.
+
+    Args:
+        prediction: Model prediction containing depth, confidence, intrinsics, extrinsics,
+            and pre-processed images.
+        export_dir: Output directory where the glTF assets will be written.
+        num_max_points: Maximum number of points retained after downsampling.
+        conf_thresh: Base confidence threshold used before percentile adjustments.
+        filter_black_bg: Mark near-black background pixels for removal during confidence filtering.
+        filter_white_bg: Mark near-white background pixels for removal during confidence filtering.
+        conf_thresh_percentile: Lower percentile used when adapting the confidence threshold.
+        ensure_thresh_percentile: Upper percentile clamp for the adaptive threshold.
+        sky_depth_def: Percentile used to fill sky pixels with plausible depth values.
+        show_cameras: Whether to render camera wireframes in the exported scene.
+        camera_size: Relative camera wireframe scale as a fraction of the scene diagonal.
+        export_depth_vis: Whether to export raster depth visualisations alongside the glTF.
+
+    Returns:
+        Path to the exported ``scene.glb`` file.
+    """
+    # 1) Use prediction.processed_images, which is already processed image data
+    assert (
+        prediction.processed_images is not None
+    ), "Export to GLB: prediction.processed_images is required but not available"
+    assert (
+        prediction.depth is not None
+    ), "Export to GLB: prediction.depth is required but not available"
+    assert (
+        prediction.intrinsics is not None
+    ), "Export to GLB: prediction.intrinsics is required but not available"
+    assert (
+        prediction.extrinsics is not None
+    ), "Export to GLB: prediction.extrinsics is required but not available"
+    assert (
+        prediction.conf is not None
+    ), "Export to GLB: prediction.conf is required but not available"
+    logger.info(f"conf_thresh_percentile: {conf_thresh_percentile}")
+    logger.info(f"num max points: {num_max_points}")
+    logger.info(f"Exporting to GLB with num_max_points: {num_max_points}")
+    if prediction.processed_images is None:
+        raise ValueError("prediction.processed_images is required but not available")
+
+    images_u8 = prediction.processed_images  # (N,H,W,3) uint8
+
+    # 2) Sky processing (if sky_mask is provided)
+    if getattr(prediction, "sky_mask", None) is not None:
+        set_sky_depth(prediction, prediction.sky_mask, sky_depth_def)
+
+    # 3) Confidence threshold (if no conf, then no filtering)
+    if filter_black_bg:
+        prediction.conf[(prediction.processed_images < 16).all(axis=-1)] = 1.0
+    if filter_white_bg:
+        prediction.conf[(prediction.processed_images >= 240).all(axis=-1)] = 1.0
+    conf_thr = get_conf_thresh(
+        prediction,
+        getattr(prediction, "sky_mask", None),
+        conf_thresh,
+        conf_thresh_percentile,
+        ensure_thresh_percentile,
+    )
+
+    # 4) Back-project to world coordinates and get colors (world frame)
+    points, colors = _depths_to_world_points_with_colors(
+        prediction.depth,
+        prediction.intrinsics,
+        prediction.extrinsics,  # w2c
+        images_u8,
+        prediction.conf,
+        conf_thr,
+    )
+
+    # 5) Based on first camera orientation + glTF axis system, center by point cloud,
+    # construct alignment transform, and apply to point cloud
+    A = _compute_alignment_transform_first_cam_glTF_center_by_points(
+        prediction.extrinsics[0], points
+    )  # (4,4)
+
+    if points.shape[0] > 0:
+        points = trimesh.transform_points(points, A)
+
+    # 6) Clean + downsample
+    points, colors = _filter_and_downsample(points, colors, num_max_points)
+
+    # 7) Assemble scene (add point cloud first)
+    scene = trimesh.Scene()
+    if scene.metadata is None:
+        scene.metadata = {}
+    scene.metadata["hf_alignment"] = A  # For camera wireframes and external reuse
+
+    if points.shape[0] > 0:
+        pc = trimesh.points.PointCloud(vertices=points, colors=colors)
+        scene.add_geometry(pc)
+
+    # 8) Draw cameras (wireframe pyramids), using the same transform A
+    if show_cameras and prediction.intrinsics is not None and prediction.extrinsics is not None:
+        scene_scale = _estimate_scene_scale(points, fallback=1.0)
+        H, W = prediction.depth.shape[1:]
+        _add_cameras_to_scene(
+            scene=scene,
+            K=prediction.intrinsics,
+            ext_w2c=prediction.extrinsics,
+            image_sizes=[(H, W)] * prediction.depth.shape[0],
+            scale=scene_scale * camera_size,
+        )
+
+    # 9) Export
+    os.makedirs(export_dir, exist_ok=True)
+    out_path = os.path.join(export_dir, "scene.glb")
+    scene.export(out_path)
+
+    if export_depth_vis:
+        export_to_depth_vis(prediction, export_dir)
+        depth_vis_thumbnail = os.path.join(export_dir, "depth_vis", "0000.jpg")
+        if os.path.isfile(depth_vis_thumbnail):
+            shutil.copy2(depth_vis_thumbnail, os.path.join(export_dir, "scene.jpg"))
+        else:
+            logger.warn(f"Depth visualization thumbnail not found at {depth_vis_thumbnail}")
+    return out_path
+
+
+# =========================
+# utilities
+# =========================
+
+
+def _as_homogeneous44(ext: np.ndarray) -> np.ndarray:
+    """
+    Accept (4,4) or (3,4) extrinsic parameters, return (4,4) homogeneous matrix.
+    """
+    if ext.shape == (4, 4):
+        return ext
+    if ext.shape == (3, 4):
+        H = np.eye(4, dtype=ext.dtype)
+        H[:3, :4] = ext
+        return H
+    raise ValueError(f"extrinsic must be (4,4) or (3,4), got {ext.shape}")
+
+
+def _depths_to_world_points_with_colors(
+    depth: np.ndarray,
+    K: np.ndarray,
+    ext_w2c: np.ndarray,
+    images_u8: np.ndarray,
+    conf: np.ndarray | None,
+    conf_thr: float,
+) -> tuple[np.ndarray, np.ndarray]:
+    """
+    For each frame, transform (u,v,1) through K^{-1} to get rays,
+    multiply by depth to camera frame, then use (w2c)^{-1} to transform to world frame.
+    Simultaneously extract colors.
+    """
+    N, H, W = depth.shape
+    us, vs = np.meshgrid(np.arange(W), np.arange(H))
+    ones = np.ones_like(us)
+    pix = np.stack([us, vs, ones], axis=-1).reshape(-1, 3)  # (H*W,3)
+
+    pts_all, col_all = [], []
+
+    for i in range(N):
+        d = depth[i]  # (H,W)
+        valid = np.isfinite(d) & (d > 0)
+        if conf is not None:
+            valid &= conf[i] >= conf_thr
+        if not np.any(valid):
+            continue
+
+        d_flat = d.reshape(-1)
+        vidx = np.flatnonzero(valid.reshape(-1))
+
+        K_inv = np.linalg.inv(K[i])  # (3,3)
+        c2w = np.linalg.inv(_as_homogeneous44(ext_w2c[i]))  # (4,4)
+
+        rays = K_inv @ pix[vidx].T  # (3,M)
+        Xc = rays * d_flat[vidx][None, :]  # (3,M)
+        Xc_h = np.vstack([Xc, np.ones((1, Xc.shape[1]))])
+        Xw = (c2w @ Xc_h)[:3].T.astype(np.float32)  # (M,3)
+
+        cols = images_u8[i].reshape(-1, 3)[vidx].astype(np.uint8)  # (M,3)
+
+        pts_all.append(Xw)
+        col_all.append(cols)
+
+    if len(pts_all) == 0:
+        return np.zeros((0, 3), dtype=np.float32), np.zeros((0, 3), dtype=np.uint8)
+
+    return np.concatenate(pts_all, 0), np.concatenate(col_all, 0)
+
+
+def _filter_and_downsample(points: np.ndarray, colors: np.ndarray, num_max: int):
+    if points.shape[0] == 0:
+        return points, colors
+    finite = np.isfinite(points).all(axis=1)
+    points, colors = points[finite], colors[finite]
+    if points.shape[0] > num_max:
+        idx = np.random.choice(points.shape[0], num_max, replace=False)
+        points, colors = points[idx], colors[idx]
+    return points, colors
+
+
+def _estimate_scene_scale(points: np.ndarray, fallback: float = 1.0) -> float:
+    if points.shape[0] < 2:
+        return fallback
+    lo = np.percentile(points, 5, axis=0)
+    hi = np.percentile(points, 95, axis=0)
+    diag = np.linalg.norm(hi - lo)
+    return float(diag if np.isfinite(diag) and diag > 0 else fallback)
+
+
+def _compute_alignment_transform_first_cam_glTF_center_by_points(
+    ext_w2c0: np.ndarray,
+    points_world: np.ndarray,
+) -> np.ndarray:
+    """Computes the transformation matrix to align the scene with glTF standards.
+
+    This function calculates a 4x4 homogeneous matrix that centers the scene's
+    point cloud and transforms its coordinate system from the computer vision (CV)
+    standard to the glTF standard.
+
+    The transformation process involves three main steps:
+    1.  **Initial Alignment**: Orients the world coordinate system to match the
+        first camera's view (x-right, y-down, z-forward).
+    2.  **Coordinate System Conversion**: Converts the CV camera frame to the
+        glTF frame (x-right, y-up, z-backward) by flipping the Y and Z axes.
+    3.  **Centering**: Translates the entire scene so that the median of the
+        point cloud becomes the new origin (0,0,0).
+
+    Returns:
+        A 4x4 homogeneous transformation matrix (torch.Tensor or np.ndarray)
+        that applies these transformations.  A: X' = A @ [X;1]
+    """
+
+    w2c0 = _as_homogeneous44(ext_w2c0).astype(np.float64)
+
+    # CV -> glTF axis transformation
+    M = np.eye(4, dtype=np.float64)
+    M[1, 1] = -1.0  # flip Y
+    M[2, 2] = -1.0  # flip Z
+
+    # Don't center first
+    A_no_center = M @ w2c0
+
+    # Calculate point cloud center in new coordinate system (use median to resist outliers)
+    if points_world.shape[0] > 0:
+        pts_tmp = trimesh.transform_points(points_world, A_no_center)
+        center = np.median(pts_tmp, axis=0)
+    else:
+        center = np.zeros(3, dtype=np.float64)
+
+    T_center = np.eye(4, dtype=np.float64)
+    T_center[:3, 3] = -center
+
+    A = T_center @ A_no_center
+    return A
+
+
+def _add_cameras_to_scene(
+    scene: trimesh.Scene,
+    K: np.ndarray,
+    ext_w2c: np.ndarray,
+    image_sizes: list[tuple[int, int]],
+    scale: float,
+) -> None:
+    """Draws camera frustums to visualize their position and orientation.
+
+    This function renders each camera as a wireframe pyramid, originating from
+    the camera's center and extending to the corners of its imaging plane.
+
+    It reads the 'hf_alignment' metadata from the scene to ensure the
+    wireframes are correctly aligned with the 3D point cloud.
+    """
+    N = K.shape[0]
+    if N == 0:
+        return
+
+    # Alignment matrix consistent with point cloud (use identity matrix if missing)
+    A = None
+    try:
+        A = scene.metadata.get("hf_alignment", None) if scene.metadata else None
+    except Exception:
+        A = None
+    if A is None:
+        A = np.eye(4, dtype=np.float64)
+
+    for i in range(N):
+        H, W = image_sizes[i]
+        segs = _camera_frustum_lines(K[i], ext_w2c[i], W, H, scale)  # (8,2,3) world frame
+        # Apply unified transformation
+        segs = trimesh.transform_points(segs.reshape(-1, 3), A).reshape(-1, 2, 3)
+        path = trimesh.load_path(segs)
+        color = _index_color_rgb(i, N)
+        if hasattr(path, "colors"):
+            path.colors = np.tile(color, (len(path.entities), 1))
+        scene.add_geometry(path)
+
+
+def _camera_frustum_lines(
+    K: np.ndarray, ext_w2c: np.ndarray, W: int, H: int, scale: float
+) -> np.ndarray:
+    corners = np.array(
+        [
+            [0, 0, 1.0],
+            [W - 1, 0, 1.0],
+            [W - 1, H - 1, 1.0],
+            [0, H - 1, 1.0],
+        ],
+        dtype=float,
+    )  # (4,3)
+
+    K_inv = np.linalg.inv(K)
+    c2w = np.linalg.inv(_as_homogeneous44(ext_w2c))
+
+    # camera center in world
+    Cw = (c2w @ np.array([0, 0, 0, 1.0]))[:3]
+
+    # rays -> z=1 plane points (camera frame)
+    rays = (K_inv @ corners.T).T
+    z = rays[:, 2:3]
+    z[z == 0] = 1.0
+    plane_cam = (rays / z) * scale  # (4,3)
+
+    # to world
+    plane_w = []
+    for p in plane_cam:
+        pw = (c2w @ np.array([p[0], p[1], p[2], 1.0]))[:3]
+        plane_w.append(pw)
+    plane_w = np.stack(plane_w, 0)  # (4,3)
+
+    segs = []
+    # center to corners
+    for k in range(4):
+        segs.append(np.stack([Cw, plane_w[k]], 0))
+    # rectangle edges
+    order = [0, 1, 2, 3, 0]
+    for a, b in zip(order[:-1], order[1:]):
+        segs.append(np.stack([plane_w[a], plane_w[b]], 0))
+
+    return np.stack(segs, 0)  # (8,2,3)
+
+
+def _index_color_rgb(i: int, n: int) -> np.ndarray:
+    h = (i + 0.5) / max(n, 1)
+    s, v = 0.85, 0.95
+    r, g, b = _hsv_to_rgb(h, s, v)
+    return (np.array([r, g, b]) * 255).astype(np.uint8)
+
+
+def _hsv_to_rgb(h: float, s: float, v: float) -> tuple[float, float, float]:
+    i = int(h * 6.0)
+    f = h * 6.0 - i
+    p = v * (1.0 - s)
+    q = v * (1.0 - f * s)
+    t = v * (1.0 - (1.0 - f) * s)
+    i = i % 6
+    if i == 0:
+        r, g, b = v, t, p
+    elif i == 1:
+        r, g, b = q, v, p
+    elif i == 2:
+        r, g, b = p, v, t
+    elif i == 3:
+        r, g, b = p, q, v
+    elif i == 4:
+        r, g, b = t, p, v
+    else:
+        r, g, b = v, p, q
+    return r, g, b

multivue/cameras/source/src/depth_anything_3/utils/export/gs.py

@@ -0,0 +1,154 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from typing import Literal, Optional
+import moviepy.editor as mpy
+import torch
+
+from depth_anything_3.model.utils.gs_renderer import run_renderer_in_chunk_w_trj_mode
+from depth_anything_3.specs import Prediction
+from depth_anything_3.utils.gsply_helpers import save_gaussian_ply
+from depth_anything_3.utils.layout_helpers import hcat, vcat
+from depth_anything_3.utils.visualize import vis_depth_map_tensor
+
+VIDEO_QUALITY_MAP = {
+    "low": {"crf": "28", "preset": "veryfast"},
+    "medium": {"crf": "23", "preset": "medium"},
+    "high": {"crf": "18", "preset": "slow"},
+}
+
+
+def export_to_gs_ply(
+    prediction: Prediction,
+    export_dir: str,
+    gs_views_interval: Optional[
+        int
+    ] = 1,  # export GS every N views, useful for extremely dense inputs
+):
+    gs_world = prediction.gaussians
+    pred_depth = torch.from_numpy(prediction.depth).unsqueeze(-1).to(gs_world.means)  # v h w 1
+    idx = 0
+    os.makedirs(os.path.join(export_dir, "gs_ply"), exist_ok=True)
+    save_path = os.path.join(export_dir, f"gs_ply/{idx:04d}.ply")
+    if gs_views_interval is None:  # select around 12 views in total
+        gs_views_interval = max(pred_depth.shape[0] // 12, 1)
+    save_gaussian_ply(
+        gaussians=gs_world,
+        save_path=save_path,
+        ctx_depth=pred_depth,
+        shift_and_scale=False,
+        save_sh_dc_only=True,
+        gs_views_interval=gs_views_interval,
+        inv_opacity=True,
+        prune_by_depth_percent=0.9,
+        prune_border_gs=True,
+        match_3dgs_mcmc_dev=False,
+    )
+
+
+def export_to_gs_video(
+    prediction: Prediction,
+    export_dir: str,
+    extrinsics: Optional[torch.Tensor] = None,  # render views' world2cam, "b v 4 4"
+    intrinsics: Optional[torch.Tensor] = None,  # render views' unnormed intrinsics, "b v 3 3"
+    out_image_hw: Optional[tuple[int, int]] = None,  # render views' resolution, (h, w)
+    chunk_size: Optional[int] = 4,
+    trj_mode: Literal[
+        "original",
+        "smooth",
+        "interpolate",
+        "interpolate_smooth",
+        "wander",
+        "dolly_zoom",
+        "extend",
+        "wobble_inter",
+    ] = "extend",
+    color_mode: Literal["RGB+D", "RGB+ED"] = "RGB+ED",
+    vis_depth: Optional[Literal["hcat", "vcat"]] = "hcat",
+    enable_tqdm: Optional[bool] = True,
+    output_name: Optional[str] = None,
+    video_quality: Literal["low", "medium", "high"] = "high",
+) -> None:
+    gs_world = prediction.gaussians
+    # if target poses are not provided, render the (smooth/interpolate) input poses
+    if extrinsics is not None:
+        tgt_extrs = extrinsics
+    else:
+        tgt_extrs = torch.from_numpy(prediction.extrinsics).unsqueeze(0).to(gs_world.means)
+        if prediction.is_metric:
+            scale_factor = prediction.scale_factor
+            if scale_factor is not None:
+                tgt_extrs[:, :, :3, 3] /= scale_factor
+    tgt_intrs = (
+        intrinsics
+        if intrinsics is not None
+        else torch.from_numpy(prediction.intrinsics).unsqueeze(0).to(gs_world.means)
+    )
+    # if render resolution is not provided, render the input ones
+    if out_image_hw is not None:
+        H, W = out_image_hw
+    else:
+        H, W = prediction.depth.shape[-2:]
+    # if single views, render wander trj
+    if tgt_extrs.shape[1] <= 1:
+        trj_mode = "wander"
+        # trj_mode = "dolly_zoom"
+
+    color, depth = run_renderer_in_chunk_w_trj_mode(
+        gaussians=gs_world,
+        extrinsics=tgt_extrs,
+        intrinsics=tgt_intrs,
+        image_shape=(H, W),
+        chunk_size=chunk_size,
+        trj_mode=trj_mode,
+        use_sh=True,
+        color_mode=color_mode,
+        enable_tqdm=enable_tqdm,
+    )
+
+    # save as video
+    ffmpeg_params = [
+        "-crf",
+        VIDEO_QUALITY_MAP[video_quality]["crf"],
+        "-preset",
+        VIDEO_QUALITY_MAP[video_quality]["preset"],
+        "-pix_fmt",
+        "yuv420p",
+    ]  # best compatibility
+
+    os.makedirs(os.path.join(export_dir, "gs_video"), exist_ok=True)
+    for idx in range(color.shape[0]):
+        video_i = color[idx]
+        if vis_depth is not None:
+            depth_i = vis_depth_map_tensor(depth[0])
+            cat_fn = hcat if vis_depth == "hcat" else vcat
+            video_i = torch.stack([cat_fn(c, d) for c, d in zip(video_i, depth_i)])
+        frames = list(
+            (video_i.clamp(0, 1) * 255).byte().permute(0, 2, 3, 1).cpu().numpy()
+        )  # T x H x W x C, uint8, numpy()
+
+        fps = 24
+        clip = mpy.ImageSequenceClip(frames, fps=fps)
+        output_name = f"{idx:04d}_{trj_mode}" if output_name is None else output_name
+        save_path = os.path.join(export_dir, f"gs_video/{output_name}.mp4")
+        # clip.write_videofile(save_path, codec="libx264", audio=False, bitrate="4000k")
+        clip.write_videofile(
+            save_path,
+            codec="libx264",
+            audio=False,
+            fps=fps,
+            ffmpeg_params=ffmpeg_params,
+        )
+    return

multivue/cameras/source/src/depth_anything_3/utils/export/npz.py

@@ -0,0 +1,73 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import numpy as np
+
+from depth_anything_3.specs import Prediction
+from depth_anything_3.utils.parallel_utils import async_call
+
+
+@async_call
+def export_to_npz(
+    prediction: Prediction,
+    export_dir: str,
+):
+    output_file = os.path.join(export_dir, "exports", "npz", "results.npz")
+    os.makedirs(os.path.dirname(output_file), exist_ok=True)
+
+    # Use prediction.processed_images, which is already processed image data
+    if prediction.processed_images is None:
+        raise ValueError("prediction.processed_images is required but not available")
+
+    image = prediction.processed_images  # (N,H,W,3) uint8
+
+    # Build save dict with only non-None values
+    save_dict = {
+        "image": image,
+        "depth": np.round(prediction.depth, 6),
+    }
+
+    if prediction.conf is not None:
+        save_dict["conf"] = np.round(prediction.conf, 2)
+    if prediction.extrinsics is not None:
+        save_dict["extrinsics"] = prediction.extrinsics
+    if prediction.intrinsics is not None:
+        save_dict["intrinsics"] = prediction.intrinsics
+
+    # aux = {k: np.round(v, 4) for k, v in prediction.aux.items()}
+    np.savez_compressed(output_file, **save_dict)
+
+
+@async_call
+def export_to_mini_npz(
+    prediction: Prediction,
+    export_dir: str,
+):
+    output_file = os.path.join(export_dir, "exports", "mini_npz", "results.npz")
+    os.makedirs(os.path.dirname(output_file), exist_ok=True)
+
+    # Build save dict with only non-None values
+    save_dict = {
+        "depth": np.round(prediction.depth, 8),
+    }
+
+    if prediction.conf is not None:
+        save_dict["conf"] = np.round(prediction.conf, 2)
+    if prediction.extrinsics is not None:
+        save_dict["extrinsics"] = prediction.extrinsics
+    if prediction.intrinsics is not None:
+        save_dict["intrinsics"] = prediction.intrinsics
+
+    np.savez_compressed(output_file, **save_dict)

multivue/cameras/source/src/depth_anything_3/utils/export/utils.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import torch
+
+
+def _denorm_and_to_uint8(image_tensor: torch.Tensor) -> np.ndarray:
+    """Denormalize to [0,255] and output (N, H, W, 3) uint8."""
+    resnet_mean = torch.tensor(
+        [0.485, 0.456, 0.406], dtype=image_tensor.dtype, device=image_tensor.device
+    )
+    resnet_std = torch.tensor(
+        [0.229, 0.224, 0.225], dtype=image_tensor.dtype, device=image_tensor.device
+    )
+    img = image_tensor * resnet_std[None, :, None, None] + resnet_mean[None, :, None, None]
+    img = torch.clamp(img, 0.0, 1.0)
+    img = (img.permute(0, 2, 3, 1).cpu().numpy() * 255.0).round().astype(np.uint8)  # (N,H,W,3)
+    return img

multivue/cameras/source/src/depth_anything_3/utils/geometry.py

@@ -0,0 +1,498 @@
+# flake8: noqa: F722
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from types import SimpleNamespace
+from typing import Optional
+import numpy as np
+import torch
+import torch.nn.functional as F
+from einops import einsum
+
+
+def as_homogeneous(ext):
+    """
+    Accept (..., 3,4) or (..., 4,4) extrinsics, return (...,4,4) homogeneous matrix.
+    Supports torch.Tensor or np.ndarray.
+    """
+    if isinstance(ext, torch.Tensor):
+        # If already in homogeneous form
+        if ext.shape[-2:] == (4, 4):
+            return ext
+        elif ext.shape[-2:] == (3, 4):
+            # Create a new homogeneous matrix
+            ones = torch.zeros_like(ext[..., :1, :4])
+            ones[..., 0, 3] = 1.0
+            return torch.cat([ext, ones], dim=-2)
+        else:
+            raise ValueError(f"Invalid shape for torch.Tensor: {ext.shape}")
+
+    elif isinstance(ext, np.ndarray):
+        if ext.shape[-2:] == (4, 4):
+            return ext
+        elif ext.shape[-2:] == (3, 4):
+            ones = np.zeros_like(ext[..., :1, :4])
+            ones[..., 0, 3] = 1.0
+            return np.concatenate([ext, ones], axis=-2)
+        else:
+            raise ValueError(f"Invalid shape for np.ndarray: {ext.shape}")
+
+    else:
+        raise TypeError("Input must be a torch.Tensor or np.ndarray.")
+
+
+@torch.jit.script
+def affine_inverse(A: torch.Tensor):
+    R = A[..., :3, :3]  # ..., 3, 3
+    T = A[..., :3, 3:]  # ..., 3, 1
+    P = A[..., 3:, :]  # ..., 1, 4
+    return torch.cat([torch.cat([R.mT, -R.mT @ T], dim=-1), P], dim=-2)
+
+
+def transpose_last_two_axes(arr):
+    """
+    for np < 2
+    """
+    if arr.ndim < 2:
+        return arr
+    axes = list(range(arr.ndim))
+    # swap the last two
+    axes[-2], axes[-1] = axes[-1], axes[-2]
+    return arr.transpose(axes)
+
+
+def affine_inverse_np(A: np.ndarray):
+    R = A[..., :3, :3]
+    T = A[..., :3, 3:]
+    P = A[..., 3:, :]
+    return np.concatenate(
+        [
+            np.concatenate([transpose_last_two_axes(R), -transpose_last_two_axes(R) @ T], axis=-1),
+            P,
+        ],
+        axis=-2,
+    )
+
+
+def quat_to_mat(quaternions: torch.Tensor) -> torch.Tensor:
+    """
+    Quaternion Order: XYZW or say ijkr, scalar-last
+
+    Convert rotations given as quaternions to rotation matrices.
+    Args:
+        quaternions: quaternions with real part last,
+            as tensor of shape (..., 4).
+
+    Returns:
+        Rotation matrices as tensor of shape (..., 3, 3).
+    """
+    i, j, k, r = torch.unbind(quaternions, -1)
+    # pyre-fixme[58]: `/` is not supported for operand types `float` and `Tensor`.
+    two_s = 2.0 / (quaternions * quaternions).sum(-1)
+
+    o = torch.stack(
+        (
+            1 - two_s * (j * j + k * k),
+            two_s * (i * j - k * r),
+            two_s * (i * k + j * r),
+            two_s * (i * j + k * r),
+            1 - two_s * (i * i + k * k),
+            two_s * (j * k - i * r),
+            two_s * (i * k - j * r),
+            two_s * (j * k + i * r),
+            1 - two_s * (i * i + j * j),
+        ),
+        -1,
+    )
+    return o.reshape(quaternions.shape[:-1] + (3, 3))
+
+
+def mat_to_quat(matrix: torch.Tensor) -> torch.Tensor:
+    """
+    Convert rotations given as rotation matrices to quaternions.
+
+    Args:
+        matrix: Rotation matrices as tensor of shape (..., 3, 3).
+
+    Returns:
+        quaternions with real part last, as tensor of shape (..., 4).
+        Quaternion Order: XYZW or say ijkr, scalar-last
+    """
+    if matrix.size(-1) != 3 or matrix.size(-2) != 3:
+        raise ValueError(f"Invalid rotation matrix shape {matrix.shape}.")
+
+    batch_dim = matrix.shape[:-2]
+    m00, m01, m02, m10, m11, m12, m20, m21, m22 = torch.unbind(
+        matrix.reshape(batch_dim + (9,)), dim=-1
+    )
+
+    q_abs = _sqrt_positive_part(
+        torch.stack(
+            [
+                1.0 + m00 + m11 + m22,
+                1.0 + m00 - m11 - m22,
+                1.0 - m00 + m11 - m22,
+                1.0 - m00 - m11 + m22,
+            ],
+            dim=-1,
+        )
+    )
+
+    # we produce the desired quaternion multiplied by each of r, i, j, k
+    quat_by_rijk = torch.stack(
+        [
+            # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and
+            #  `int`.
+            torch.stack([q_abs[..., 0] ** 2, m21 - m12, m02 - m20, m10 - m01], dim=-1),
+            # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and
+            #  `int`.
+            torch.stack([m21 - m12, q_abs[..., 1] ** 2, m10 + m01, m02 + m20], dim=-1),
+            # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and
+            #  `int`.
+            torch.stack([m02 - m20, m10 + m01, q_abs[..., 2] ** 2, m12 + m21], dim=-1),
+            # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and
+            #  `int`.
+            torch.stack([m10 - m01, m20 + m02, m21 + m12, q_abs[..., 3] ** 2], dim=-1),
+        ],
+        dim=-2,
+    )
+
+    # We floor here at 0.1 but the exact level is not important; if q_abs is small,
+    # the candidate won't be picked.
+    flr = torch.tensor(0.1).to(dtype=q_abs.dtype, device=q_abs.device)
+    quat_candidates = quat_by_rijk / (2.0 * q_abs[..., None].max(flr))
+
+    # if not for numerical problems, quat_candidates[i] should be same (up to a sign),
+    # forall i; we pick the best-conditioned one (with the largest denominator)
+    out = quat_candidates[F.one_hot(q_abs.argmax(dim=-1), num_classes=4) > 0.5, :].reshape(
+        batch_dim + (4,)
+    )
+
+    # Convert from rijk to ijkr
+    out = out[..., [1, 2, 3, 0]]
+
+    out = standardize_quaternion(out)
+
+    return out
+
+
+def _sqrt_positive_part(x: torch.Tensor) -> torch.Tensor:
+    """
+    Returns torch.sqrt(torch.max(0, x))
+    but with a zero subgradient where x is 0.
+    """
+    ret = torch.zeros_like(x)
+    positive_mask = x > 0
+    if torch.is_grad_enabled():
+        ret[positive_mask] = torch.sqrt(x[positive_mask])
+    else:
+        ret = torch.where(positive_mask, torch.sqrt(x), ret)
+    return ret
+
+
+def standardize_quaternion(quaternions: torch.Tensor) -> torch.Tensor:
+    """
+    Convert a unit quaternion to a standard form: one in which the real
+    part is non negative.
+
+    Args:
+        quaternions: Quaternions with real part last,
+            as tensor of shape (..., 4).
+
+    Returns:
+        Standardized quaternions as tensor of shape (..., 4).
+    """
+    return torch.where(quaternions[..., 3:4] < 0, -quaternions, quaternions)
+
+
+def sample_image_grid(
+    shape: tuple[int, ...],
+    device: torch.device = torch.device("cpu"),
+) -> tuple[
+    torch.Tensor,  # float coordinates (xy indexing), "*shape dim"
+    torch.Tensor,  # integer indices (ij indexing), "*shape dim"
+]:
+    """Get normalized (range 0 to 1) coordinates and integer indices for an image."""
+
+    # Each entry is a pixel-wise integer coordinate. In the 2D case, each entry is a
+    # (row, col) coordinate.
+    indices = [torch.arange(length, device=device) for length in shape]
+    stacked_indices = torch.stack(torch.meshgrid(*indices, indexing="ij"), dim=-1)
+
+    # Each entry is a floating-point coordinate in the range (0, 1). In the 2D case,
+    # each entry is an (x, y) coordinate.
+    coordinates = [(idx + 0.5) / length for idx, length in zip(indices, shape)]
+    coordinates = reversed(coordinates)
+    coordinates = torch.stack(torch.meshgrid(*coordinates, indexing="xy"), dim=-1)
+
+    return coordinates, stacked_indices
+
+
+def homogenize_points(points: torch.Tensor) -> torch.Tensor:  # "*batch dim"  # "*batch dim+1"
+    """Convert batched points (xyz) to (xyz1)."""
+    return torch.cat([points, torch.ones_like(points[..., :1])], dim=-1)
+
+
+def homogenize_vectors(vectors: torch.Tensor) -> torch.Tensor:  #  "*batch dim"  # "*batch dim+1"
+    """Convert batched vectors (xyz) to (xyz0)."""
+    return torch.cat([vectors, torch.zeros_like(vectors[..., :1])], dim=-1)
+
+
+def transform_rigid(
+    homogeneous_coordinates: torch.Tensor,  # "*#batch dim"
+    transformation: torch.Tensor,  # "*#batch dim dim"
+) -> torch.Tensor:  # "*batch dim"
+    """Apply a rigid-body transformation to points or vectors."""
+    return einsum(
+        transformation,
+        homogeneous_coordinates.to(transformation.dtype),
+        "... i j, ... j -> ... i",
+    )
+
+
+def transform_cam2world(
+    homogeneous_coordinates: torch.Tensor,  # "*#batch dim"
+    extrinsics: torch.Tensor,  # "*#batch dim dim"
+) -> torch.Tensor:  # "*batch dim"
+    """Transform points from 3D camera coordinates to 3D world coordinates."""
+    return transform_rigid(homogeneous_coordinates, extrinsics)
+
+
+def unproject(
+    coordinates: torch.Tensor,  # "*#batch dim"
+    z: torch.Tensor,  # "*#batch"
+    intrinsics: torch.Tensor,  # "*#batch dim+1 dim+1"
+) -> torch.Tensor:  # "*batch dim+1"
+    """Unproject 2D camera coordinates with the given Z values."""
+
+    # Apply the inverse intrinsics to the coordinates.
+    coordinates = homogenize_points(coordinates)
+    ray_directions = einsum(
+        intrinsics.float().inverse().to(intrinsics),
+        coordinates.to(intrinsics.dtype),
+        "... i j, ... j -> ... i",
+    )
+
+    # Apply the supplied depth values.
+    return ray_directions * z[..., None]
+
+
+def get_world_rays(
+    coordinates: torch.Tensor,  # "*#batch dim"
+    extrinsics: torch.Tensor,  # "*#batch dim+2 dim+2"
+    intrinsics: torch.Tensor,  # "*#batch dim+1 dim+1"
+) -> tuple[
+    torch.Tensor,  # origins, "*batch dim+1"
+    torch.Tensor,  # directions, "*batch dim+1"
+]:
+    # Get camera-space ray directions.
+    directions = unproject(
+        coordinates,
+        torch.ones_like(coordinates[..., 0]),
+        intrinsics,
+    )
+    directions = directions / directions.norm(dim=-1, keepdim=True)
+
+    # Transform ray directions to world coordinates.
+    directions = homogenize_vectors(directions)
+    directions = transform_cam2world(directions, extrinsics)[..., :-1]
+
+    # Tile the ray origins to have the same shape as the ray directions.
+    origins = extrinsics[..., :-1, -1].broadcast_to(directions.shape)
+
+    return origins, directions
+
+
+def get_fov(intrinsics: torch.Tensor) -> torch.Tensor:  # "batch 3 3" -> "batch 2"
+    intrinsics_inv = intrinsics.float().inverse().to(intrinsics)
+
+    def process_vector(vector):
+        vector = torch.tensor(vector, dtype=intrinsics.dtype, device=intrinsics.device)
+        vector = einsum(intrinsics_inv, vector, "b i j, j -> b i")
+        return vector / vector.norm(dim=-1, keepdim=True)
+
+    left = process_vector([0, 0.5, 1])
+    right = process_vector([1, 0.5, 1])
+    top = process_vector([0.5, 0, 1])
+    bottom = process_vector([0.5, 1, 1])
+    fov_x = (left * right).sum(dim=-1).acos()
+    fov_y = (top * bottom).sum(dim=-1).acos()
+    return torch.stack((fov_x, fov_y), dim=-1)
+
+
+def map_pdf_to_opacity(
+    pdf: torch.Tensor,  # " *batch"
+    global_step: int = 0,
+    opacity_mapping: Optional[dict] = None,
+) -> torch.Tensor:  # " *batch"
+    # https://www.desmos.com/calculator/opvwti3ba9
+
+    # Figure out the exponent.
+    if opacity_mapping is not None:
+        cfg = SimpleNamespace(**opacity_mapping)
+        x = cfg.initial + min(global_step / cfg.warm_up, 1) * (cfg.final - cfg.initial)
+    else:
+        x = 0.0
+    exponent = 2**x
+
+    # Map the probability density to an opacity.
+    return 0.5 * (1 - (1 - pdf) ** exponent + pdf ** (1 / exponent))
+
+def normalize_homogenous_points(points):
+    """Normalize the point vectors"""
+    return points / points[..., -1:]
+
+def inverse_intrinsic_matrix(ixts):
+    """ """
+    return torch.inverse(ixts)
+
+def pixel_space_to_camera_space(pixel_space_points, depth, intrinsics):
+    """
+    Convert pixel space points to camera space points.
+
+    Args:
+        pixel_space_points (torch.Tensor): Pixel space points with shape (h, w, 2)
+        depth (torch.Tensor): Depth map with shape (b, v, h, w, 1)
+        intrinsics (torch.Tensor): Camera intrinsics with shape (b, v, 3, 3)
+
+    Returns:
+        torch.Tensor: Camera space points with shape (b, v, h, w, 3).
+    """
+    pixel_space_points = homogenize_points(pixel_space_points)
+    # camera_space_points = torch.einsum(
+    #     "b v i j , h w j -> b v h w i", intrinsics.inverse(), pixel_space_points
+    # )
+    camera_space_points = torch.einsum(
+        "b v i j , h w j -> b v h w i", inverse_intrinsic_matrix(intrinsics), pixel_space_points
+    )
+    camera_space_points = camera_space_points * depth
+    return camera_space_points
+
+
+def camera_space_to_world_space(camera_space_points, c2w):
+    """
+    Convert camera space points to world space points.
+
+    Args:
+        camera_space_points (torch.Tensor): Camera space points with shape (b, v, h, w, 3)
+        c2w (torch.Tensor): Camera to world extrinsics matrix with shape (b, v, 4, 4)
+
+    Returns:
+        torch.Tensor: World space points with shape (b, v, h, w, 3).
+    """
+    camera_space_points = homogenize_points(camera_space_points)
+    world_space_points = torch.einsum("b v i j , b v h w j -> b v h w i", c2w, camera_space_points)
+    return world_space_points[..., :3]
+
+
+def camera_space_to_pixel_space(camera_space_points, intrinsics):
+    """
+    Convert camera space points to pixel space points.
+
+    Args:
+        camera_space_points (torch.Tensor): Camera space points with shape (b, v1, v2, h, w, 3)
+        c2w (torch.Tensor): Camera to world extrinsics matrix with shape (b, v2, 3, 3)
+
+    Returns:
+        torch.Tensor: World space points with shape (b, v1, v2, h, w, 2).
+    """
+    camera_space_points = normalize_homogenous_points(camera_space_points)
+    pixel_space_points = torch.einsum(
+        "b u i j , b v u h w j -> b v u h w i", intrinsics, camera_space_points
+    )
+    return pixel_space_points[..., :2]
+
+
+def world_space_to_camera_space(world_space_points, c2w):
+    """
+    Convert world space points to pixel space points.
+
+    Args:
+        world_space_points (torch.Tensor): World space points with shape (b, v1, h, w, 3)
+        c2w (torch.Tensor): Camera to world extrinsics matrix with shape (b, v2, 4, 4)
+
+    Returns:
+        torch.Tensor: Camera space points with shape (b, v1, v2, h, w, 3).
+    """
+    world_space_points = homogenize_points(world_space_points)
+    camera_space_points = torch.einsum(
+        "b u i j , b v h w j -> b v u h w i", c2w.inverse(), world_space_points
+    )
+    return camera_space_points[..., :3]
+
+
+def unproject_depth(
+    depth, intrinsics, c2w=None, ixt_normalized=False, num_patches_x=None, num_patches_y=None
+):
+    """
+    Turn the depth map into a 3D point cloud in world space
+
+    Args:
+        depth: (b, v, h, w, 1)
+        intrinsics: (b, v, 3, 3)
+        c2w: (b, v, 4, 4)
+
+    Returns:
+        torch.Tensor: World space points with shape (b, v, h, w, 3).
+    """
+    if c2w is None:
+        c2w = torch.eye(4, device=depth.device, dtype=depth.dtype)
+        c2w = c2w[None, None].repeat(depth.shape[0], depth.shape[1], 1, 1)
+
+    if not ixt_normalized:
+        # Compute indices of pixels
+        h, w = depth.shape[-3], depth.shape[-2]
+        x_grid, y_grid = torch.meshgrid(
+            torch.arange(w, device=depth.device, dtype=depth.dtype),
+            torch.arange(h, device=depth.device, dtype=depth.dtype),
+            indexing="xy",
+        )  # (h, w), (h, w)
+    else:
+        # ixt_normalized: h=w=2.0. cx, cy, fx, fy are normalized according to h=w=2.0
+        assert num_patches_x is not None and num_patches_y is not None
+        dx = 1 / num_patches_x
+        dy = 1 / num_patches_y
+        max_y = 1 - dy
+        min_y = -max_y
+        max_x = 1 - dx
+        min_x = -max_x
+
+        grid_shift = 1.0
+        y_grid, x_grid = torch.meshgrid(
+            torch.linspace(
+                min_y + grid_shift,
+                max_y + grid_shift,
+                num_patches_y,
+                dtype=torch.float32,
+                device=depth.device,
+            ),
+            torch.linspace(
+                min_x + grid_shift,
+                max_x + grid_shift,
+                num_patches_x,
+                dtype=torch.float32,
+                device=depth.device,
+            ),
+            indexing="ij",
+        )
+
+    # Compute coordinates of pixels in camera space
+    pixel_space_points = torch.stack((x_grid, y_grid), dim=-1)  # (..., h, w, 2)
+    camera_points = pixel_space_to_camera_space(
+        pixel_space_points, depth, intrinsics
+    )  # (..., h, w, 3)
+
+    # Convert points to world space
+    world_points = camera_space_to_world_space(camera_points, c2w)  # (..., h, w, 3)
+
+    return world_points

multivue/cameras/source/src/depth_anything_3/utils/gsply_helpers.py

@@ -0,0 +1,173 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from pathlib import Path
+from typing import Optional
+import numpy as np
+import torch
+from einops import rearrange, repeat
+from plyfile import PlyData, PlyElement
+from torch import Tensor
+
+from depth_anything_3.specs import Gaussians
+
+
+def construct_list_of_attributes(num_rest: int) -> list[str]:
+    attributes = ["x", "y", "z", "nx", "ny", "nz"]
+    for i in range(3):
+        attributes.append(f"f_dc_{i}")
+    for i in range(num_rest):
+        attributes.append(f"f_rest_{i}")
+    attributes.append("opacity")
+    for i in range(3):
+        attributes.append(f"scale_{i}")
+    for i in range(4):
+        attributes.append(f"rot_{i}")
+    return attributes
+
+
+def export_ply(
+    means: Tensor,  # "gaussian 3"
+    scales: Tensor,  # "gaussian 3"
+    rotations: Tensor,  # "gaussian 4"
+    harmonics: Tensor,  # "gaussian 3 d_sh"
+    opacities: Tensor,  # "gaussian"
+    path: Path,
+    shift_and_scale: bool = False,
+    save_sh_dc_only: bool = True,
+    match_3dgs_mcmc_dev: Optional[bool] = False,
+):
+    if shift_and_scale:
+        # Shift the scene so that the median Gaussian is at the origin.
+        means = means - means.median(dim=0).values
+
+        # Rescale the scene so that most Gaussians are within range [-1, 1].
+        scale_factor = means.abs().quantile(0.95, dim=0).max()
+        means = means / scale_factor
+        scales = scales / scale_factor
+
+    rotations = rotations.detach().cpu().numpy()
+
+    # Since current model use SH_degree = 4,
+    # which require large memory to store, we can only save the DC band to save memory.
+    f_dc = harmonics[..., 0]
+    f_rest = harmonics[..., 1:].flatten(start_dim=1)
+
+    if match_3dgs_mcmc_dev:
+        sh_degree = 3
+        n_rest = 3 * (sh_degree + 1) ** 2 - 3
+        f_rest = repeat(
+            torch.zeros_like(harmonics[..., :1]), "... i -> ... (n i)", n=(n_rest // 3)
+        ).flatten(start_dim=1)
+        dtype_full = [
+            (attribute, "f4")
+            for attribute in construct_list_of_attributes(num_rest=n_rest)
+            if attribute not in ("nx", "ny", "nz")
+        ]
+    else:
+        dtype_full = [
+            (attribute, "f4")
+            for attribute in construct_list_of_attributes(
+                0 if save_sh_dc_only else f_rest.shape[1]
+            )
+        ]
+    elements = np.empty(means.shape[0], dtype=dtype_full)
+    attributes = [
+        means.detach().cpu().numpy(),
+        torch.zeros_like(means).detach().cpu().numpy(),
+        f_dc.detach().cpu().contiguous().numpy(),
+        f_rest.detach().cpu().contiguous().numpy(),
+        opacities[..., None].detach().cpu().numpy(),
+        scales.log().detach().cpu().numpy(),
+        rotations,
+    ]
+    if match_3dgs_mcmc_dev:
+        attributes.pop(1)  # dummy normal is not needed
+    elif save_sh_dc_only:
+        attributes.pop(3)  # remove f_rest from attributes
+
+    attributes = np.concatenate(attributes, axis=1)
+    elements[:] = list(map(tuple, attributes))
+    path.parent.mkdir(exist_ok=True, parents=True)
+    PlyData([PlyElement.describe(elements, "vertex")]).write(path)
+
+
+def inverse_sigmoid(x):
+    return torch.log(x / (1 - x))
+
+
+def save_gaussian_ply(
+    gaussians: Gaussians,
+    save_path: str,
+    ctx_depth: torch.Tensor,  # depth of input views; for getting shape and filtering, "v h w 1"
+    shift_and_scale: bool = False,
+    save_sh_dc_only: bool = True,
+    gs_views_interval: int = 1,
+    inv_opacity: Optional[bool] = True,
+    prune_by_depth_percent: Optional[float] = 1.0,
+    prune_border_gs: Optional[bool] = True,
+    match_3dgs_mcmc_dev: Optional[bool] = False,
+):
+    b = gaussians.means.shape[0]
+    assert b == 1, "must set batch_size=1 when exporting 3D gaussians"
+    src_v, out_h, out_w, _ = ctx_depth.shape
+
+    # extract gs params
+    world_means = gaussians.means
+    world_shs = gaussians.harmonics
+    world_rotations = gaussians.rotations
+    gs_scales = gaussians.scales
+    gs_opacities = inverse_sigmoid(gaussians.opacities) if inv_opacity else gaussians.opacities
+
+    # Create a mask to filter the Gaussians.
+
+    # TODO: prune the sky region here
+
+    # throw away Gaussians at the borders, since they're generally of lower quality.
+    if prune_border_gs:
+        mask = torch.zeros_like(ctx_depth, dtype=torch.bool)
+        gstrim_h = int(8 / 256 * out_h)
+        gstrim_w = int(8 / 256 * out_w)
+        mask[:, gstrim_h:-gstrim_h, gstrim_w:-gstrim_w, :] = 1
+    else:
+        mask = torch.ones_like(ctx_depth, dtype=torch.bool)
+
+    # trim the far away point based on depth;
+    if prune_by_depth_percent is not None and prune_by_depth_percent < 1:
+        in_depths = ctx_depth
+        d_percentile = torch.quantile(
+            in_depths.view(in_depths.shape[0], -1), q=prune_by_depth_percent, dim=1
+        ).view(-1, 1, 1)
+        d_mask = (in_depths[..., 0] <= d_percentile).unsqueeze(-1)
+        mask = mask & d_mask
+    mask = mask.squeeze(-1)  # v h w
+
+    # helper fn, must place after mask
+    def trim_select_reshape(element):
+        selected_element = rearrange(
+            element[0], "(v h w) ... -> v h w ...", v=src_v, h=out_h, w=out_w
+        )
+        selected_element = selected_element[::gs_views_interval][mask[::gs_views_interval]]
+        return selected_element
+
+    export_ply(
+        means=trim_select_reshape(world_means),
+        scales=trim_select_reshape(gs_scales),
+        rotations=trim_select_reshape(world_rotations),
+        harmonics=trim_select_reshape(world_shs),
+        opacities=trim_select_reshape(gs_opacities),
+        path=Path(save_path),
+        shift_and_scale=shift_and_scale,
+        save_sh_dc_only=save_sh_dc_only,
+        match_3dgs_mcmc_dev=match_3dgs_mcmc_dev,
+    )

multivue/cameras/source/src/depth_anything_3/utils/io/input_processor.py

@@ -0,0 +1,501 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Input processor for Depth Anything 3 (parallelized).
+
+This version removes the square center-crop step for "*crop" methods (same as your note).
+In addition, it parallelizes per-image preprocessing using the provided `parallel_execution`.
+"""
+
+from __future__ import annotations
+
+from typing import Sequence
+import cv2
+import numpy as np
+import torch
+import torchvision.transforms as T
+from PIL import Image
+
+from depth_anything_3.utils.logger import logger
+from depth_anything_3.utils.parallel_utils import parallel_execution
+
+
+class InputProcessor:
+    """Prepares a batch of images for model inference.
+    This processor converts a list of image file paths into a single, model-ready
+    tensor. The processing pipeline is executed in parallel across multiple workers
+    for efficiency.
+
+    Pipeline:
+      1) Load image and convert to RGB
+      2) Boundary resize (upper/lower bound, preserving aspect ratio)
+      3) Enforce divisibility by PATCH_SIZE:
+         - "*resize" methods: each dimension is rounded to nearest multiple
+           (may up/downscale a few px)
+         - "*crop"   methods: each dimension is floored to nearest multiple via center crop
+      4) Convert to tensor and apply ImageNet normalization
+      5) Stack into (1, N, 3, H, W)
+
+    Parallelization:
+      - Each image is processed independently in a worker.
+      - Order of outputs matches the input order.
+    """
+
+    NORMALIZE = T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+    PATCH_SIZE = 14
+
+    def __init__(self):
+        pass
+
+    # -----------------------------
+    # Public API
+    # -----------------------------
+    def __call__(
+        self,
+        image: list[np.ndarray | Image.Image | str],
+        extrinsics: np.ndarray | None = None,
+        intrinsics: np.ndarray | None = None,
+        process_res: int = 504,
+        process_res_method: str = "upper_bound_resize",
+        *,
+        num_workers: int = 8,
+        print_progress: bool = False,
+        sequential: bool | None = None,
+        desc: str | None = "Preprocess",
+    ) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
+        """
+        Returns:
+            (tensor, extrinsics_list, intrinsics_list)
+            tensor shape: (1, N, 3, H, W)
+        """
+        sequential = self._resolve_sequential(sequential, num_workers)
+        exts_list, ixts_list = self._validate_and_pack_meta(image, extrinsics, intrinsics)
+
+        results = self._run_parallel(
+            image=image,
+            exts_list=exts_list,
+            ixts_list=ixts_list,
+            process_res=process_res,
+            process_res_method=process_res_method,
+            num_workers=num_workers,
+            print_progress=print_progress,
+            sequential=sequential,
+            desc=desc,
+        )
+
+        proc_imgs, out_sizes, out_ixts, out_exts = self._unpack_results(results)
+        proc_imgs, out_sizes, out_ixts = self._unify_batch_shapes(proc_imgs, out_sizes, out_ixts)
+
+        batch_tensor = self._stack_batch(proc_imgs)
+        out_exts = (
+            torch.from_numpy(np.asarray(out_exts)).float()
+            if out_exts is not None and out_exts[0] is not None
+            else None
+        )
+        out_ixts = (
+            torch.from_numpy(np.asarray(out_ixts)).float()
+            if out_ixts is not None and out_ixts[0] is not None
+            else None
+        )
+        return (batch_tensor, out_exts, out_ixts)
+
+    # -----------------------------
+    # __call__ helpers
+    # -----------------------------
+    def _resolve_sequential(self, sequential: bool | None, num_workers: int) -> bool:
+        return (num_workers <= 1) if sequential is None else sequential
+
+    def _validate_and_pack_meta(
+        self,
+        images: list[np.ndarray | Image.Image | str],
+        extrinsics: np.ndarray | None,
+        intrinsics: np.ndarray | None,
+    ) -> tuple[list[np.ndarray | None] | None, list[np.ndarray | None] | None]:
+        if extrinsics is not None and len(extrinsics) != len(images):
+            raise ValueError("Length of extrinsics must match images when provided.")
+        if intrinsics is not None and len(intrinsics) != len(images):
+            raise ValueError("Length of intrinsics must match images when provided.")
+        exts_list = [e for e in extrinsics] if extrinsics is not None else None
+        ixts_list = [k for k in intrinsics] if intrinsics is not None else None
+        return exts_list, ixts_list
+
+    def _run_parallel(
+        self,
+        *,
+        image: list[np.ndarray | Image.Image | str],
+        exts_list: list[np.ndarray | None] | None,
+        ixts_list: list[np.ndarray | None] | None,
+        process_res: int,
+        process_res_method: str,
+        num_workers: int,
+        print_progress: bool,
+        sequential: bool,
+        desc: str | None,
+    ):
+        results = parallel_execution(
+            image,
+            exts_list,
+            ixts_list,
+            action=self._process_one,  # (img, extrinsic, intrinsic, ...)
+            num_processes=num_workers,
+            print_progress=print_progress,
+            sequential=sequential,
+            desc=desc,
+            process_res=process_res,
+            process_res_method=process_res_method,
+        )
+        if not results:
+            raise RuntimeError(
+                "No preprocessing results returned. Check inputs and parallel_execution."
+            )
+        return results
+
+    def _unpack_results(self, results):
+        """
+        results: List[Tuple[torch.Tensor, Tuple[H, W], Optional[np.ndarray], Optional[np.ndarray]]]
+        -> processed_images, out_sizes, out_intrinsics, out_extrinsics
+        """
+        try:
+            processed_images, out_sizes, out_intrinsics, out_extrinsics = zip(*results)
+        except Exception as e:
+            raise RuntimeError(
+                "Unexpected results structure from parallel_execution: "
+                f"{type(results)} / sample: {results[0]}"
+            ) from e
+
+        return list(processed_images), list(out_sizes), list(out_intrinsics), list(out_extrinsics)
+
+    def _unify_batch_shapes(
+        self,
+        processed_images: list[torch.Tensor],
+        out_sizes: list[tuple[int, int]],
+        out_intrinsics: list[np.ndarray | None],
+    ) -> tuple[list[torch.Tensor], list[tuple[int, int]], list[np.ndarray | None]]:
+        """Center-crop all tensors to the smallest H, W; adjust intrinsics' cx, cy accordingly."""
+        if len(set(out_sizes)) <= 1:
+            return processed_images, out_sizes, out_intrinsics
+
+        min_h = min(h for h, _ in out_sizes)
+        min_w = min(w for _, w in out_sizes)
+        logger.warn(
+            f"Images in batch have different sizes {out_sizes}; "
+            f"center-cropping all to smallest ({min_h},{min_w})"
+        )
+
+        center_crop = T.CenterCrop((min_h, min_w))
+        new_imgs, new_sizes, new_ixts = [], [], []
+        for img_t, (H, W), K in zip(processed_images, out_sizes, out_intrinsics):
+            crop_top = max(0, (H - min_h) // 2)
+            crop_left = max(0, (W - min_w) // 2)
+            new_imgs.append(center_crop(img_t))
+            new_sizes.append((min_h, min_w))
+            if K is None:
+                new_ixts.append(None)
+            else:
+                K_adj = K.copy()
+                K_adj[0, 2] -= crop_left
+                K_adj[1, 2] -= crop_top
+                new_ixts.append(K_adj)
+        return new_imgs, new_sizes, new_ixts
+
+    def _stack_batch(self, processed_images: list[torch.Tensor]) -> torch.Tensor:
+        return torch.stack(processed_images)
+
+    # -----------------------------
+    # Per-item worker
+    # -----------------------------
+    def _process_one(
+        self,
+        img: np.ndarray | Image.Image | str,
+        extrinsic: np.ndarray | None = None,
+        intrinsic: np.ndarray | None = None,
+        *,
+        process_res: int,
+        process_res_method: str,
+    ) -> tuple[torch.Tensor, tuple[int, int], np.ndarray | None, np.ndarray | None]:
+        # Load & remember original size
+        pil_img = self._load_image(img)
+        orig_w, orig_h = pil_img.size
+
+        # Boundary resize
+        pil_img = self._resize_image(pil_img, process_res, process_res_method)
+        w, h = pil_img.size
+        intrinsic = self._resize_ixt(intrinsic, orig_w, orig_h, w, h)
+
+        # Enforce divisibility by PATCH_SIZE
+        if process_res_method.endswith("resize"):
+            pil_img = self._make_divisible_by_resize(pil_img, self.PATCH_SIZE)
+            new_w, new_h = pil_img.size
+            intrinsic = self._resize_ixt(intrinsic, w, h, new_w, new_h)
+            w, h = new_w, new_h
+        elif process_res_method.endswith("crop"):
+            pil_img = self._make_divisible_by_crop(pil_img, self.PATCH_SIZE)
+            new_w, new_h = pil_img.size
+            intrinsic = self._crop_ixt(intrinsic, w, h, new_w, new_h)
+            w, h = new_w, new_h
+        else:
+            raise ValueError(f"Unsupported process_res_method: {process_res_method}")
+
+        # Convert to tensor & normalize
+        img_tensor = self._normalize_image(pil_img)
+        _, H, W = img_tensor.shape
+        assert (W, H) == (w, h), "Tensor size mismatch with PIL image size after processing."
+
+        # Return: (img_tensor, (H, W), intrinsic, extrinsic)
+        return img_tensor, (H, W), intrinsic, extrinsic
+
+    # -----------------------------
+    # Intrinsics transforms
+    # -----------------------------
+    def _resize_ixt(
+        self,
+        intrinsic: np.ndarray | None,
+        orig_w: int,
+        orig_h: int,
+        w: int,
+        h: int,
+    ) -> np.ndarray | None:
+        if intrinsic is None:
+            return None
+        K = intrinsic.copy()
+        # scale fx, cx by w ratio; fy, cy by h ratio
+        K[:1] *= w / float(orig_w)
+        K[1:2] *= h / float(orig_h)
+        return K
+
+    def _crop_ixt(
+        self,
+        intrinsic: np.ndarray | None,
+        orig_w: int,
+        orig_h: int,
+        w: int,
+        h: int,
+    ) -> np.ndarray | None:
+        if intrinsic is None:
+            return None
+        K = intrinsic.copy()
+        crop_h = (orig_h - h) // 2
+        crop_w = (orig_w - w) // 2
+        K[0, 2] -= crop_w
+        K[1, 2] -= crop_h
+        return K
+
+    # -----------------------------
+    # I/O & normalization
+    # -----------------------------
+    def _load_image(self, img: np.ndarray | Image.Image | str) -> Image.Image:
+        if isinstance(img, str):
+            return Image.open(img).convert("RGB")
+        elif isinstance(img, np.ndarray):
+            # Assume HxWxC uint8/RGB
+            return Image.fromarray(img).convert("RGB")
+        elif isinstance(img, Image.Image):
+            return img.convert("RGB")
+        else:
+            raise ValueError(f"Unsupported image type: {type(img)}")
+
+    def _normalize_image(self, img: Image.Image) -> torch.Tensor:
+        img_tensor = T.ToTensor()(img)
+        return self.NORMALIZE(img_tensor)
+
+    # -----------------------------
+    # Boundary resizing
+    # -----------------------------
+    def _resize_image(self, img: Image.Image, target_size: int, method: str) -> Image.Image:
+        if method in ("upper_bound_resize", "upper_bound_crop"):
+            return self._resize_longest_side(img, target_size)
+        elif method in ("lower_bound_resize", "lower_bound_crop"):
+            return self._resize_shortest_side(img, target_size)
+        else:
+            raise ValueError(f"Unsupported resize method: {method}")
+
+    def _resize_longest_side(self, img: Image.Image, target_size: int) -> Image.Image:
+        w, h = img.size
+        longest = max(w, h)
+        if longest == target_size:
+            return img
+        scale = target_size / float(longest)
+        new_w = max(1, int(round(w * scale)))
+        new_h = max(1, int(round(h * scale)))
+        interpolation = cv2.INTER_CUBIC if scale > 1.0 else cv2.INTER_AREA
+        arr = cv2.resize(np.asarray(img), (new_w, new_h), interpolation=interpolation)
+        return Image.fromarray(arr)
+
+    def _resize_shortest_side(self, img: Image.Image, target_size: int) -> Image.Image:
+        w, h = img.size
+        shortest = min(w, h)
+        if shortest == target_size:
+            return img
+        scale = target_size / float(shortest)
+        new_w = max(1, int(round(w * scale)))
+        new_h = max(1, int(round(h * scale)))
+        interpolation = cv2.INTER_CUBIC if scale > 1.0 else cv2.INTER_AREA
+        arr = cv2.resize(np.asarray(img), (new_w, new_h), interpolation=interpolation)
+        return Image.fromarray(arr)
+
+    # -----------------------------
+    # Make divisible by PATCH_SIZE
+    # -----------------------------
+    def _make_divisible_by_crop(self, img: Image.Image, patch: int) -> Image.Image:
+        """
+        Floor each dimension to the nearest multiple of PATCH_SIZE via center crop.
+        Example: 504x377 -> 504x364
+        """
+        w, h = img.size
+        new_w = (w // patch) * patch
+        new_h = (h // patch) * patch
+        if new_w == w and new_h == h:
+            return img
+        left = (w - new_w) // 2
+        top = (h - new_h) // 2
+        return img.crop((left, top, left + new_w, top + new_h))
+
+    def _make_divisible_by_resize(self, img: Image.Image, patch: int) -> Image.Image:
+        """
+        Round each dimension to nearest multiple of PATCH_SIZE via small resize.
+        """
+        w, h = img.size
+
+        def nearest_multiple(x: int, p: int) -> int:
+            down = (x // p) * p
+            up = down + p
+            return up if abs(up - x) <= abs(x - down) else down
+
+        new_w = max(1, nearest_multiple(w, patch))
+        new_h = max(1, nearest_multiple(h, patch))
+        if new_w == w and new_h == h:
+            return img
+        upscale = (new_w > w) or (new_h > h)
+        interpolation = cv2.INTER_CUBIC if upscale else cv2.INTER_AREA
+        arr = cv2.resize(np.asarray(img), (new_w, new_h), interpolation=interpolation)
+        return Image.fromarray(arr)
+
+
+# Backward compatibility alias
+InputAdapter = InputProcessor
+
+
+# ===========================
+# Minimal test runner (parallel execution)
+# ===========================
+if __name__ == "__main__":
+    """
+    Minimal test suite:
+      - Creates pairs of images so batch shapes match.
+      - Tests all four process_res_methods.
+      - Prints fx fy cx cy IN->OUT per image.
+      - Includes cases with K/E provided and with None.
+    """
+
+    def fmt_k_line(K: np.ndarray | None) -> str:
+        if K is None:
+            return "None"
+        fx, fy, cx, cy = float(K[0, 0]), float(K[1, 1]), float(K[0, 2]), float(K[1, 2])
+        return f"fx={fx:.3f} fy={fy:.3f} cx={cx:.3f} cy={cy:.3f}"
+
+    def show_result(
+        tag: str,
+        tensor: torch.Tensor,
+        Ks_in: Sequence[np.ndarray | None] | None = None,
+        Ks_out: Sequence[np.ndarray | None] | None = None,
+    ):
+        B, N, C, H, W = tensor.shape
+        print(f"[{tag}] shape={tuple(tensor.shape)}  HxW=({H},{W})  div14=({H%14==0},{W%14==0})")
+        assert H % 14 == 0 and W % 14 == 0, f"{tag}: output size not divisible by 14!"
+        if Ks_in is not None or Ks_out is not None:
+            Ks_in = Ks_in or [None] * N
+            Ks_out = Ks_out or [None] * N
+            for i in range(N):
+                print(f"  K[{i}]: {fmt_k_line(Ks_in[i])}  ->  {fmt_k_line(Ks_out[i])}")
+
+    proc = InputProcessor()
+    process_res = 504
+    methods = ["upper_bound_resize", "upper_bound_crop", "lower_bound_resize", "lower_bound_crop"]
+
+    # Example sizes (two orientations)
+    small_sizes = [(680, 1208), (1208, 680)]
+    large_sizes = [(1208, 680), (680, 1208)]
+
+    def make_K(w, h, fx=1200.0, fy=1100.0):
+        cx, cy = w / 2.0, h / 2.0
+        K = np.array([[fx, 0, cx], [0, fy, cy], [0, 0, 1]], dtype=np.float32)
+        return K
+
+    def run_suite(suite_name: str, sizes: list[tuple[int, int]]):
+        print(f"\n===== {suite_name} =====")
+        for w, h in sizes:
+            img = Image.new("RGB", (w, h), color=(123, 222, 100))
+            batch_imgs = [img, img]
+
+            # intrinsics / extrinsics examples
+            Ks_in = [make_K(w, h), make_K(w, h)]
+            Es_in = [np.eye(4, dtype=np.float32), np.eye(4, dtype=np.float32)]
+
+            for m in methods:
+                tensor, Es_out, Ks_out = proc(
+                    image=batch_imgs,
+                    process_res=process_res,
+                    process_res_method=m,
+                    num_workers=8,
+                    print_progress=False,
+                    intrinsics=Ks_in,  # test with non-None
+                    extrinsics=Es_in,
+                )
+                show_result(f"{suite_name} size=({w},{h}) | {m}", tensor, Ks_in, Ks_out)
+
+            # Also test None path
+            tensor2, Es_out2, Ks_out2 = proc(
+                image=batch_imgs,
+                process_res=process_res,
+                process_res_method="upper_bound_resize",
+                num_workers=8,
+                intrinsics=None,
+                extrinsics=None,
+            )
+            show_result(
+                f"{suite_name} size=({w},{h}) | upper_bound_resize | no K/E",
+                tensor2,
+                None,
+                Ks_out2,
+            )
+
+    run_suite("SMALL", small_sizes)
+    run_suite("LARGE", large_sizes)
+
+    # Extra sanity for 504x376
+    print("\n===== EXTRA sanity for 504x376 =====")
+    img_example = Image.new("RGB", (504, 376), color=(10, 20, 30))
+    Ks_in_extra = [make_K(504, 376, fx=900.0, fy=900.0), make_K(504, 376, fx=900.0, fy=900.0)]
+
+    out_r, _, Ks_out_r = proc(
+        image=[img_example, img_example],
+        process_res=504,
+        process_res_method="upper_bound_resize",
+        num_workers=8,
+        intrinsics=Ks_in_extra,
+    )
+    out_c, _, Ks_out_c = proc(
+        image=[img_example, img_example],
+        process_res=504,
+        process_res_method="upper_bound_crop",
+        num_workers=8,
+        intrinsics=Ks_in_extra,
+    )
+    _, _, _, Hr, Wr = out_r.shape
+    _, _, _, Hc, Wc = out_c.shape
+    print(f"upper_bound_resize -> ({Hr},{Wr})  (rounded to nearest multiple of 14)")
+    show_result("Ks after upper_bound_resize", out_r, Ks_in_extra, Ks_out_r)
+    print(f"upper_bound_crop   -> ({Hc},{Wc})  (floored to multiple of 14)")
+    show_result("Ks after upper_bound_crop", out_c, Ks_in_extra, Ks_out_c)

multivue/cameras/source/src/depth_anything_3/utils/io/output_processor.py

@@ -0,0 +1,172 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Output processor for Depth Anything 3.
+
+This module handles model output processing, including tensor-to-numpy conversion,
+batch dimension removal, and Prediction object creation.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import torch
+from addict import Dict as AddictDict
+
+from depth_anything_3.specs import Prediction
+
+
+class OutputProcessor:
+    """
+    Output processor for converting model outputs to Prediction objects.
+
+    Handles tensor-to-numpy conversion, batch dimension removal,
+    and creates structured Prediction objects with proper data types.
+    """
+
+    def __init__(self) -> None:
+        """Initialize the output processor."""
+
+    def __call__(self, model_output: dict[str, torch.Tensor]) -> Prediction:
+        """
+        Convert model output to Prediction object.
+
+        Args:
+            model_output: Model output dictionary containing depth, conf, extrinsics, intrinsics
+                         Expected shapes: depth (B, N, 1, H, W), conf (B, N, 1, H, W),
+                         extrinsics (B, N, 4, 4), intrinsics (B, N, 3, 3)
+
+        Returns:
+            Prediction: Object containing depth estimation results with shapes:
+                       depth (N, H, W), conf (N, H, W), extrinsics (N, 4, 4), intrinsics (N, 3, 3)
+        """
+        # Extract data from batch dimension (B=1, N=number of images)
+        depth = self._extract_depth(model_output)
+        conf = self._extract_conf(model_output)
+        extrinsics = self._extract_extrinsics(model_output)
+        intrinsics = self._extract_intrinsics(model_output)
+        sky = self._extract_sky(model_output)
+        aux = self._extract_aux(model_output)
+        gaussians = model_output.get("gaussians", None)
+        scale_factor = model_output.get("scale_factor", None)
+
+        return Prediction(
+            depth=depth,
+            sky=sky,
+            conf=conf,
+            extrinsics=extrinsics,
+            intrinsics=intrinsics,
+            is_metric=getattr(model_output, "is_metric", 0),
+            gaussians=gaussians,
+            aux=aux,
+            scale_factor=scale_factor,
+        )
+
+    def _extract_depth(self, model_output: dict[str, torch.Tensor]) -> np.ndarray:
+        """
+        Extract depth tensor from model output and convert to numpy.
+
+        Args:
+            model_output: Model output dictionary
+
+        Returns:
+            Depth array with shape (N, H, W)
+        """
+        depth = model_output["depth"].squeeze(0).squeeze(-1).cpu().numpy()  # (N, H, W)
+        return depth
+
+    def _extract_conf(self, model_output: dict[str, torch.Tensor]) -> np.ndarray | None:
+        """
+        Extract confidence tensor from model output and convert to numpy.
+
+        Args:
+            model_output: Model output dictionary
+
+        Returns:
+            Confidence array with shape (N, H, W) or None
+        """
+        conf = model_output.get("depth_conf", None)
+        if conf is not None:
+            conf = conf.squeeze(0).cpu().numpy()  # (N, H, W)
+        return conf
+
+    def _extract_extrinsics(self, model_output: dict[str, torch.Tensor]) -> np.ndarray | None:
+        """
+        Extract extrinsics tensor from model output and convert to numpy.
+
+        Args:
+            model_output: Model output dictionary
+
+        Returns:
+            Extrinsics array with shape (N, 4, 4) or None
+        """
+        extrinsics = model_output.get("extrinsics", None)
+        if extrinsics is not None:
+            extrinsics = extrinsics.squeeze(0).cpu().numpy()  # (N, 4, 4)
+        return extrinsics
+
+    def _extract_intrinsics(self, model_output: dict[str, torch.Tensor]) -> np.ndarray | None:
+        """
+        Extract intrinsics tensor from model output and convert to numpy.
+
+        Args:
+            model_output: Model output dictionary
+
+        Returns:
+            Intrinsics array with shape (N, 3, 3) or None
+        """
+        intrinsics = model_output.get("intrinsics", None)
+        if intrinsics is not None:
+            intrinsics = intrinsics.squeeze(0).cpu().numpy()  # (N, 3, 3)
+        return intrinsics
+
+    def _extract_sky(self, model_output: dict[str, torch.Tensor]) -> np.ndarray | None:
+        """
+        Extract sky tensor from model output and convert to numpy.
+
+        Args:
+            model_output: Model output dictionary
+
+        Returns:
+            Sky mask array with shape (N, H, W) or None
+        """
+        sky = model_output.get("sky", None)
+        if sky is not None:
+            sky = sky.squeeze(0).cpu().numpy() >= 0.5  # (N, H, W)
+        return sky
+
+    def _extract_aux(self, model_output: dict[str, torch.Tensor]) -> AddictDict:
+        """
+        Extract auxiliary data from model output and convert to numpy.
+
+        Args:
+            model_output: Model output dictionary
+
+        Returns:
+            Dictionary containing auxiliary data
+        """
+        aux = model_output.get("aux", None)
+        ret = AddictDict()
+        if aux is not None:
+            for k in aux.keys():
+                if isinstance(aux[k], torch.Tensor):
+                    ret[k] = aux[k].squeeze(0).cpu().numpy()
+                else:
+                    ret[k] = aux[k]
+        return ret
+
+
+# Backward compatibility alias
+OutputAdapter = OutputProcessor

multivue/cameras/source/src/depth_anything_3/utils/layout_helpers.py

@@ -0,0 +1,216 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This file contains useful layout utilities for images. They are:
+
+- add_border: Add a border to an image.
+- cat/hcat/vcat: Join images by arranging them in a line. If the images have different
+  sizes, they are aligned as specified (start, end, center). Allows you to specify a gap
+  between images.
+
+Images are assumed to be float32 tensors with shape (channel, height, width).
+"""
+
+from typing import Any, Generator, Iterable, Literal, Union
+import torch
+from torch import Tensor
+
+Alignment = Literal["start", "center", "end"]
+Axis = Literal["horizontal", "vertical"]
+Color = Union[
+    int,
+    float,
+    Iterable[int],
+    Iterable[float],
+    Tensor,
+    Tensor,
+]
+
+
+def _sanitize_color(color: Color) -> Tensor:  # "#channel"
+    # Convert tensor to list (or individual item).
+    if isinstance(color, torch.Tensor):
+        color = color.tolist()
+
+    # Turn iterators and individual items into lists.
+    if isinstance(color, Iterable):
+        color = list(color)
+    else:
+        color = [color]
+
+    return torch.tensor(color, dtype=torch.float32)
+
+
+def _intersperse(iterable: Iterable, delimiter: Any) -> Generator[Any, None, None]:
+    it = iter(iterable)
+    yield next(it)
+    for item in it:
+        yield delimiter
+        yield item
+
+
+def _get_main_dim(main_axis: Axis) -> int:
+    return {
+        "horizontal": 2,
+        "vertical": 1,
+    }[main_axis]
+
+
+def _get_cross_dim(main_axis: Axis) -> int:
+    return {
+        "horizontal": 1,
+        "vertical": 2,
+    }[main_axis]
+
+
+def _compute_offset(base: int, overlay: int, align: Alignment) -> slice:
+    assert base >= overlay
+    offset = {
+        "start": 0,
+        "center": (base - overlay) // 2,
+        "end": base - overlay,
+    }[align]
+    return slice(offset, offset + overlay)
+
+
+def overlay(
+    base: Tensor,  # "channel base_height base_width"
+    overlay: Tensor,  # "channel overlay_height overlay_width"
+    main_axis: Axis,
+    main_axis_alignment: Alignment,
+    cross_axis_alignment: Alignment,
+) -> Tensor:  # "channel base_height base_width"
+    # The overlay must be smaller than the base.
+    _, base_height, base_width = base.shape
+    _, overlay_height, overlay_width = overlay.shape
+    assert base_height >= overlay_height and base_width >= overlay_width
+
+    # Compute spacing on the main dimension.
+    main_dim = _get_main_dim(main_axis)
+    main_slice = _compute_offset(
+        base.shape[main_dim], overlay.shape[main_dim], main_axis_alignment
+    )
+
+    # Compute spacing on the cross dimension.
+    cross_dim = _get_cross_dim(main_axis)
+    cross_slice = _compute_offset(
+        base.shape[cross_dim], overlay.shape[cross_dim], cross_axis_alignment
+    )
+
+    # Combine the slices and paste the overlay onto the base accordingly.
+    selector = [..., None, None]
+    selector[main_dim] = main_slice
+    selector[cross_dim] = cross_slice
+    result = base.clone()
+    result[selector] = overlay
+    return result
+
+
+def cat(
+    main_axis: Axis,
+    *images: Iterable[Tensor],  # "channel _ _"
+    align: Alignment = "center",
+    gap: int = 8,
+    gap_color: Color = 1,
+) -> Tensor:  # "channel height width"
+    """Arrange images in a line. The interface resembles a CSS div with flexbox."""
+    device = images[0].device
+    gap_color = _sanitize_color(gap_color).to(device)
+
+    # Find the maximum image side length in the cross axis dimension.
+    cross_dim = _get_cross_dim(main_axis)
+    cross_axis_length = max(image.shape[cross_dim] for image in images)
+
+    # Pad the images.
+    padded_images = []
+    for image in images:
+        # Create an empty image with the correct size.
+        padded_shape = list(image.shape)
+        padded_shape[cross_dim] = cross_axis_length
+        base = torch.ones(padded_shape, dtype=torch.float32, device=device)
+        base = base * gap_color[:, None, None]
+        padded_images.append(overlay(base, image, main_axis, "start", align))
+
+    # Intersperse separators if necessary.
+    if gap > 0:
+        # Generate a separator.
+        c, _, _ = images[0].shape
+        separator_size = [gap, gap]
+        separator_size[cross_dim - 1] = cross_axis_length
+        separator = torch.ones((c, *separator_size), dtype=torch.float32, device=device)
+        separator = separator * gap_color[:, None, None]
+
+        # Intersperse the separator between the images.
+        padded_images = list(_intersperse(padded_images, separator))
+
+    return torch.cat(padded_images, dim=_get_main_dim(main_axis))
+
+
+def hcat(
+    *images: Iterable[Tensor],  # "channel _ _"
+    align: Literal["start", "center", "end", "top", "bottom"] = "start",
+    gap: int = 8,
+    gap_color: Color = 1,
+):
+    """Shorthand for a horizontal linear concatenation."""
+    return cat(
+        "horizontal",
+        *images,
+        align={
+            "start": "start",
+            "center": "center",
+            "end": "end",
+            "top": "start",
+            "bottom": "end",
+        }[align],
+        gap=gap,
+        gap_color=gap_color,
+    )
+
+
+def vcat(
+    *images: Iterable[Tensor],  # "channel _ _"
+    align: Literal["start", "center", "end", "left", "right"] = "start",
+    gap: int = 8,
+    gap_color: Color = 1,
+):
+    """Shorthand for a horizontal linear concatenation."""
+    return cat(
+        "vertical",
+        *images,
+        align={
+            "start": "start",
+            "center": "center",
+            "end": "end",
+            "left": "start",
+            "right": "end",
+        }[align],
+        gap=gap,
+        gap_color=gap_color,
+    )
+
+
+def add_border(
+    image: Tensor,  # "channel height width"
+    border: int = 8,
+    color: Color = 1,
+) -> Tensor:  # "channel new_height new_width"
+    color = _sanitize_color(color).to(image)
+    c, h, w = image.shape
+    result = torch.empty(
+        (c, h + 2 * border, w + 2 * border), dtype=torch.float32, device=image.device
+    )
+    result[:] = color[:, None, None]
+    result[:, border : h + border, border : w + border] = image
+    return result

multivue/cameras/source/src/depth_anything_3/utils/logger.py

@@ -0,0 +1,82 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+
+class Color:
+    RED = "\033[91m"
+    YELLOW = "\033[93m"
+    WHITE = "\033[97m"
+    GREEN = "\033[92m"
+    RESET = "\033[0m"
+
+
+LOG_LEVELS = {"ERROR": 0, "WARN": 1, "INFO": 2, "DEBUG": 3}
+
+COLOR_MAP = {"ERROR": Color.RED, "WARN": Color.YELLOW, "INFO": Color.WHITE, "DEBUG": Color.GREEN}
+
+
+def get_env_log_level():
+    level = os.environ.get("DA3_LOG_LEVEL", "INFO").upper()
+    return LOG_LEVELS.get(level, LOG_LEVELS["INFO"])
+
+
+class Logger:
+    def __init__(self):
+        self.level = get_env_log_level()
+
+    def log(self, level_str, *args, **kwargs):
+        level_key = level_str.split(":")[0].strip()
+        level_val = LOG_LEVELS.get(level_key)
+        if level_val is None:
+            raise ValueError(f"Unknown log level: {level_str}")
+        if self.level >= level_val:
+            color = COLOR_MAP[level_key]
+            msg = " ".join(str(arg) for arg in args)
+
+            # Align log level output in square brackets
+            # ERROR and DEBUG are 5 characters, INFO and WARN have an extra space for alignment
+            tag = level_key
+            if tag in ("INFO", "WARN"):
+                tag += " "
+            print(
+                f"{color}[{tag}] {msg}{Color.RESET}",
+                file=sys.stderr if level_key == "ERROR" else sys.stdout,
+                **kwargs,
+            )
+
+    def error(self, *args, **kwargs):
+        self.log("ERROR:", *args, **kwargs)
+
+    def warn(self, *args, **kwargs):
+        self.log("WARN:", *args, **kwargs)
+
+    def info(self, *args, **kwargs):
+        self.log("INFO:", *args, **kwargs)
+
+    def debug(self, *args, **kwargs):
+        self.log("DEBUG:", *args, **kwargs)
+
+
+logger = Logger()
+
+__all__ = ["logger"]
+
+if __name__ == "__main__":
+    logger.info("This is an info message")
+    logger.warn("This is a warning message")
+    logger.error("This is an error message")
+    logger.debug("This is a debug message")

multivue/cameras/source/src/depth_anything_3/utils/memory.py

@@ -0,0 +1,127 @@
+"""
+GPU memory utility helpers.
+
+Shared cleanup and memory checking logic used by both the backend API and
+the Gradio UI to keep memory-management behavior consistent.
+"""
+from __future__ import annotations
+
+import gc
+
+from typing import Any, Dict, Optional
+
+import torch
+
+
+def get_gpu_memory_info() -> Optional[Dict[str, Any]]:
+    """Return a snapshot of current GPU memory usage or None if CUDA not available.
+
+    Keys in returned dict: total_gb, allocated_gb, reserved_gb, free_gb, utilization
+    """
+    if not torch.cuda.is_available():
+        return None
+
+    try:
+        device = torch.cuda.current_device()
+        total_memory = torch.cuda.get_device_properties(device).total_memory
+        allocated_memory = torch.cuda.memory_allocated(device)
+        reserved_memory = torch.cuda.memory_reserved(device)
+        free_memory = total_memory - reserved_memory
+
+        return {
+            "total_gb": total_memory / 1024 ** 3,
+            "allocated_gb": allocated_memory / 1024 ** 3,
+            "reserved_gb": reserved_memory / 1024 ** 3,
+            "free_gb": free_memory / 1024 ** 3,
+            "utilization": (reserved_memory / total_memory) * 100,
+        }
+    except Exception:
+        return None
+
+
+def cleanup_cuda_memory() -> None:
+    """Perform a robust GPU cleanup sequence.
+
+    This includes synchronizing, emptying caches, collecting IPC handles and
+    running the Python garbage collector. Use this instead of a raw
+    ``torch.cuda.empty_cache()`` where you need reliable freeing of GPU memory
+    between model loads or in error handling paths.
+    """
+    try:
+        if torch.cuda.is_available():
+            mem_before = get_gpu_memory_info()
+
+            torch.cuda.synchronize()
+            torch.cuda.empty_cache()
+            # Collect cross-process cuda resources
+            try:
+                torch.cuda.ipc_collect()
+            except Exception:
+                # Older PyTorch versions or non-cuda devices may not support
+                # ipc_collect (no-op if not available)
+                pass
+            gc.collect()
+
+            mem_after = get_gpu_memory_info()
+            if mem_before and mem_after:
+                freed = mem_before["reserved_gb"] - mem_after["reserved_gb"]
+                print(
+                    f"CUDA cleanup: freed {freed:.2f}GB, "
+                    f"available: {mem_after['free_gb']:.2f}GB/{mem_after['total_gb']:.2f}GB"
+                )
+            else:
+                print("CUDA memory cleanup completed")
+    except Exception as e:
+        print(f"Warning: CUDA cleanup failed: {e}")
+
+
+def check_memory_availability(required_gb: float = 2.0) -> tuple[bool, str]:
+    """Return whether at least ``required_gb`` seems available on the current GPU.
+
+    The returned tuple is (is_available, message) with a human-friendly message.
+    """
+    try:
+        if not torch.cuda.is_available():
+            return False, "CUDA is not available"
+
+        mem_info = get_gpu_memory_info()
+        if mem_info is None:
+            return True, "Cannot check memory, proceeding anyway"
+
+        if mem_info["free_gb"] < required_gb:
+            return (
+                False,
+                (
+                    f"Insufficient GPU memory: {mem_info['free_gb']:.2f}GB available, "
+                    f"{required_gb:.2f}GB required. Total: {mem_info['total_gb']:.2f}GB, "
+                    f"Used: {mem_info['reserved_gb']:.2f}GB ({mem_info['utilization']:.1f}%)"
+                ),
+            )
+
+        return (
+            True,
+            (
+                f"Memory check passed: {mem_info['free_gb']:.2f}GB available, "
+                f"{required_gb:.2f}GB required"
+            ),
+        )
+    except Exception as e:
+        return True, f"Memory check failed: {e}, proceeding anyway"
+def estimate_memory_requirement(num_images: int, process_res: int) -> float:
+    """Heuristic estimate for memory usage (GB) based on image count and resolution.
+
+    This mirrors the simple policy used by the backend service so other code
+    (e.g., Gradio UI) can make consistent decisions when checking available
+    memory before loading a model or running inference.
+
+    Args:
+        num_images: Number of images to process.
+        process_res: Processing resolution.
+
+    Returns:
+        Estimated memory requirement in GB.
+    """
+    base_memory = 2.0
+    per_image_memory = (process_res / 504) ** 2 * 0.5
+    total_memory = base_memory + (num_images * per_image_memory * 0.1)
+    return total_memory

multivue/cameras/source/src/depth_anything_3/utils/model_loading.py

@@ -0,0 +1,149 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Model loading and state dict conversion utilities.
+"""
+
+from typing import Dict, Tuple
+import torch
+
+from depth_anything_3.utils.logger import logger
+
+
+def convert_general_state_dict(state_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
+    """
+    Convert general model state dict to match current model architecture.
+
+    Args:
+        state_dict: Original state dictionary
+
+    Returns:
+        Converted state dictionary
+    """
+    # Replace module prefixes
+    state_dict = {k.replace("module.", "model."): v for k, v in state_dict.items()}
+    state_dict = {k.replace(".net.", ".backbone."): v for k, v in state_dict.items()}
+
+    # Remove camera token if present
+    if "model.backbone.pretrained.camera_token" in state_dict:
+        del state_dict["model.backbone.pretrained.camera_token"]
+
+    # Replace camera token naming
+    state_dict = {
+        k.replace(".camera_token_extra", ".camera_token"): v for k, v in state_dict.items()
+    }
+
+    # Replace head naming
+    state_dict = {
+        k.replace("model.all_heads.camera_cond_head", "model.cam_enc"): v
+        for k, v in state_dict.items()
+    }
+    state_dict = {
+        k.replace("model.all_heads.camera_head", "model.cam_dec"): v for k, v in state_dict.items()
+    }
+    state_dict = {k.replace(".more_mlps.", ".backbone."): v for k, v in state_dict.items()}
+    state_dict = {k.replace(".fc_rot.", ".fc_qvec."): v for k, v in state_dict.items()}
+    state_dict = {
+        k.replace("model.all_heads.head", "model.head"): v for k, v in state_dict.items()
+    }
+
+    # Replace output naming
+    state_dict = {
+        k.replace("output_conv2_additional.sky_mask", "sky_output_conv2"): v
+        for k, v in state_dict.items()
+    }
+    state_dict = {k.replace("_ray.", "_aux."): v for k, v in state_dict.items()}
+
+    # Update GS-DPT head naming and value
+    state_dict = {k.replace("gaussian_param_head.", "gs_head."): v for k, v in state_dict.items()}
+
+    return state_dict
+
+
+def convert_metric_state_dict(state_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
+    """
+    Convert metric model state dict to match current model architecture.
+
+    Args:
+        state_dict: Original metric state dictionary
+
+    Returns:
+        Converted state dictionary
+    """
+    # Add module prefix for metric models
+    state_dict = {"module." + k: v for k, v in state_dict.items()}
+    return convert_general_state_dict(state_dict)
+
+
+def load_pretrained_weights(model, model_path: str, is_metric: bool = False) -> Tuple[list, list]:
+    """
+    Load pretrained weights for a single model.
+
+    Args:
+        model: Model instance to load weights into
+        model_path: Path to the pretrained weights
+        is_metric: Whether this is a metric model
+
+    Returns:
+        Tuple of (missed_keys, unexpected_keys)
+    """
+    state_dict = torch.load(model_path, map_location="cpu")
+
+    if is_metric:
+        state_dict = convert_metric_state_dict(state_dict)
+    else:
+        state_dict = convert_general_state_dict(state_dict)
+
+    missed, unexpected = model.load_state_dict(state_dict, strict=False)
+    logger.info("Missed keys:", missed)
+    logger.info("Unexpected keys:", unexpected)
+
+    return missed, unexpected
+
+
+def load_pretrained_nested_weights(
+    model, main_model_path: str, metric_model_path: str
+) -> Tuple[list, list]:
+    """
+    Load pretrained weights for a nested model with both main and metric branches.
+
+    Args:
+        model: Nested model instance
+        main_model_path: Path to main model weights
+        metric_model_path: Path to metric model weights
+
+    Returns:
+        Tuple of (missed_keys, unexpected_keys)
+    """
+    # Load main model weights
+    state_dict0 = torch.load(main_model_path, map_location="cpu")
+    state_dict0 = convert_general_state_dict(state_dict0)
+    state_dict0 = {k.replace("model.", "model.da3."): v for k, v in state_dict0.items()}
+
+    # Load metric model weights
+    state_dict1 = torch.load(metric_model_path, map_location="cpu")
+    state_dict1 = convert_metric_state_dict(state_dict1)
+    state_dict1 = {k.replace("model.", "model.da3_metric."): v for k, v in state_dict1.items()}
+
+    # Combine state dictionaries
+    combined_state_dict = state_dict0.copy()
+    combined_state_dict.update(state_dict1)
+
+    missed, unexpected = model.load_state_dict(combined_state_dict, strict=False)
+
+    print("Missed keys:", missed)
+    print("Unexpected keys:", unexpected)
+
+    return missed, unexpected

multivue/cameras/source/src/depth_anything_3/utils/parallel_utils.py

@@ -0,0 +1,133 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import os
+from functools import wraps
+from multiprocessing.pool import ThreadPool
+from threading import Thread
+from typing import Callable, Dict, List
+import imageio
+from tqdm import tqdm
+
+
+def async_call_func(func):
+    @wraps(func)
+    async def wrapper(*args, **kwargs):
+        loop = asyncio.get_event_loop()
+        # Use run_in_executor to run the blocking function in a separate thread
+        return await loop.run_in_executor(None, func, *args, **kwargs)
+
+    return wrapper
+
+
+slice_func = lambda chunk_index, chunk_dim, chunk_size: [slice(None)] * chunk_dim + [
+    slice(chunk_index, chunk_index + chunk_size)
+]
+
+
+def async_call(fn):
+    def wrapper(*args, **kwargs):
+        Thread(target=fn, args=args, kwargs=kwargs).start()
+
+    return wrapper
+
+
+def _save_image_impl(save_img, save_path):
+    """Common implementation for saving images synchronously or asynchronously"""
+    os.makedirs(os.path.dirname(save_path), exist_ok=True)
+    imageio.imwrite(save_path, save_img)
+
+
+@async_call
+def save_image_async(save_img, save_path):
+    """Save image asynchronously"""
+    _save_image_impl(save_img, save_path)
+
+
+def save_image(save_img, save_path):
+    """Save image synchronously"""
+    _save_image_impl(save_img, save_path)
+
+
+def parallel_execution(
+    *args,
+    action: Callable,
+    num_processes=32,
+    print_progress=False,
+    sequential=False,
+    async_return=False,
+    desc=None,
+    **kwargs,
+):
+    # Partially copy from EasyVolumetricVideo (parallel_execution)
+    # NOTE: we expect first arg / or kwargs to be distributed
+    # NOTE: print_progress arg is reserved.
+    # `*args` packs all positional arguments passed to the function into a tuple
+    args = list(args)
+
+    def get_length(args: List, kwargs: Dict):
+        for a in args:
+            if isinstance(a, list):
+                return len(a)
+        for v in kwargs.values():
+            if isinstance(v, list):
+                return len(v)
+        raise NotImplementedError
+
+    def get_action_args(length: int, args: List, kwargs: Dict, i: int):
+        action_args = [
+            (arg[i] if isinstance(arg, list) and len(arg) == length else arg) for arg in args
+        ]
+        # TODO: Support all types of iterable
+        action_kwargs = {
+            key: (
+                kwargs[key][i]
+                if isinstance(kwargs[key], list) and len(kwargs[key]) == length
+                else kwargs[key]
+            )
+            for key in kwargs
+        }
+        return action_args, action_kwargs
+
+    if not sequential:
+        # Create ThreadPool
+        pool = ThreadPool(processes=num_processes)
+
+        # Spawn threads
+        results = []
+        asyncs = []
+        length = get_length(args, kwargs)
+        for i in range(length):
+            action_args, action_kwargs = get_action_args(length, args, kwargs, i)
+            async_result = pool.apply_async(action, action_args, action_kwargs)
+            asyncs.append(async_result)
+
+        # Join threads and get return values
+        if not async_return:
+            for async_result in tqdm(asyncs, desc=desc, disable=not print_progress):
+                results.append(async_result.get())  # will sync the corresponding thread
+            pool.close()
+            pool.join()
+            return results
+        else:
+            return pool
+    else:
+        results = []
+        length = get_length(args, kwargs)
+        for i in tqdm(range(length), desc=desc, disable=not print_progress):
+            action_args, action_kwargs = get_action_args(length, args, kwargs, i)
+            async_result = action(*action_args, **action_kwargs)
+            results.append(async_result)
+        return results

multivue/cameras/source/src/depth_anything_3/utils/pca_utils.py

@@ -0,0 +1,284 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+PCA utilities for feature visualization and dimensionality reduction (video-friendly).
+- Support frame-by-frame: transform_frame / transform_video
+- Support one-time global PCA fitting and reuse (mean, V3) for stable colors
+- Support Procrustes alignment (solving principal component order/sign/rotation jumps)
+- Support global fixed or temporal EMA for percentiles (time dimension only, no spatial)
+"""
+
+import numpy as np
+import torch
+
+
+def pca_to_rgb_4d_bf16_percentile(
+    x_np: np.ndarray,
+    device=None,
+    q_oversample: int = 6,
+    clip_percent: float = 10.0,  # Percentage to clip from top and bottom (0~49.9)
+    return_uint8: bool = False,
+    enable_autocast_bf16: bool = True,
+):
+    """
+    Reduce numpy array of shape (49, 27, 36, 3072) to 3D via PCA and visualize as (49, 27, 36, 3).
+    - PCA uses torch.pca_lowrank (randomized SVD), defaults to GPU.
+    - Uses CUDA bf16 autocast in computation (if available),
+      then per-channel percentile clipping and normalization.
+    - Default removes 5% outliers from top and bottom (adjustable via clip_percent) to
+      improve visualization contrast.
+
+    Parameters
+    ----------
+    x_np : np.ndarray
+        Shape must be (49, 27, 36, 3072). dtype recommended float32/float64.
+    device : str | None
+        Specify 'cuda' or 'cpu'. Auto-select if None (prefer cuda).
+    q_oversample : int
+        Oversampling q for pca_lowrank, must be >= 3.
+        Slightly larger than target dim (3) is more stable, default 6.
+    clip_percent : float
+        Percentage to clip from top and bottom (0~49.9),
+        e.g. 5.0 means clip lowest 5% and highest 5% per channel.
+    return_uint8 : bool
+        True returns uint8(0~255), otherwise returns float32(0~1).
+    enable_autocast_bf16 : bool
+        Enable bf16 autocast on CUDA.
+
+    Returns
+    -------
+    np.ndarray
+        Array of shape (49, 27, 36, 3), float32[0,1] or uint8[0,255].
+    """
+    assert (
+        x_np.ndim == 4
+    )  # and x_np.shape[-1] == 3072, f"expect (49,27,36,3072), got {x_np.shape}"
+    B1, B2, B3, D = x_np.shape
+    N = B1 * B2 * B3
+
+    # Device selection
+    if device is None:
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    # Convert input to torch, unified float32
+    X = torch.from_numpy(x_np.reshape(N, D)).to(device=device, dtype=torch.float32)
+
+    # Parameter and safety checks
+    k = 3
+    q = max(int(q_oversample), k)
+    clip_percent = float(clip_percent)
+    if not (0.0 <= clip_percent < 50.0):
+        raise ValueError(
+            "clip_percent must be in [0, 50), e.g. 5.0 means clip 5% from top and bottom"
+        )
+    low = clip_percent / 100.0
+    high = 1.0 - low
+
+    with torch.no_grad():
+        # Zero mean
+        mean = X.mean(dim=0, keepdim=True)
+        Xc = X - mean
+
+        # Main computation: PCA + projection, try to use bf16
+        # (auto-fallback if operator not supported)
+        device.startswith("cuda") and enable_autocast_bf16
+        U, S, V = torch.pca_lowrank(Xc, q=q, center=False)  # V: (D, q)
+        V3 = V[:, :k]  # (3072, 3)
+        PCs = Xc @ V3  # (N, 3)
+
+        # === Per-channel percentile clipping and normalization to [0,1] ===
+        # Vectorized one-time calculation of low/high percentiles for each channel
+        qs = torch.tensor([low, high], device=PCs.device, dtype=PCs.dtype)
+        qvals = torch.quantile(PCs, q=qs, dim=0)  # Shape (2, 3)
+        lo = qvals[0]  # (3,)
+        hi = qvals[1]  # (3,)
+
+        # Avoid degenerate case where hi==lo
+        denom = torch.clamp(hi - lo, min=1e-8)
+
+        # Broadcast clipping + normalization
+        PCs = torch.clamp(PCs, lo, hi)
+        PCs = (PCs - lo) / denom  # (N, 3) in [0,1]
+
+        # Restore 4D
+        PCs = PCs.reshape(B1, B2, B3, k)
+
+        # Output
+        if return_uint8:
+            out = (PCs * 255.0).round().clamp(0, 255).to(torch.uint8).cpu().numpy()
+        else:
+            out = PCs.clamp(0, 1).to(torch.float32).cpu().numpy()
+
+    return out
+
+
+class PCARGBVisualizer:
+    """
+    Stable PCA→RGB for video features shaped (T, H, W, D) or a single frame (H, W, D).
+    - Global mean/V3 reference for stable colors
+    - Per-frame PCA with Procrustes alignment to V3_ref (basis_mode='procrustes')
+    - Percentile normalization with global or EMA stats (time-only, no spatial smoothing)
+    """
+
+    def __init__(
+        self,
+        device=None,
+        q_oversample: int = 16,
+        clip_percent: float = 10.0,
+        return_uint8: bool = False,
+        enable_autocast_bf16: bool = True,
+        basis_mode: str = "procrustes",  # 'fixed' | 'procrustes'
+        percentile_mode: str = "ema",  # 'global' | 'ema'
+        ema_alpha: float = 0.1,
+        denom_eps: float = 1e-4,
+    ):
+        assert 0.0 <= clip_percent < 50.0
+        assert basis_mode in ("fixed", "procrustes")
+        assert percentile_mode in ("global", "ema")
+        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
+        self.q = max(int(q_oversample), 6)
+        self.clip_percent = float(clip_percent)
+        self.return_uint8 = return_uint8
+        self.enable_autocast_bf16 = enable_autocast_bf16
+        self.basis_mode = basis_mode
+        self.percentile_mode = percentile_mode
+        self.ema_alpha = float(ema_alpha)
+        self.denom_eps = float(denom_eps)
+
+        # reference state
+        self.mean_ref = None  # (1, D)
+        self.V3_ref = None  # (D, 3)
+        self.lo_ref = None  # (3,)
+        self.hi_ref = None  # (3,)
+
+    @torch.no_grad()
+    def fit_reference(self, frames):
+        """
+        Fit global mean/V3 and initialize percentiles from a reference set.
+        frames: ndarray (T,H,W,D) or list of (H,W,D)
+        """
+        if isinstance(frames, np.ndarray):
+            if frames.ndim != 4:
+                raise ValueError("fit_reference expects (T,H,W,D) ndarray.")
+            T, H, W, D = frames.shape
+            X = torch.from_numpy(frames.reshape(T * H * W, D))
+        else:  # list of (H,W,D)
+            xs = [torch.from_numpy(x.reshape(-1, x.shape[-1])) for x in frames]
+            D = xs[0].shape[-1]
+            X = torch.cat(xs, dim=0)
+
+        X = X.to(self.device, dtype=torch.float32)
+        X = torch.nan_to_num(X, nan=0.0, posinf=1e6, neginf=-1e6)
+
+        mean = X.mean(0, keepdim=True)
+        Xc = X - mean
+
+        U, S, V = torch.pca_lowrank(Xc, q=max(self.q, 8), center=False)
+        V3 = V[:, :3]  # (D,3)
+
+        PCs = Xc @ V3
+        low = self.clip_percent / 100.0
+        high = 1.0 - low
+        qs = torch.tensor([low, high], device=PCs.device, dtype=PCs.dtype)
+        qvals = torch.quantile(PCs, q=qs, dim=0)
+        lo, hi = qvals[0], qvals[1]
+
+        self.mean_ref = mean
+        self.V3_ref = V3
+        if self.percentile_mode == "global":
+            self.lo_ref, self.hi_ref = lo, hi
+        else:
+            self.lo_ref = lo.clone()
+            self.hi_ref = hi.clone()
+
+    @torch.no_grad()
+    def _project_with_stable_colors(self, X: torch.Tensor) -> torch.Tensor:
+        """
+        X: (N,D) where N = H*W
+        Returns PCs_raw: (N,3) using stable basis (fixed or Procrustes-aligned)
+        """
+        assert self.mean_ref is not None and self.V3_ref is not None, "Call fit_reference() first."
+        X = torch.nan_to_num(X, nan=0.0, posinf=1e6, neginf=-1e6)
+        Xc = X - self.mean_ref
+
+        if self.basis_mode == "fixed":
+            V3_used = self.V3_ref
+        else:
+            U, S, V = torch.pca_lowrank(Xc, q=max(self.q, 6), center=False)
+            V3 = V[:, :3]  # (D,3)
+            M = V3.T @ self.V3_ref
+            Uo, So, Vh = torch.linalg.svd(M)
+            R = Uo @ Vh
+            V3_used = V3 @ R
+            # Optional polarity fix via anchor
+            a = self.V3_ref.mean(0, keepdim=True)
+            sign = torch.sign((V3_used * a).sum(0, keepdim=True)).clamp(min=-1)
+            V3_used = V3_used * sign
+
+        return Xc @ V3_used
+
+    @torch.no_grad()
+    def _normalize_rgb(self, PCs_raw: torch.Tensor) -> torch.Tensor:
+        assert self.lo_ref is not None and self.hi_ref is not None
+        if self.percentile_mode == "global":
+            lo, hi = self.lo_ref, self.hi_ref
+        else:
+            low = self.clip_percent / 100.0
+            high = 1.0 - low
+            qs = torch.tensor([low, high], device=PCs_raw.device, dtype=PCs_raw.dtype)
+            qvals = torch.quantile(PCs_raw, q=qs, dim=0)
+            lo_now, hi_now = qvals[0], qvals[1]
+            a = self.ema_alpha
+            self.lo_ref = (1 - a) * self.lo_ref + a * lo_now
+            self.hi_ref = (1 - a) * self.hi_ref + a * hi_now
+            lo, hi = self.lo_ref, self.hi_ref
+
+        denom = torch.clamp(hi - lo, min=self.denom_eps)
+        PCs = torch.clamp(PCs_raw, lo, hi)
+        PCs = (PCs - lo) / denom
+        return PCs.clamp_(0, 1)
+
+    @torch.no_grad()
+    def transform_frame(self, frame: np.ndarray) -> np.ndarray:
+        """
+        frame: (H,W,D) -> (H,W,3)
+        """
+        if frame.ndim != 3:
+            raise ValueError("transform_frame expects (H,W,D).")
+        H, W, D = frame.shape
+        X = torch.from_numpy(frame.reshape(H * W, D)).to(self.device, dtype=torch.float32)
+        PCs_raw = self._project_with_stable_colors(X)
+        PCs = self._normalize_rgb(PCs_raw).reshape(H, W, 3)
+        if self.return_uint8:
+            return (PCs * 255.0).round().clamp(0, 255).to(torch.uint8).cpu().numpy()
+        return PCs.to(torch.float32).cpu().numpy()
+
+    @torch.no_grad()
+    def transform_video(self, frames) -> np.ndarray:
+        """
+        frames: (T,H,W,D) or list of (H,W,D)
+        returns: (T,H,W,3)
+        """
+        outs = []
+        if isinstance(frames, np.ndarray):
+            if frames.ndim != 4:
+                raise ValueError("transform_video expects (T,H,W,D).")
+            T, H, W, D = frames.shape
+            for t in range(T):
+                outs.append(self.transform_frame(frames[t]))
+        else:
+            for f in frames:
+                outs.append(self.transform_frame(f))
+        return np.stack(outs, axis=0)

multivue/cameras/source/src/depth_anything_3/utils/pose_align.py

@@ -0,0 +1,347 @@
+# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+import numpy as np
+import torch
+from evo.core.trajectory import PosePath3D
+
+from depth_anything_3.utils.geometry import affine_inverse, affine_inverse_np
+
+
+def batch_apply_alignment_to_enc(
+    rots: torch.Tensor, trans: torch.Tensor, scales: torch.Tensor, enc_list: List[torch.Tensor]
+):
+    pass
+
+
+def batch_apply_alignment_to_ext(
+    rots: torch.Tensor, trans: torch.Tensor, scales: torch.Tensor, ext: torch.Tensor
+):
+    device, _ = ext.device, ext.dtype
+    if ext.shape[-2:] == (3, 4):
+        pad = torch.zeros((*ext.shape[:-2], 4, 4), dtype=ext.dtype, device=device)
+        pad[..., :3, :4] = ext
+        pad[..., 3, 3] = 1.0
+        ext = pad
+    pose_est = affine_inverse(ext)
+    pose_new_align_rot = rots[:, None] @ pose_est[..., :3, :3]
+    pose_new_align_trans = (
+        scales[:, None, None] * (rots[:, None] @ pose_est[..., :3, 3:])[..., 0] + trans[:, None]
+    )
+    pose_new_align = torch.zeros_like(ext)
+    pose_new_align[..., :3, :3] = pose_new_align_rot
+    pose_new_align[..., :3, 3] = pose_new_align_trans
+    pose_new_align[..., 3, 3] = 1.0
+    return affine_inverse(pose_new_align)[:, :3]
+
+
+def batch_align_poses_umeyama(ext_ref: torch.Tensor, ext_est: torch.Tensor):
+    device, dtype = ext_ref.device, ext_ref.dtype
+    assert ext_ref.dtype in [torch.float32, torch.float64]
+    assert ext_est.dtype in [torch.float32, torch.float64]
+    assert ext_ref.requires_grad is False
+    assert ext_est.requires_grad is False
+    rots, trans, scales = [], [], []
+    for b in range(ext_ref.shape[0]):
+        r, t, s = align_poses_umeyama(ext_ref[b].cpu().numpy(), ext_est[b].cpu().numpy())
+        rots.append(torch.from_numpy(r).to(device=device, dtype=dtype))
+        trans.append(torch.from_numpy(t).to(device=device, dtype=dtype))
+        scales.append(torch.tensor(s, device=device, dtype=dtype))
+    return torch.stack(rots), torch.stack(trans), torch.stack(scales)
+
+
+# Dependencies: affine_inverse_np, PosePath3D (maintain consistency with your existing project)
+
+
+def _to44(ext):
+    if ext.shape[1] == 3:
+        out = np.eye(4)[None].repeat(len(ext), 0)
+        out[:, :3, :4] = ext
+        return out
+    return ext
+
+
+def _poses_from_ext(ext_ref, ext_est):
+    ext_ref = _to44(ext_ref)
+    ext_est = _to44(ext_est)
+    pose_ref = affine_inverse_np(ext_ref)
+    pose_est = affine_inverse_np(ext_est)
+    return pose_ref, pose_est
+
+
+def _umeyama_sim3_from_paths(pose_ref, pose_est):
+    path_ref = PosePath3D(poses_se3=pose_ref.copy())
+    path_est = PosePath3D(poses_se3=pose_est.copy())
+    r, t, s = path_est.align(path_ref, correct_scale=True)
+    pose_est_aligned = np.stack(path_est.poses_se3)
+    return r, t, s, pose_est_aligned
+
+
+def _apply_sim3_to_poses(poses, r, t, s):
+    out = poses.copy()
+    Ri = poses[:, :3, :3]
+    ti = poses[:, :3, 3]
+    out[:, :3, :3] = r @ Ri
+    out[:, :3, 3] = (r @ (s * ti.T)).T + t
+    return out
+
+
+def _median_nn_thresh(pose_ref, pose_est_aligned):
+    P_ref = pose_ref[:, :3, 3]
+    P_est = pose_est_aligned[:, :3, 3]
+    dists = []
+    for p in P_est:
+        dd = np.linalg.norm(P_ref - p[None, :], axis=1)
+        dists.append(dd.min())
+    return float(np.median(dists)) if dists else 0.0
+
+
+def _ransac_align_sim3(
+    pose_ref, pose_est, sub_n=None, inlier_thresh=None, max_iters=10, random_state=None
+):
+    rng = np.random.default_rng(random_state)
+    N = pose_ref.shape[0]
+    idx_all = np.arange(N)
+    if sub_n is None:
+        sub_n = max(3, (N + 1) // 2)
+    else:
+        sub_n = max(3, min(sub_n, N))
+
+    # Pre-alignment + default threshold
+    r0, t0, s0, pose_est0 = _umeyama_sim3_from_paths(pose_ref, pose_est)
+    if inlier_thresh is None:
+        inlier_thresh = _median_nn_thresh(pose_ref, pose_est0)
+
+    P_ref_all = pose_ref[:, :3, 3]
+
+    best_model = (r0, t0, s0)
+    best_inliers = None
+    best_score = (-1, np.inf)  # (num_inliers, mean_err)
+
+    for _ in range(max_iters):
+        sample = rng.choice(idx_all, size=sub_n, replace=False)
+        try:
+            r, t, s, _ = _umeyama_sim3_from_paths(pose_ref[sample], pose_est[sample])
+        except Exception:
+            continue
+        pose_h = _apply_sim3_to_poses(pose_est, r, t, s)
+        P_h = pose_h[:, :3, 3]
+        errs = np.linalg.norm(P_h - P_ref_all, axis=1)  # Match by same index
+        inliers = errs <= inlier_thresh
+        k = int(inliers.sum())
+        mean_err = float(errs[inliers].mean()) if k > 0 else np.inf
+        if (k > best_score[0]) or (k == best_score[0] and mean_err < best_score[1]):
+            best_score = (k, mean_err)
+            best_model = (r, t, s)
+            best_inliers = inliers
+
+    # Fit again with best inliers
+    if best_inliers is not None and best_inliers.sum() >= 3:
+        r, t, s, _ = _umeyama_sim3_from_paths(pose_ref[best_inliers], pose_est[best_inliers])
+    else:
+        r, t, s = best_model
+    return r, t, s
+
+
+def align_poses_umeyama(
+    ext_ref: np.ndarray,
+    ext_est: np.ndarray,
+    return_aligned=False,
+    ransac=False,
+    sub_n=None,
+    inlier_thresh=None,
+    ransac_max_iters=10,
+    random_state=None,
+):
+    """
+    Align estimated trajectory to reference using Umeyama Sim(3).
+    Default no RANSAC; if ransac=True, use RANSAC (max iterations default 10).
+    - sub_n defaults to half the number of frames (rounded up, at least 3)
+    - inlier_thresh defaults to median of "distance from each estimated pose to
+      nearest reference pose after pre-alignment"
+    Returns rotation (3x3), translation (3,), scale; optionally returns aligned extrinsics (4x4).
+    """
+    pose_ref, pose_est = _poses_from_ext(ext_ref, ext_est)
+
+    if not ransac:
+        r, t, s, pose_est_aligned = _umeyama_sim3_from_paths(pose_ref, pose_est)
+    else:
+        r, t, s = _ransac_align_sim3(
+            pose_ref,
+            pose_est,
+            sub_n=sub_n,
+            inlier_thresh=inlier_thresh,
+            max_iters=ransac_max_iters,
+            random_state=random_state,
+        )
+        pose_est_aligned = _apply_sim3_to_poses(pose_est, r, t, s)
+
+    if return_aligned:
+        ext_est_aligned = affine_inverse_np(pose_est_aligned)
+        return r, t, s, ext_est_aligned
+    return r, t, s
+
+
+# def align_poses_umeyama(ext_ref: np.ndarray, ext_est: np.ndarray, return_aligned=False):
+#     """
+#     Align estimated trajectory to reference trajectory using Umeyama Sim(3)
+#     alignment (via evo PosePath3D). # noqa
+#     Returns rotation, translation, and scale.
+#     """
+#     # If input extrinsics are 3x4, convert to 4x4 by padding
+#     if ext_ref.shape[1] == 3:
+#         ext_ref_ = np.eye(4)[None].repeat(len(ext_ref), 0)
+#         ext_ref_[:, :3] = ext_ref
+#         ext_ref = ext_ref_
+#     if ext_est.shape[1] == 3:
+#         ext_est_ = np.eye(4)[None].repeat(len(ext_est), 0)
+#         ext_est_[:, :3] = ext_est
+#         ext_est = ext_est_
+
+#     # Convert to camera poses (inverse extrinsics)
+#     pose_ref = affine_inverse_np(ext_ref)
+#     pose_est = affine_inverse_np(ext_est)
+
+#     # Create evo PosePath3D objects
+#     path_ref = PosePath3D(poses_se3=pose_ref)
+#     path_est = PosePath3D(poses_se3=pose_est)
+#     r, t, s = path_est.align(path_ref, correct_scale=True)
+#     if return_aligned:
+#         return r, t, s, affine_inverse_np(np.stack(path_est.poses_se3))
+#     else:
+#         return r, t, s
+
+
+def apply_umeyama_alignment_to_ext(
+    rot: np.ndarray,  # (3,3)
+    trans: np.ndarray,  # (3,) or (1,3)
+    scale: float,
+    ext_est: np.ndarray,  # (...,4,4) or (...,3,4)
+) -> np.ndarray:
+    """
+    Apply Sim(3) (R, t, s) to a batch of world-to-camera extrinsics ext_est.
+    Returns the aligned extrinsics, with the same shape as input.
+    """
+
+    # Allow 3x4 extrinsics: pad to 4x4
+    if ext_est.shape[-2:] == (3, 4):
+        pad = np.zeros((*ext_est.shape[:-2], 4, 4), dtype=ext_est.dtype)
+        pad[..., :3, :4] = ext_est
+        pad[..., 3, 3] = 1.0
+        ext_est = pad
+
+    # Convert world-to-camera to camera-to-world
+    pose_est = affine_inverse_np(ext_est)  # (...,4,4)
+    R_e = pose_est[..., :3, :3]  # (...,3,3)
+    t_e = pose_est[..., :3, 3]  # (...,3)
+
+    # Apply Sim(3) transformation
+    R_a = np.einsum("ij,...jk->...ik", rot, R_e)  # (...,3,3)
+    t_a = scale * np.einsum("ij,...j->...i", rot, t_e) + trans  # (...,3)
+
+    # Assemble the transformed pose
+    pose_a = np.zeros_like(pose_est)
+    pose_a[..., :3, :3] = R_a
+    pose_a[..., :3, 3] = t_a
+    pose_a[..., 3, 3] = 1.0
+
+    # Convert back to world-to-camera
+    return affine_inverse_np(pose_a)
+
+
+def transform_points_sim3(points, rot, trans, scale, inverse=False):
+    """
+    Sim(3) transform point cloud
+    points: (N, 3)
+    rot: (3, 3)
+    trans: (3,) or (1, 3)
+    scale: float
+    inverse: Whether to do inverse transform (ref->est)
+    Returns: (N, 3)
+    """
+    if not inverse:
+        # Forward: est -> ref
+        return scale * (points @ rot.T) + trans
+    else:
+        # Inverse: ref -> est
+        return ((points - trans) @ rot) / scale
+
+
+def _rand_rot():
+    u1, u2, u3 = np.random.rand(3)
+    q = np.array(
+        [
+            np.sqrt(1 - u1) * np.sin(2 * np.math.pi * u2),
+            np.sqrt(1 - u1) * np.cos(2 * np.math.pi * u2),
+            np.sqrt(u1) * np.sin(2 * np.math.pi * u3),
+            np.sqrt(u1) * np.cos(2 * np.math.pi * u3),
+        ]
+    )
+    w, x, y, z = q
+    return np.array(
+        [
+            [1 - 2 * (y * y + z * z), 2 * (x * y - z * w), 2 * (x * z + y * w)],
+            [2 * (x * y + z * w), 1 - 2 * (x * x + z * z), 2 * (y * z - x * w)],
+            [2 * (x * z - y * w), 2 * (y * z + x * w), 1 - 2 * (x * x + y * y)],
+        ]
+    )
+
+
+def _rand_pose():
+    R, t = _rand_rot(), np.random.randn(3)
+    P = np.eye(4)
+    P[:3, :3] = R
+    P[:3, 3] = t
+    return P
+
+
+if __name__ == "__main__":
+    np.random.seed(42)
+    # 1. Randomly generate reference trajectory and Sim(3)
+    N = 8
+    pose_ref = np.stack([_rand_pose() for _ in range(N)])  # (N,4,4)  cam→world
+    rot_gt = _rand_rot()
+    scale_gt = 2.3
+    trans_gt = np.random.randn(3)
+    # 2. Generate estimated trajectory (apply Sim(3))
+    pose_est = np.zeros_like(pose_ref)
+    for i in range(N):
+        R = pose_ref[i][:3, :3]
+        t = pose_ref[i][:3, 3]
+        pose_est[i][:3, :3] = rot_gt @ R
+        pose_est[i][:3, 3] = scale_gt * (rot_gt @ t) + trans_gt
+        pose_est[i][3, 3] = 1.0
+    # 3. Get extrinsics (world->cam)
+    ext_ref = affine_inverse_np(pose_ref)
+    ext_est = affine_inverse_np(pose_est)
+    # 4. Use umeyama alignment, estimate Sim(3)
+    r_est, t_est, s_est = align_poses_umeyama(ext_ref, ext_est)
+    print("GT scale:", scale_gt, "Estimated:", s_est)
+    print("GT trans:", trans_gt, "Estimated:", t_est)
+    print("GT rot:\n", rot_gt, "\nEstimated:\n", r_est)
+    # 5. Random point cloud, in ref frame
+    num_points = 100
+    points_ref = np.random.randn(num_points, 3)
+    # 6. Use GT Sim(3) inverse transform to est frame
+    points_est = transform_points_sim3(points_ref, rot_gt, trans_gt, scale_gt, inverse=True)
+    # 7. Use estimated Sim(3) forward transform back to ref frame
+    points_ref_recovered = transform_points_sim3(points_est, r_est, t_est, s_est, inverse=False)
+    # 8. Check error
+    err = np.abs(points_ref_recovered - points_ref)
+    print("Point cloud sim3 transform error (mean abs):", err.mean())
+    print("Point cloud sim3 transform error (max abs):", err.max())
+    assert err.mean() < 1e-6, "Mean sim3 transform error too large!"
+    assert err.max() < 1e-5, "Max sim3 transform error too large!"
+    print("Sim(3) point cloud transform & alignment test passed!")