Spaces:
Sleeping
Sleeping
File size: 4,266 Bytes
bf825fe | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | """Robust scale-only recovery for DA3 camera-conditioned depth on (near-)colinear
camera trajectories — applied at the inference layer, no engine/DA3 source edits.
Why: DA3 aligns its predicted depth to the input camera poses with a full Sim(3)
Umeyama (rotation + translation + scale) over each short window, in
`depth_anything_3.api._align_to_input_extrinsics_intrinsics` ->
`depth_anything_3.utils.pose_align.align_poses_umeyama`, then rescales the depth by
that scale (`prediction.depth /= scale`). Over a COLINEAR window — a straight
forward dolly, a pure sideways/vertical move, i.e. any constant-direction segment —
the camera centers are rank-1, so evo's Umeyama raises
`GeometryException("Degenerate covariance rank, Umeyama alignment is not possible")`.
The engine then drops camera conditioning entirely and the depth stays in DA3's
arbitrary monocular scale, so forward motion renders as a uniform ZOOM instead of
real parallax.
Fix: the ROTATION is what needs rank>=2; the SCALE is a scalar (the ratio of the
two trajectories' extents) and is well-defined even for a colinear/straight path.
This wraps `align_poses_umeyama` so that, on exactly that degeneracy, it returns a
rotation-free scale-only result (R=I, t=0, s = extent(ref centers)/extent(est
centers)). `prediction.depth /= s` then applies the correct metric scale — forward
motion becomes real parallax with no change to the user's pose (so no camera-
following-benchmark penalty). Non-degenerate windows are untouched (original path).
Call `apply_da3_robust_scale()` once after the engine has loaded DA3 (so
`depth_anything_3` is importable); it is idempotent and safe to call repeatedly.
"""
from __future__ import annotations
def _robust_scale(ext_ref, ext_est) -> float:
"""Umeyama scale (est -> ref) recovered from camera-center extents only.
ext_* are world-to-camera [N,3or4,4]; centers are the c2w translations."""
import numpy as np
from depth_anything_3.utils.geometry import affine_inverse_np
def _centers(ext):
c2w = np.asarray(affine_inverse_np(np.asarray(ext)))
return c2w[:, :3, 3]
cref, cest = _centers(ext_ref), _centers(ext_est)
sref = float(np.sqrt(((cref - cref.mean(0)) ** 2).sum(1).mean()))
sest = float(np.sqrt(((cest - cest.mean(0)) ** 2).sum(1).mean()))
s = max(sref, 1e-8) / max(sest, 1e-8)
# guard the pathological stationary-camera degeneracy (both extents ~0)
return s if (np.isfinite(s) and s > 0.0) else 1.0
def apply_da3_robust_scale() -> bool:
"""Monkeypatch align_poses_umeyama to fall back to scale-only on Sim(3)
degeneracy. Returns True if applied (DA3 importable), False otherwise."""
try:
import depth_anything_3.api as _api
from depth_anything_3.utils import pose_align as _pa
except Exception:
return False # DA3 not importable yet — call again after the engine loads it
import numpy as np
orig = _pa.align_poses_umeyama
if getattr(orig, "_robust_scale_wrapped", False):
return True
def wrapped(ext_ref, ext_est, return_aligned=False, **kwargs):
try:
return orig(ext_ref, ext_est, return_aligned=return_aligned, **kwargs)
except Exception as exc: # evo GeometryException et al.
msg = str(exc).lower()
if not ("degenerate" in msg or "covariance" in msg or "umeyama" in msg or "rank" in msg):
raise # unrelated failure — don't mask it
s = _robust_scale(ext_ref, ext_est)
r = np.eye(3, dtype=np.float64)
t = np.zeros(3, dtype=np.float64)
print(
f"[da3_patch] Umeyama degenerate (colinear window) -> scale-only s={s:.4f}",
flush=True,
)
if return_aligned:
# aligned extrinsics are only consumed when align_to_input_ext_scale=False,
# which is not our path (we use the scale). Return input as a safe placeholder.
return r, t, s, np.asarray(ext_est)
return r, t, s
wrapped._robust_scale_wrapped = True
# Patch both the definition module and api's already-bound reference.
_pa.align_poses_umeyama = wrapped
_api.align_poses_umeyama = wrapped
return True
|