Spaces:
Sleeping
Sleeping
| """Robust scale-only recovery for DA3 camera-conditioned depth on (near-)colinear | |
| camera trajectories — applied at the inference layer, no engine/DA3 source edits. | |
| Why: DA3 aligns its predicted depth to the input camera poses with a full Sim(3) | |
| Umeyama (rotation + translation + scale) over each short window, in | |
| `depth_anything_3.api._align_to_input_extrinsics_intrinsics` -> | |
| `depth_anything_3.utils.pose_align.align_poses_umeyama`, then rescales the depth by | |
| that scale (`prediction.depth /= scale`). Over a COLINEAR window — a straight | |
| forward dolly, a pure sideways/vertical move, i.e. any constant-direction segment — | |
| the camera centers are rank-1, so evo's Umeyama raises | |
| `GeometryException("Degenerate covariance rank, Umeyama alignment is not possible")`. | |
| The engine then drops camera conditioning entirely and the depth stays in DA3's | |
| arbitrary monocular scale, so forward motion renders as a uniform ZOOM instead of | |
| real parallax. | |
| Fix: the ROTATION is what needs rank>=2; the SCALE is a scalar (the ratio of the | |
| two trajectories' extents) and is well-defined even for a colinear/straight path. | |
| This wraps `align_poses_umeyama` so that, on exactly that degeneracy, it returns a | |
| rotation-free scale-only result (R=I, t=0, s = extent(ref centers)/extent(est | |
| centers)). `prediction.depth /= s` then applies the correct metric scale — forward | |
| motion becomes real parallax with no change to the user's pose (so no camera- | |
| following-benchmark penalty). Non-degenerate windows are untouched (original path). | |
| Call `apply_da3_robust_scale()` once after the engine has loaded DA3 (so | |
| `depth_anything_3` is importable); it is idempotent and safe to call repeatedly. | |
| """ | |
| from __future__ import annotations | |
| def _robust_scale(ext_ref, ext_est) -> float: | |
| """Umeyama scale (est -> ref) recovered from camera-center extents only. | |
| ext_* are world-to-camera [N,3or4,4]; centers are the c2w translations.""" | |
| import numpy as np | |
| from depth_anything_3.utils.geometry import affine_inverse_np | |
| def _centers(ext): | |
| c2w = np.asarray(affine_inverse_np(np.asarray(ext))) | |
| return c2w[:, :3, 3] | |
| cref, cest = _centers(ext_ref), _centers(ext_est) | |
| sref = float(np.sqrt(((cref - cref.mean(0)) ** 2).sum(1).mean())) | |
| sest = float(np.sqrt(((cest - cest.mean(0)) ** 2).sum(1).mean())) | |
| s = max(sref, 1e-8) / max(sest, 1e-8) | |
| # guard the pathological stationary-camera degeneracy (both extents ~0) | |
| return s if (np.isfinite(s) and s > 0.0) else 1.0 | |
| def apply_da3_robust_scale() -> bool: | |
| """Monkeypatch align_poses_umeyama to fall back to scale-only on Sim(3) | |
| degeneracy. Returns True if applied (DA3 importable), False otherwise.""" | |
| try: | |
| import depth_anything_3.api as _api | |
| from depth_anything_3.utils import pose_align as _pa | |
| except Exception: | |
| return False # DA3 not importable yet — call again after the engine loads it | |
| import numpy as np | |
| orig = _pa.align_poses_umeyama | |
| if getattr(orig, "_robust_scale_wrapped", False): | |
| return True | |
| def wrapped(ext_ref, ext_est, return_aligned=False, **kwargs): | |
| try: | |
| return orig(ext_ref, ext_est, return_aligned=return_aligned, **kwargs) | |
| except Exception as exc: # evo GeometryException et al. | |
| msg = str(exc).lower() | |
| if not ("degenerate" in msg or "covariance" in msg or "umeyama" in msg or "rank" in msg): | |
| raise # unrelated failure — don't mask it | |
| s = _robust_scale(ext_ref, ext_est) | |
| r = np.eye(3, dtype=np.float64) | |
| t = np.zeros(3, dtype=np.float64) | |
| print( | |
| f"[da3_patch] Umeyama degenerate (colinear window) -> scale-only s={s:.4f}", | |
| flush=True, | |
| ) | |
| if return_aligned: | |
| # aligned extrinsics are only consumed when align_to_input_ext_scale=False, | |
| # which is not our path (we use the scale). Return input as a safe placeholder. | |
| return r, t, s, np.asarray(ext_est) | |
| return r, t, s | |
| wrapped._robust_scale_wrapped = True | |
| # Patch both the definition module and api's already-bound reference. | |
| _pa.align_poses_umeyama = wrapped | |
| _api.align_poses_umeyama = wrapped | |
| return True | |