Initial commit
Browse files- .gitignore +3 -0
- README.md +23 -1
- app.py +562 -0
- dvsm/config.py +165 -0
- dvsm/dataset.py +523 -0
- dvsm/loss.py +225 -0
- dvsm/models/DVSM.py +436 -0
- dvsm/models/backbones.py +82 -0
- dvsm/models/blocks.py +403 -0
- dvsm/models/modules.py +273 -0
- dvsm/models/plucker.py +113 -0
- dvsm/models/pose_preprocessor.py +50 -0
- dvsm/models/ttb.py +340 -0
- packages.txt +1 -0
- requirements.txt +24 -0
- viser_app/__init__.py +0 -0
- viser_app/cam_convention.py +133 -0
- viser_app/pi3_estimator.py +153 -0
- viser_app/recon_scene.py +303 -0
- viser_app/render.py +96 -0
- viser_app/scene_loaders.py +622 -0
- viser_app/viewer.py +620 -0
- viser_app/viewer_optim.py +201 -0
.gitignore
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.pyc
|
| 3 |
+
*.recon.pt
|
README.md
CHANGED
|
@@ -5,10 +5,32 @@ colorFrom: yellow
|
|
| 5 |
colorTo: indigo
|
| 6 |
sdk: gradio
|
| 7 |
sdk_version: 6.16.0
|
| 8 |
-
python_version: '3.
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
| 11 |
license: other
|
| 12 |
---
|
| 13 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
|
|
|
| 5 |
colorTo: indigo
|
| 6 |
sdk: gradio
|
| 7 |
sdk_version: 6.16.0
|
| 8 |
+
python_version: '3.10'
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
| 11 |
license: other
|
| 12 |
---
|
| 13 |
|
| 14 |
+
# TTB-DVSM β interactive view synthesis on ZeroGPU
|
| 15 |
+
|
| 16 |
+
Upload images / a short orbit video β ΟΒ³ pose estimation + TTB-DVSM
|
| 17 |
+
reconstruction (one GPU call) β a time-boxed interactive viser viewer.
|
| 18 |
+
|
| 19 |
+
Fixed model: `DVSM-TTB-p8-DL3DV-960p` (`nvidia/DVSM`).
|
| 20 |
+
|
| 21 |
+
## ZeroGPU notes
|
| 22 |
+
Select **ZeroGPU** hardware in the Space Settings (it is not set via this
|
| 23 |
+
frontmatter). ZeroGPU forks a fresh worker per `@spaces.GPU` call and cannot hold a GPU
|
| 24 |
+
permanently. The interactive viewer therefore runs inside a `@spaces.GPU`
|
| 25 |
+
**generator** that holds the GPU for one bounded session (`SESSION_SECONDS`,
|
| 26 |
+
default 180); the main process reverse-proxies `/viser-proxy/*` to the
|
| 27 |
+
worker-local viser port. When the session expires the GPU is released β click
|
| 28 |
+
"Start / Continue Live Session" to resume from the already-computed artifact.
|
| 29 |
+
|
| 30 |
+
`flash-attn` is not installed; `dvsm/models/modules.py` falls back to torch SDPA
|
| 31 |
+
(numerically equivalent). A second-phase optimization is to add a FlashAttn-3
|
| 32 |
+
branch in `_attn_func`.
|
| 33 |
+
|
| 34 |
+
Env knobs: `SESSION_SECONDS`, `RECON_DURATION`, `BENCH_WIDTHS`.
|
| 35 |
+
|
| 36 |
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
app.py
ADDED
|
@@ -0,0 +1,562 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""TTB-DVSM HF Spaces (ZeroGPU) demo.
|
| 2 |
+
|
| 3 |
+
Upload images / a short orbit video -> ΟΒ³ pose estimation + TTB-DVSM
|
| 4 |
+
reconstruction (one GPU call) -> a *time-boxed* interactive viser viewer that
|
| 5 |
+
holds the GPU for the duration of one `@spaces.GPU` session.
|
| 6 |
+
|
| 7 |
+
Why this shape (ZeroGPU constraints, see README):
|
| 8 |
+
- ZeroGPU forks a fresh worker process for each `@spaces.GPU` call and
|
| 9 |
+
releases the GPU when the call returns; GPUs cannot be held permanently.
|
| 10 |
+
- A generator decorated with `@spaces.GPU` keeps the GPU for as long as it is
|
| 11 |
+
being iterated. We exploit that: `_live_session_worker` is a generator that
|
| 12 |
+
starts a viser server *inside the GPU worker* and stays alive (yielding
|
| 13 |
+
heartbeats) until its `duration` budget runs out. The main process reverse-
|
| 14 |
+
proxies `/viser-proxy/*` to that worker-local viser port so the viewer is
|
| 15 |
+
reachable from the single public Gradio port.
|
| 16 |
+
- When the session expires the worker dies (GPU released). The user clicks
|
| 17 |
+
"Start / Continue" to fork a new worker and rebuild the viewer from the
|
| 18 |
+
already-computed artifact on disk (no re-reconstruction).
|
| 19 |
+
|
| 20 |
+
Fixed model: DVSM-TTB-p8-DL3DV-960p (nvidia/DVSM).
|
| 21 |
+
"""
|
| 22 |
+
from __future__ import annotations
|
| 23 |
+
|
| 24 |
+
import os
|
| 25 |
+
import shutil
|
| 26 |
+
import socket
|
| 27 |
+
import sys
|
| 28 |
+
import tempfile
|
| 29 |
+
import time
|
| 30 |
+
from pathlib import Path
|
| 31 |
+
|
| 32 |
+
import numpy as np
|
| 33 |
+
|
| 34 |
+
PROJECT_ROOT = Path(__file__).resolve().parent
|
| 35 |
+
sys.path.insert(0, str(PROJECT_ROOT))
|
| 36 |
+
|
| 37 |
+
# ZeroGPU. `spaces` is a no-op-ish shim off-platform; provide a fallback so the
|
| 38 |
+
# module is importable for local testing without the package.
|
| 39 |
+
try:
|
| 40 |
+
import spaces
|
| 41 |
+
except Exception: # pragma: no cover - local dev without `spaces`
|
| 42 |
+
class _SpacesShim:
|
| 43 |
+
def GPU(self, *args, **kwargs):
|
| 44 |
+
# Support both @spaces.GPU and @spaces.GPU(duration=...)
|
| 45 |
+
if args and callable(args[0]):
|
| 46 |
+
return args[0]
|
| 47 |
+
def _wrap(fn):
|
| 48 |
+
return fn
|
| 49 |
+
return _wrap
|
| 50 |
+
spaces = _SpacesShim()
|
| 51 |
+
|
| 52 |
+
import gradio as gr
|
| 53 |
+
|
| 54 |
+
from viser_app.scene_loaders import (
|
| 55 |
+
IMAGE_EXTS, VIDEO_EXTS, extract_video_frames,
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
# ---------------------------------------------------------------------------
|
| 59 |
+
# Config
|
| 60 |
+
# ---------------------------------------------------------------------------
|
| 61 |
+
HF_VARIANT = "DVSM-TTB-p8-DL3DV-960p"
|
| 62 |
+
REPO_ID = "nvidia/DVSM"
|
| 63 |
+
# How long one held-GPU interactive session lasts before it must be renewed.
|
| 64 |
+
# Bounded by the per-call duration cap and the user's daily quota
|
| 65 |
+
# (anon 2min / free 5min / PRO 40min). Override via env for PRO accounts.
|
| 66 |
+
SESSION_SECONDS = int(os.environ.get("SESSION_SECONDS", "120"))
|
| 67 |
+
RECON_DURATION = int(os.environ.get("RECON_DURATION", "120"))
|
| 68 |
+
# Reduced benchmark width set keeps per-session warmup short (every session
|
| 69 |
+
# re-warms from scratch β CUDA graphs can't persist across GPU workers).
|
| 70 |
+
BENCH_WIDTHS = [int(x) for x in os.environ.get("BENCH_WIDTHS", "512 768").split()]
|
| 71 |
+
|
| 72 |
+
ARTIFACTS_DIR = Path(tempfile.gettempdir()) / "dvsm_artifacts"
|
| 73 |
+
ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True)
|
| 74 |
+
|
| 75 |
+
# ---------------------------------------------------------------------------
|
| 76 |
+
# Module-level model load: download + keep on CPU (ZeroGPU convention).
|
| 77 |
+
# Real `.cuda()` happens inside @spaces.GPU workers.
|
| 78 |
+
# ---------------------------------------------------------------------------
|
| 79 |
+
print(f"[app] loading {HF_VARIANT} + ΟΒ³ to CPU ...", flush=True)
|
| 80 |
+
from dvsm.models.DVSM import DecoderViewSynthesisModel
|
| 81 |
+
from viser_app.pi3_estimator import load_pi3_model
|
| 82 |
+
|
| 83 |
+
# APP_MODEL_DIR: load the DVSM model from a local HF-format directory instead of
|
| 84 |
+
# downloading from the Hub (faster / offline; used for local GPU-node testing).
|
| 85 |
+
_LOCAL_MODEL_DIR = os.environ.get("APP_MODEL_DIR")
|
| 86 |
+
if _LOCAL_MODEL_DIR:
|
| 87 |
+
print(f"[app] loading DVSM from local dir {_LOCAL_MODEL_DIR}", flush=True)
|
| 88 |
+
DVSM_MODEL_CPU = DecoderViewSynthesisModel.from_pretrained(_LOCAL_MODEL_DIR).eval()
|
| 89 |
+
else:
|
| 90 |
+
DVSM_MODEL_CPU = DecoderViewSynthesisModel.from_hf_pretrained(HF_VARIANT).eval()
|
| 91 |
+
PI3_MODEL_CPU = load_pi3_model(flavor="pi3x", device="cpu").eval()
|
| 92 |
+
print("[app] models ready (CPU)", flush=True)
|
| 93 |
+
|
| 94 |
+
# ---------------------------------------------------------------------------
|
| 95 |
+
# Shared state between the session generator and the reverse-proxy.
|
| 96 |
+
# ---------------------------------------------------------------------------
|
| 97 |
+
_state = {"viser_port": None, "session_alive": False}
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def _pick_free_port() -> int:
|
| 101 |
+
s = socket.socket()
|
| 102 |
+
s.bind(("", 0))
|
| 103 |
+
port = s.getsockname()[1]
|
| 104 |
+
s.close()
|
| 105 |
+
return port
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
# ---------------------------------------------------------------------------
|
| 109 |
+
# GPU boundary (a): reconstruction
|
| 110 |
+
# ---------------------------------------------------------------------------
|
| 111 |
+
@spaces.GPU(duration=RECON_DURATION)
|
| 112 |
+
def reconstruct_gpu(img_dir: str, target_h: int, target_w: int, max_views: int) -> str:
|
| 113 |
+
"""ΟΒ³ pose estimation + TTB-DVSM forward -> saved .recon.pt. Runs in a GPU
|
| 114 |
+
worker; returns the artifact path (on shared /tmp) for the session call."""
|
| 115 |
+
import torch
|
| 116 |
+
from viser_app.pi3_estimator import _list_images, pi3_to_scene_data, run_pi3
|
| 117 |
+
from viser_app.recon_scene import reconstruct, save_artifact
|
| 118 |
+
|
| 119 |
+
image_paths = _list_images(Path(img_dir))
|
| 120 |
+
if max_views and len(image_paths) > max_views:
|
| 121 |
+
idxs = np.linspace(0, len(image_paths) - 1, max_views).round().astype(int)
|
| 122 |
+
image_paths = [image_paths[i] for i in idxs]
|
| 123 |
+
|
| 124 |
+
pi3 = PI3_MODEL_CPU.cuda()
|
| 125 |
+
est = run_pi3(pi3, image_paths, image_size=518)
|
| 126 |
+
del pi3
|
| 127 |
+
torch.cuda.empty_cache()
|
| 128 |
+
scene_data = pi3_to_scene_data(est, target_h, target_w)
|
| 129 |
+
|
| 130 |
+
# Reuse the module-level CPU model (moved to cuda inside reconstruct()).
|
| 131 |
+
renderer, batch, ishape, c2w_r, c2w_t, intr = reconstruct(
|
| 132 |
+
ckpt_root=None, ckpt_name="", scene_data=scene_data, cmd_lst=[],
|
| 133 |
+
hf_variant=HF_VARIANT, model=DVSM_MODEL_CPU,
|
| 134 |
+
)
|
| 135 |
+
out_path = ARTIFACTS_DIR / f"{int(time.time() * 1000)}.recon.pt"
|
| 136 |
+
save_artifact(renderer, batch, ishape, c2w_r, c2w_t, intr, out_path,
|
| 137 |
+
hf_variant=HF_VARIANT)
|
| 138 |
+
return str(out_path)
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
# ---------------------------------------------------------------------------
|
| 142 |
+
# GPU boundary (b): time-boxed interactive viser session (generator)
|
| 143 |
+
# ---------------------------------------------------------------------------
|
| 144 |
+
class _SessionArgs:
|
| 145 |
+
"""Mimics the argparse namespace install_viser_handlers expects."""
|
| 146 |
+
max_res = 768
|
| 147 |
+
no_async_push = False
|
| 148 |
+
bench_widths = BENCH_WIDTHS
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
@spaces.GPU(duration=SESSION_SECONDS)
|
| 152 |
+
def _live_session_worker(artifact_path: str, port: int):
|
| 153 |
+
"""Generator running in the GPU worker. Builds the renderer, starts a viser
|
| 154 |
+
server on `port`, then holds the GPU (yielding heartbeats) until the session
|
| 155 |
+
budget expires. Each yield is the integer seconds remaining; the first yield
|
| 156 |
+
signals the viser server is up and serving."""
|
| 157 |
+
import torch
|
| 158 |
+
import viser
|
| 159 |
+
from dvsm.models.pose_preprocessor import PosePreprocessor
|
| 160 |
+
from viser_app.render import build_renderer
|
| 161 |
+
from viser_app.viewer import install_viser_handlers
|
| 162 |
+
from viser_app.viewer_optim import install_optimizations
|
| 163 |
+
|
| 164 |
+
model = DVSM_MODEL_CPU.cuda().eval()
|
| 165 |
+
artifact = torch.load(artifact_path, weights_only=False, map_location="cpu")
|
| 166 |
+
renderer = build_renderer(model, artifact)
|
| 167 |
+
scene_meta = artifact["scene_meta"]
|
| 168 |
+
|
| 169 |
+
pp_cpu = PosePreprocessor.__new__(PosePreprocessor)
|
| 170 |
+
pp_cpu.r = artifact["pose_preproc"]["r"].cpu()
|
| 171 |
+
pp_cpu.t = artifact["pose_preproc"]["t"].cpu()
|
| 172 |
+
pp_cpu.s = artifact["pose_preproc"]["s"].cpu()
|
| 173 |
+
renderer.pose_preprocessor = None
|
| 174 |
+
pp_scale = float(pp_cpu.s.reshape(-1)[0])
|
| 175 |
+
|
| 176 |
+
renderer = install_optimizations(
|
| 177 |
+
renderer, use_fast_plucker=True, use_cuda_graph=True,
|
| 178 |
+
n_warmup=5, max_graphs=len(BENCH_WIDTHS) + 2,
|
| 179 |
+
)
|
| 180 |
+
|
| 181 |
+
server = viser.ViserServer(host="0.0.0.0", port=port)
|
| 182 |
+
server.scene.set_up_direction("+y")
|
| 183 |
+
install_viser_handlers(server, renderer, scene_meta, pp_cpu, pp_scale,
|
| 184 |
+
model, _SessionArgs())
|
| 185 |
+
|
| 186 |
+
deadline = time.monotonic() + SESSION_SECONDS - 5 # margin before hard kill
|
| 187 |
+
first = True
|
| 188 |
+
while True:
|
| 189 |
+
remaining = int(deadline - time.monotonic())
|
| 190 |
+
if remaining <= 0:
|
| 191 |
+
break
|
| 192 |
+
yield remaining if not first else 0 # first yield = "ready"
|
| 193 |
+
first = False
|
| 194 |
+
time.sleep(1.0)
|
| 195 |
+
try:
|
| 196 |
+
server.stop()
|
| 197 |
+
except Exception:
|
| 198 |
+
pass
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def _iframe_html(port: int) -> str:
|
| 202 |
+
return (
|
| 203 |
+
f"<div style='width:100%;aspect-ratio:16/9;border:1px solid #444;"
|
| 204 |
+
f"border-radius:8px;background:#111;overflow:hidden'>"
|
| 205 |
+
f"<iframe src='/viser-proxy/?_t={port}' style='width:100%;height:100%;border:0' "
|
| 206 |
+
f"allow='clipboard-write'></iframe></div>"
|
| 207 |
+
)
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def _status(msg: str, color: str = "#76B900") -> str:
|
| 211 |
+
return f"<p style='color:{color};font-weight:600;margin:0.3em 0'>{msg}</p>"
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
# ---------------------------------------------------------------------------
|
| 215 |
+
# Gradio callbacks (main process)
|
| 216 |
+
# ---------------------------------------------------------------------------
|
| 217 |
+
def reconstruct_cb(input_files, n_frames, target_w,
|
| 218 |
+
progress=gr.Progress(track_tqdm=False)):
|
| 219 |
+
"""CPU pre-processing (decode video / copy images / size) then one GPU
|
| 220 |
+
reconstruction call. Returns (artifact_path, status_html, start_btn_update)."""
|
| 221 |
+
if not input_files:
|
| 222 |
+
return "", _status("Upload a video or at least 3 images.", "#c66"), gr.update(interactive=False)
|
| 223 |
+
|
| 224 |
+
work = Path(tempfile.mkdtemp(prefix="dvsm_in_"))
|
| 225 |
+
img_dir = work / "images"
|
| 226 |
+
img_dir.mkdir()
|
| 227 |
+
|
| 228 |
+
paths = [Path(f.name if hasattr(f, "name") else f) for f in input_files]
|
| 229 |
+
videos = [p for p in paths if p.suffix.lower() in VIDEO_EXTS]
|
| 230 |
+
images = [p for p in paths if p.suffix.lower() in IMAGE_EXTS]
|
| 231 |
+
n_frames = int(n_frames)
|
| 232 |
+
|
| 233 |
+
if videos:
|
| 234 |
+
progress(0.05, desc=f"Extracting {n_frames} frames from video...")
|
| 235 |
+
try:
|
| 236 |
+
n = extract_video_frames(videos[0], img_dir, n_frames)
|
| 237 |
+
except Exception as e:
|
| 238 |
+
return "", _status(f"Video decode failed: {e}", "#c66"), gr.update(interactive=False)
|
| 239 |
+
if n < 3:
|
| 240 |
+
return "", _status(f"Only {n} frames extracted; need β₯3.", "#c66"), gr.update(interactive=False)
|
| 241 |
+
n_inputs = n
|
| 242 |
+
elif len(images) >= 3:
|
| 243 |
+
if len(images) > n_frames:
|
| 244 |
+
idxs = np.linspace(0, len(images) - 1, n_frames).round().astype(int)
|
| 245 |
+
images = [images[i] for i in idxs]
|
| 246 |
+
progress(0.05, desc=f"Copying {len(images)} files...")
|
| 247 |
+
for src in images:
|
| 248 |
+
shutil.copy(src, img_dir / src.name)
|
| 249 |
+
n_inputs = len(images)
|
| 250 |
+
else:
|
| 251 |
+
return "", _status("Upload a video or at least 3 images.", "#c66"), gr.update(interactive=False)
|
| 252 |
+
|
| 253 |
+
# Snap target dims to the input aspect; slider is the long axis; multiples of 8.
|
| 254 |
+
try:
|
| 255 |
+
from PIL import Image as _PILImage
|
| 256 |
+
first_img = next(iter(sorted(img_dir.iterdir())))
|
| 257 |
+
with _PILImage.open(first_img) as im:
|
| 258 |
+
iw, ih = im.size
|
| 259 |
+
input_aspect = iw / ih
|
| 260 |
+
except Exception:
|
| 261 |
+
input_aspect = 16 / 9
|
| 262 |
+
long_axis = (int(target_w) // 8) * 8
|
| 263 |
+
if input_aspect >= 1.0:
|
| 264 |
+
target_w, target_h = long_axis, int(round(long_axis / input_aspect))
|
| 265 |
+
else:
|
| 266 |
+
target_h, target_w = long_axis, int(round(long_axis * input_aspect))
|
| 267 |
+
target_h = max(8, (target_h // 8) * 8)
|
| 268 |
+
target_w = max(8, (target_w // 8) * 8)
|
| 269 |
+
|
| 270 |
+
progress(0.2, desc=f"Running ΟΒ³ + TTB-DVSM ({n_inputs} frames)...")
|
| 271 |
+
try:
|
| 272 |
+
artifact_path = reconstruct_gpu(str(img_dir), target_h, target_w, int(n_inputs))
|
| 273 |
+
except Exception as e:
|
| 274 |
+
return "", _status(f"Reconstruction failed: {e}", "#c66"), gr.update(interactive=False)
|
| 275 |
+
|
| 276 |
+
return (
|
| 277 |
+
artifact_path,
|
| 278 |
+
_status(f"Reconstruction done ({n_inputs} views, {target_w}Γ{target_h}). "
|
| 279 |
+
f"Click βStart / Continue Live Sessionβ."),
|
| 280 |
+
gr.update(interactive=True),
|
| 281 |
+
)
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
def _session_bar(remaining: int, total: int, expired: bool = False) -> str:
|
| 285 |
+
"""Our own animated green countdown bar (one bar, rendered below the viewer).
|
| 286 |
+
We draw it ourselves rather than relying on Gradio's auto progress, which
|
| 287 |
+
would put a bar on every output component of the streaming event."""
|
| 288 |
+
if expired:
|
| 289 |
+
return (
|
| 290 |
+
"<div style='margin-top:4px'>"
|
| 291 |
+
"<div style='height:8px;background:#333;border-radius:4px'></div>"
|
| 292 |
+
"<div style='font-weight:600;color:#c89b3c;margin-top:5px'>Session expired "
|
| 293 |
+
"(GPU released). Click βStart / Continue Live Sessionβ to resume.</div></div>"
|
| 294 |
+
)
|
| 295 |
+
pct = max(0, min(100, round(100 * remaining / max(1, total))))
|
| 296 |
+
return (
|
| 297 |
+
"<div style='margin-top:4px'>"
|
| 298 |
+
"<div style='height:8px;background:#333;border-radius:4px;overflow:hidden'>"
|
| 299 |
+
f"<div style='height:100%;width:{pct}%;background:#76B900;"
|
| 300 |
+
"transition:width 1s linear'></div></div>"
|
| 301 |
+
f"<div style='font-weight:600;color:#76B900;margin-top:5px'>Live β ~{remaining}s "
|
| 302 |
+
"of GPU time left. Drag to explore.</div></div>"
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
def live_session(artifact_path):
|
| 307 |
+
"""Gradio generator handler. Drives `_live_session_worker` (which holds the
|
| 308 |
+
GPU) and streams the iframe + green countdown bar to the UI.
|
| 309 |
+
|
| 310 |
+
Outputs: (viewer_html, session_status, status_html). status_html is cleared
|
| 311 |
+
on start so the reconstruction message disappears once the viewer opens."""
|
| 312 |
+
if not artifact_path:
|
| 313 |
+
yield gr.update(), _status("Reconstruct a scene first.", "#c66"), gr.update()
|
| 314 |
+
return
|
| 315 |
+
|
| 316 |
+
port = _pick_free_port()
|
| 317 |
+
_state["viser_port"] = port
|
| 318 |
+
_state["session_alive"] = True
|
| 319 |
+
try:
|
| 320 |
+
first = True
|
| 321 |
+
for remaining in _live_session_worker(artifact_path, port):
|
| 322 |
+
if first:
|
| 323 |
+
first = False
|
| 324 |
+
# Set the iframe, start the countdown bar, and clear the recon msg.
|
| 325 |
+
yield (_iframe_html(port), _session_bar(SESSION_SECONDS, SESSION_SECONDS), "")
|
| 326 |
+
else:
|
| 327 |
+
yield (gr.update(), _session_bar(int(remaining), SESSION_SECONDS), gr.update())
|
| 328 |
+
finally:
|
| 329 |
+
_state["session_alive"] = False
|
| 330 |
+
_state["viser_port"] = None
|
| 331 |
+
yield (gr.update(), _session_bar(0, SESSION_SECONDS, expired=True), gr.update())
|
| 332 |
+
|
| 333 |
+
|
| 334 |
+
# ---------------------------------------------------------------------------
|
| 335 |
+
# Reverse proxy: /viser-proxy/* -> worker-local viser port
|
| 336 |
+
# ---------------------------------------------------------------------------
|
| 337 |
+
def _build_proxy_handlers():
|
| 338 |
+
import asyncio
|
| 339 |
+
import httpx
|
| 340 |
+
import websockets as ws_pkg
|
| 341 |
+
from starlette.requests import Request
|
| 342 |
+
from starlette.responses import Response
|
| 343 |
+
from starlette.websockets import WebSocket
|
| 344 |
+
|
| 345 |
+
http_client = httpx.AsyncClient(timeout=60.0)
|
| 346 |
+
|
| 347 |
+
def _port_or_none():
|
| 348 |
+
if not _state.get("session_alive"):
|
| 349 |
+
return None
|
| 350 |
+
return _state.get("viser_port")
|
| 351 |
+
|
| 352 |
+
async def proxy_http(request: Request) -> Response:
|
| 353 |
+
path = request.path_params.get("path", "")
|
| 354 |
+
port = _port_or_none()
|
| 355 |
+
if port is None:
|
| 356 |
+
return Response(b"No active viser session.", status_code=503)
|
| 357 |
+
url = f"http://localhost:{port}/{path}"
|
| 358 |
+
if request.url.query:
|
| 359 |
+
url += "?" + request.url.query
|
| 360 |
+
headers = {k: v for k, v in request.headers.items()
|
| 361 |
+
if k.lower() not in ("host", "content-length")}
|
| 362 |
+
body = await request.body()
|
| 363 |
+
try:
|
| 364 |
+
r = await http_client.request(request.method, url, headers=headers,
|
| 365 |
+
content=body, follow_redirects=False)
|
| 366 |
+
except Exception as e:
|
| 367 |
+
return Response(f"viser unreachable: {e}".encode(), status_code=502)
|
| 368 |
+
resp_headers = {k: v for k, v in r.headers.items()
|
| 369 |
+
if k.lower() not in ("content-encoding", "transfer-encoding",
|
| 370 |
+
"content-length", "connection")}
|
| 371 |
+
return Response(content=r.content, status_code=r.status_code, headers=resp_headers)
|
| 372 |
+
|
| 373 |
+
async def proxy_ws(websocket: WebSocket) -> None:
|
| 374 |
+
path = websocket.path_params.get("path", "")
|
| 375 |
+
port = _port_or_none()
|
| 376 |
+
offered = list(websocket.scope.get("subprotocols") or [])
|
| 377 |
+
if port is None:
|
| 378 |
+
await websocket.close(code=1011)
|
| 379 |
+
return
|
| 380 |
+
try:
|
| 381 |
+
upstream = await ws_pkg.connect(
|
| 382 |
+
f"ws://localhost:{port}/{path}", subprotocols=offered or None,
|
| 383 |
+
max_size=50 * 1024 * 1024, compression=None,
|
| 384 |
+
)
|
| 385 |
+
except Exception:
|
| 386 |
+
await websocket.close(code=1011)
|
| 387 |
+
return
|
| 388 |
+
await websocket.accept(subprotocol=upstream.subprotocol)
|
| 389 |
+
|
| 390 |
+
async def c2u():
|
| 391 |
+
try:
|
| 392 |
+
while True:
|
| 393 |
+
msg = await websocket.receive()
|
| 394 |
+
if msg.get("type") == "websocket.disconnect":
|
| 395 |
+
return
|
| 396 |
+
b, t = msg.get("bytes"), msg.get("text")
|
| 397 |
+
if b is not None:
|
| 398 |
+
await upstream.send(b)
|
| 399 |
+
elif t is not None:
|
| 400 |
+
await upstream.send(t)
|
| 401 |
+
except Exception:
|
| 402 |
+
pass
|
| 403 |
+
finally:
|
| 404 |
+
try:
|
| 405 |
+
await upstream.close()
|
| 406 |
+
except Exception:
|
| 407 |
+
pass
|
| 408 |
+
|
| 409 |
+
async def u2c():
|
| 410 |
+
try:
|
| 411 |
+
async for msg in upstream:
|
| 412 |
+
if isinstance(msg, (bytes, bytearray)):
|
| 413 |
+
await websocket.send_bytes(bytes(msg))
|
| 414 |
+
else:
|
| 415 |
+
await websocket.send_text(msg)
|
| 416 |
+
except Exception:
|
| 417 |
+
pass
|
| 418 |
+
finally:
|
| 419 |
+
try:
|
| 420 |
+
await websocket.close()
|
| 421 |
+
except Exception:
|
| 422 |
+
pass
|
| 423 |
+
|
| 424 |
+
await asyncio.gather(c2u(), u2c())
|
| 425 |
+
|
| 426 |
+
return proxy_http, proxy_ws
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
# ---------------------------------------------------------------------------
|
| 430 |
+
# UI
|
| 431 |
+
# ---------------------------------------------------------------------------
|
| 432 |
+
NVIDIA_GREEN = "#76B900"
|
| 433 |
+
NVIDIA_GREEN_DARK = "#5C8F00"
|
| 434 |
+
_PALETTE = gr.themes.Color(
|
| 435 |
+
name="nvidia_green", c50="#F0F8E0", c100="#E1F0C2", c200="#C7E690",
|
| 436 |
+
c300="#A8D958", c400="#8BC53F", c500=NVIDIA_GREEN, c600=NVIDIA_GREEN_DARK,
|
| 437 |
+
c700="#4A7300", c800="#395800", c900="#283D00", c950="#1A2900",
|
| 438 |
+
)
|
| 439 |
+
_theme = gr.themes.Default(primary_hue=_PALETTE).set(
|
| 440 |
+
button_primary_background_fill=NVIDIA_GREEN,
|
| 441 |
+
button_primary_background_fill_hover=NVIDIA_GREEN_DARK,
|
| 442 |
+
button_primary_text_color="white",
|
| 443 |
+
slider_color=NVIDIA_GREEN,
|
| 444 |
+
)
|
| 445 |
+
# Gradio 6 ignores Blocks(theme=...) when the app is served via
|
| 446 |
+
# mount_gradio_app (theme moved to launch()). Force NVIDIA green with CSS so the
|
| 447 |
+
# palette applies regardless of how the app is launched.
|
| 448 |
+
_custom_css = """
|
| 449 |
+
button.primary, button[variant='primary'], .gr-button-primary {
|
| 450 |
+
background: #76B900 !important; background-image: none !important;
|
| 451 |
+
border-color: #76B900 !important; box-shadow: none !important; color: #fff !important;
|
| 452 |
+
}
|
| 453 |
+
button.primary:hover, button[variant='primary']:hover, .gr-button-primary:hover {
|
| 454 |
+
background: #5C8F00 !important; background-image: none !important; border-color: #5C8F00 !important;
|
| 455 |
+
}
|
| 456 |
+
input[type='range']::-webkit-slider-thumb { background: #76B900 !important; }
|
| 457 |
+
input[type='range']::-moz-range-thumb { background: #76B900 !important; }
|
| 458 |
+
.gr-slider .filled, .slider .filled { background: #76B900 !important; }
|
| 459 |
+
*:focus-visible, *:focus { outline-color: #76B900 !important; }
|
| 460 |
+
.gradio-container input:focus, .gradio-container textarea:focus, .gradio-container select:focus {
|
| 461 |
+
border-color: #76B900 !important; box-shadow: 0 0 0 2px rgba(118, 185, 0, 0.4) !important;
|
| 462 |
+
}
|
| 463 |
+
.gallery .selected, .gr-gallery .selected,
|
| 464 |
+
.gallery .thumbnail:hover, .gr-gallery .thumbnail:hover { border-color: #76B900 !important; }
|
| 465 |
+
.progress, .gr-loader, .loader { background: #76B900 !important; }
|
| 466 |
+
.tab-nav .selected, .gr-tab-nav .selected { border-bottom-color: #76B900 !important; }
|
| 467 |
+
"""
|
| 468 |
+
|
| 469 |
+
with gr.Blocks(title="TTB-DVSM (ZeroGPU)", theme=_theme, css=_custom_css) as demo:
|
| 470 |
+
artifact_state = gr.State("")
|
| 471 |
+
with gr.Row():
|
| 472 |
+
with gr.Column(scale=1, min_width=320):
|
| 473 |
+
gr.HTML(
|
| 474 |
+
"<h1 style='margin-bottom:0.1em'>TTB-DVSM</h1>"
|
| 475 |
+
"<h2 style='font-style:normal;font-weight:500;font-size:1.6em;"
|
| 476 |
+
"margin-top:0;color:var(--body-text-color)'>"
|
| 477 |
+
"Test-time MLP Baking for Efficient Rendering of "
|
| 478 |
+
"Decoder-only View Synthesis Models"
|
| 479 |
+
"</h2>"
|
| 480 |
+
"<p style='color:var(--body-text-color);font-size:1.1em;margin-top:0.5em;line-height:1.6'>"
|
| 481 |
+
"TTB-DVSM bakes the scene into fast weights in a single forward pass, "
|
| 482 |
+
"and the result is browseable in the viser viewer."
|
| 483 |
+
"</p>"
|
| 484 |
+
"<div style='color:var(--body-text-color);font-size:1.1em;margin-top:0.8em;line-height:1.7'>"
|
| 485 |
+
"<div style='color:#76B900;font-weight:700;font-size:1.15em;margin-bottom:0.3em'>"
|
| 486 |
+
"Getting Started</div>"
|
| 487 |
+
"<b>1.</b> Drop 16β128 photos <i>or</i> a short orbit video into the input box.<br>"
|
| 488 |
+
"<b>2.</b> Pick how many frames to use and the reconstruction width, "
|
| 489 |
+
"then click <b>Reconstruct</b>.<br>"
|
| 490 |
+
"<b>3.</b> Click <b>Start / Continue Live Session</b> to open the "
|
| 491 |
+
"interactive viewer.<br>"
|
| 492 |
+
"<i>ZeroGPU holds the GPU only for a bounded session; when it expires, "
|
| 493 |
+
"click Continue to resume.</i>"
|
| 494 |
+
"<div style='color:#76B900;font-weight:700;font-size:1.15em;margin:0.7em 0 0.3em'>"
|
| 495 |
+
"Viewer controls</div>"
|
| 496 |
+
" β’ <b>Left-drag</b> orbit "
|
| 497 |
+
"<b>Right-drag</b> pan "
|
| 498 |
+
"<b>Scroll</b> zoom<br>"
|
| 499 |
+
" β’ <b>W A S D</b> fly through the scene "
|
| 500 |
+
"<b>Q / E</b> down / up "
|
| 501 |
+
"<b>Shift</b> faster<br>"
|
| 502 |
+
" β’ Right-side panel: <b>Max W</b>, "
|
| 503 |
+
"<b>Jump to input view</b>, <b>Reset View</b>."
|
| 504 |
+
"</div>"
|
| 505 |
+
)
|
| 506 |
+
inputs = gr.File(
|
| 507 |
+
file_count="multiple",
|
| 508 |
+
file_types=["image", "video", ".mp4", ".mov", ".mkv",
|
| 509 |
+
".webm", ".avi", ".m4v"],
|
| 510 |
+
label="Drop photos OR a short video (orbiting one scene)",
|
| 511 |
+
)
|
| 512 |
+
n_frames = gr.Slider(8, 128, 32, step=4, label="Frames to sample")
|
| 513 |
+
tw = gr.Slider(384, 960, 768, step=64,
|
| 514 |
+
label="Reconstruction width (long axis)")
|
| 515 |
+
recon_btn = gr.Button("Reconstruct", variant="primary")
|
| 516 |
+
session_btn = gr.Button("Start / Continue Live Session",
|
| 517 |
+
variant="primary", interactive=False)
|
| 518 |
+
with gr.Column(scale=3):
|
| 519 |
+
# Reconstruction status/progress above the viewer; the live-session
|
| 520 |
+
# countdown sits BELOW the viewer.
|
| 521 |
+
status_html = gr.HTML()
|
| 522 |
+
viewer_html = gr.HTML(label="Viewer")
|
| 523 |
+
session_status = gr.HTML()
|
| 524 |
+
|
| 525 |
+
recon_btn.click(reconstruct_cb, [inputs, n_frames, tw],
|
| 526 |
+
[artifact_state, status_html, session_btn])
|
| 527 |
+
# show_progress="hidden": the session generator streams for the whole GPU
|
| 528 |
+
# hold; without this Gradio overlays an auto progress bar on EACH output
|
| 529 |
+
# component β multiple bars. We draw our own green countdown bar instead.
|
| 530 |
+
# status_html (3rd output) is cleared so the recon message disappears.
|
| 531 |
+
session_btn.click(live_session, [artifact_state],
|
| 532 |
+
[viewer_html, session_status, status_html],
|
| 533 |
+
show_progress="hidden")
|
| 534 |
+
|
| 535 |
+
|
| 536 |
+
# ---------------------------------------------------------------------------
|
| 537 |
+
# Launch Gradio (which starts uvicorn) and attach the viser reverse-proxy routes
|
| 538 |
+
# onto gradio's own FastAPI app. Using demo.launch() (not mount_gradio_app) lets
|
| 539 |
+
# us pass theme/css the Gradio-6 way and matches the proven viser_app pattern.
|
| 540 |
+
# ---------------------------------------------------------------------------
|
| 541 |
+
def main():
|
| 542 |
+
proxy_http, proxy_ws = _build_proxy_handlers()
|
| 543 |
+
demo.queue(max_size=8)
|
| 544 |
+
demo.launch(
|
| 545 |
+
server_name="0.0.0.0",
|
| 546 |
+
server_port=int(os.environ.get("PORT", "7860")),
|
| 547 |
+
prevent_thread_lock=True,
|
| 548 |
+
theme=_theme,
|
| 549 |
+
)
|
| 550 |
+
app = demo.app # the FastAPI app gradio created
|
| 551 |
+
methods = ["GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS"]
|
| 552 |
+
app.add_route("/viser-proxy", proxy_http, methods=methods)
|
| 553 |
+
app.add_route("/viser-proxy/{path:path}", proxy_http, methods=methods)
|
| 554 |
+
app.router.add_websocket_route("/viser-proxy", proxy_ws)
|
| 555 |
+
app.router.add_websocket_route("/viser-proxy/{path:path}", proxy_ws)
|
| 556 |
+
print("[app] viser-proxy routes attached; ready", flush=True)
|
| 557 |
+
while True:
|
| 558 |
+
time.sleep(3600)
|
| 559 |
+
|
| 560 |
+
|
| 561 |
+
if __name__ == "__main__":
|
| 562 |
+
main()
|
dvsm/config.py
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# NVIDIA CORPORATION and its licensors retain all intellectual property
|
| 4 |
+
# and proprietary rights in and to this software, related documentation
|
| 5 |
+
# and any modifications thereto. Any use, reproduction, disclosure or
|
| 6 |
+
# distribution of this software and related documentation without an express
|
| 7 |
+
# license agreement from NVIDIA CORPORATION is strictly prohibited.
|
| 8 |
+
|
| 9 |
+
import argparse
|
| 10 |
+
from yacs.config import CfgNode
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
cfg = CfgNode()
|
| 14 |
+
|
| 15 |
+
cfg.ckpt = CfgNode(dict(
|
| 16 |
+
exp_dir = "outputs/exps/",
|
| 17 |
+
exp_name = "dev",
|
| 18 |
+
period = 1000,
|
| 19 |
+
|
| 20 |
+
pretrained = "",
|
| 21 |
+
))
|
| 22 |
+
|
| 23 |
+
cfg.dataset = CfgNode(dict(
|
| 24 |
+
# Training setups
|
| 25 |
+
train_path = "configs/dl3dv_train.json",
|
| 26 |
+
min_img_per_seq = 2,
|
| 27 |
+
max_img_per_seq = 32,
|
| 28 |
+
num_input_per_gpu = 32,
|
| 29 |
+
num_target_per_gpu = 32,
|
| 30 |
+
divisible_only = True, # Only sample #views divisible by num_input_per_gpu
|
| 31 |
+
fixed_img_per_seq = False, # All gpu load the same #views
|
| 32 |
+
num_workers = 8,
|
| 33 |
+
|
| 34 |
+
num_pixels = 256 * 256,
|
| 35 |
+
|
| 36 |
+
ap_ratio_lst = "16:9", # Enumerate aspect ratio as 16:9,1:1,...
|
| 37 |
+
crop_pad_ratio = 0.0,
|
| 38 |
+
skip_divisible_resize = False,
|
| 39 |
+
|
| 40 |
+
randomize_static_time = False,
|
| 41 |
+
|
| 42 |
+
# Validation setupts
|
| 43 |
+
valid_path = "configs/dl3dv_test.json",
|
| 44 |
+
valid_ap_ratio = "16:9",
|
| 45 |
+
valid_crop_pad_ratio = 0.,
|
| 46 |
+
))
|
| 47 |
+
|
| 48 |
+
cfg.model = CfgNode(dict(
|
| 49 |
+
patch_size=8,
|
| 50 |
+
latent_dim=768,
|
| 51 |
+
pose_enc='plucker',
|
| 52 |
+
render_up_scale=0.,
|
| 53 |
+
recon_up_scale=0.,
|
| 54 |
+
auto_divisible_pad=False,
|
| 55 |
+
|
| 56 |
+
backbone='',
|
| 57 |
+
freeze_backbone=False,
|
| 58 |
+
|
| 59 |
+
num_register=0,
|
| 60 |
+
|
| 61 |
+
block_type='IMCM',
|
| 62 |
+
num_layers=12,
|
| 63 |
+
head_dim=64,
|
| 64 |
+
expand_ratio=4,
|
| 65 |
+
use_bias=False,
|
| 66 |
+
use_qk_norm=True,
|
| 67 |
+
use_gating=False,
|
| 68 |
+
use_swiglumlp=False,
|
| 69 |
+
use_sandwich=False,
|
| 70 |
+
|
| 71 |
+
ttb_feedback=True,
|
| 72 |
+
ttb_use_gating=True,
|
| 73 |
+
ttb_head_dim=0,
|
| 74 |
+
ttb_inter_multi=4.0,
|
| 75 |
+
ttb_base_lr=0.01,
|
| 76 |
+
ttb_muon_steps=5,
|
| 77 |
+
ttb_num_steps=3, # Number of inner-loop update steps on same data (>1 defers weight norm to last step)
|
| 78 |
+
ttb_target_residual=True, # True: TTB target = norm(x_img) - norm(x_cam); False: TTB target = norm(x_img)
|
| 79 |
+
ttb_pre_cm_target=True, # Use x_img BEFORE cross-view attn+MLP as TTB target (only affects CMTB/IMCMTB)
|
| 80 |
+
ttb_use_mse_loss=True, # Use MSE error-correction in TTB inner loop (False = Hebbian)
|
| 81 |
+
ttb_per_step_norm=False, # Weight norm every update step (LaCT-style) vs deferred to last step (default)
|
| 82 |
+
ttb_share_qk_proj=True, # Share Q/K projection in TTB (ensures read/write address space alignment)
|
| 83 |
+
inner_grad_checkpoint=True, # Gradient checkpoint each TTB inner loop step (save memory, recompute activations)
|
| 84 |
+
|
| 85 |
+
ttb_per_cluster_K=0,
|
| 86 |
+
ttb_per_cluster_boundary_ratio=0.7,
|
| 87 |
+
|
| 88 |
+
use_amp=True,
|
| 89 |
+
use_compile=True,
|
| 90 |
+
grad_checkpoint=True,
|
| 91 |
+
))
|
| 92 |
+
|
| 93 |
+
cfg.optim = CfgNode(dict(
|
| 94 |
+
iter = 100_000,
|
| 95 |
+
lr = 4e-4,
|
| 96 |
+
lr_scaler = 1.0,
|
| 97 |
+
beta1 = 0.9,
|
| 98 |
+
beta2 = 0.95,
|
| 99 |
+
wd = 0.,
|
| 100 |
+
|
| 101 |
+
grad_max_norm = 1.0,
|
| 102 |
+
|
| 103 |
+
sche_warpup_iter = 2500,
|
| 104 |
+
sche_anneal_strategy = "cos",
|
| 105 |
+
sche_div_factor = 100.,
|
| 106 |
+
|
| 107 |
+
lambda_percep = 0.2,
|
| 108 |
+
lambda_ssim = 0.,
|
| 109 |
+
lambda_lpips = 0.,
|
| 110 |
+
))
|
| 111 |
+
|
| 112 |
+
for i_cfg in cfg.values():
|
| 113 |
+
i_cfg.set_new_allowed(True)
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def everytype2bool(v):
|
| 117 |
+
if v.isnumeric():
|
| 118 |
+
return bool(int(v))
|
| 119 |
+
v = v.lower()
|
| 120 |
+
if v in ['n', 'no', 'none', 'false']:
|
| 121 |
+
return False
|
| 122 |
+
return True
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def update_argparser(parser):
|
| 126 |
+
for name in cfg.keys():
|
| 127 |
+
group = parser.add_argument_group(name)
|
| 128 |
+
for key, value in getattr(cfg, name).items():
|
| 129 |
+
full_key = f"{name}.{key}"
|
| 130 |
+
t = type(value)
|
| 131 |
+
|
| 132 |
+
if t == bool:
|
| 133 |
+
group.add_argument(f"--{full_key}", default=value, type=everytype2bool)
|
| 134 |
+
elif t == list:
|
| 135 |
+
group.add_argument(f"--{full_key}", default=value, type=type(value[0]), nargs="*")
|
| 136 |
+
elif t == tuple:
|
| 137 |
+
group.add_argument(f"--{full_key}", default=value, type=type(value[0]), nargs=len(value))
|
| 138 |
+
else:
|
| 139 |
+
group.add_argument(f"--{full_key}", default=value, type=t)
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def update_config(cfg_files, cmd_lst=[]):
|
| 143 |
+
# Update from config files
|
| 144 |
+
if isinstance(cfg_files, str):
|
| 145 |
+
cfg_files = [cfg_files]
|
| 146 |
+
for cfg_path in cfg_files:
|
| 147 |
+
cfg.merge_from_file(cfg_path)
|
| 148 |
+
|
| 149 |
+
if len(cmd_lst) == 0:
|
| 150 |
+
return
|
| 151 |
+
|
| 152 |
+
# Parse the arguments from command line
|
| 153 |
+
internal_parser = argparse.ArgumentParser()
|
| 154 |
+
update_argparser(internal_parser)
|
| 155 |
+
internal_args = internal_parser.parse_args(cmd_lst)
|
| 156 |
+
|
| 157 |
+
# Update from command line args
|
| 158 |
+
for name in cfg.keys():
|
| 159 |
+
cfg_subgroup = getattr(cfg, name)
|
| 160 |
+
for key in cfg_subgroup.keys():
|
| 161 |
+
full_key = f"{name}.{key}"
|
| 162 |
+
arg_val = getattr(internal_args, full_key)
|
| 163 |
+
# Check if the default values is updated
|
| 164 |
+
if internal_parser.get_default(full_key) != arg_val:
|
| 165 |
+
cfg_subgroup[key] = arg_val
|
dvsm/dataset.py
ADDED
|
@@ -0,0 +1,523 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import h5py
|
| 3 |
+
import numpy as np
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class MixedTrainingDataset(torch.utils.data.Dataset):
|
| 10 |
+
'''
|
| 11 |
+
Load one image from the mixed dataset to serve MixedBatchRandomSampler.
|
| 12 |
+
'''
|
| 13 |
+
def __init__(self, datasets_json_path, randomize_static_time=False, max_ts=60., keep_unscaled_image=False):
|
| 14 |
+
# Parse datasets composition
|
| 15 |
+
if isinstance(datasets_json_path, dict):
|
| 16 |
+
datasets_json = datasets_json_path
|
| 17 |
+
else:
|
| 18 |
+
with open(datasets_json_path) as f:
|
| 19 |
+
datasets_json = json.load(f)
|
| 20 |
+
|
| 21 |
+
self.randomize_static_time = randomize_static_time
|
| 22 |
+
self.max_ts = max_ts
|
| 23 |
+
self.keep_unscaled_image = keep_unscaled_image
|
| 24 |
+
|
| 25 |
+
# Init loaders
|
| 26 |
+
self.reader_lst = []
|
| 27 |
+
self.path_lst = []
|
| 28 |
+
self.ds_name_lst = []
|
| 29 |
+
self.weight_lst = []
|
| 30 |
+
self.dynamic_lst = []
|
| 31 |
+
self.min_frame_skip_lst = []
|
| 32 |
+
self.max_frame_skip_lst = []
|
| 33 |
+
self.extra_skip_lst = []
|
| 34 |
+
self.scene_names_lst = []
|
| 35 |
+
self.image_res_lst = []
|
| 36 |
+
self.num_frames_lst = []
|
| 37 |
+
self.total_scenes = 0
|
| 38 |
+
|
| 39 |
+
for i_ds, (ds_name, meta) in enumerate(datasets_json.items()):
|
| 40 |
+
self.reader_lst.append(None) # Should be init by each process
|
| 41 |
+
self.path_lst.append(meta["path"])
|
| 42 |
+
self.ds_name_lst.append(ds_name)
|
| 43 |
+
self.weight_lst.append(meta.get("weight", 1.0))
|
| 44 |
+
self.dynamic_lst.append(meta.get("is_dynamic", False))
|
| 45 |
+
self.min_frame_skip_lst.append(meta.get("min_frame_skip", 1))
|
| 46 |
+
self.max_frame_skip_lst.append(meta.get("max_frame_skip", None))
|
| 47 |
+
self.extra_skip_lst.append(meta.get("extra_skip", 0))
|
| 48 |
+
|
| 49 |
+
path = Path(meta["path"])
|
| 50 |
+
statpath = path.with_suffix('.json')
|
| 51 |
+
if not statpath.exists():
|
| 52 |
+
print(f"Preparing {statpath} for the first time")
|
| 53 |
+
with h5py.File(path, 'r') as f:
|
| 54 |
+
scene_names = sorted(list(f.keys()))
|
| 55 |
+
num_frames = []
|
| 56 |
+
resolutions = []
|
| 57 |
+
for i_scene, sn in enumerate(scene_names):
|
| 58 |
+
num_frames.append(int(f[sn].attrs['n_frames']))
|
| 59 |
+
resolutions.append(f[sn]['image'].shape[1:3])
|
| 60 |
+
with open(statpath, 'w') as f:
|
| 61 |
+
json.dump({
|
| 62 |
+
'scene_names': scene_names,
|
| 63 |
+
'num_frames': num_frames,
|
| 64 |
+
'resolutions': resolutions,
|
| 65 |
+
}, f)
|
| 66 |
+
|
| 67 |
+
with open(statpath) as f:
|
| 68 |
+
stat = json.load(f)
|
| 69 |
+
self.scene_names_lst.append(stat['scene_names'])
|
| 70 |
+
self.total_scenes += len(stat['scene_names'])
|
| 71 |
+
self.image_res_lst.append(stat['resolutions'])
|
| 72 |
+
self.num_frames_lst.append(stat['num_frames'])
|
| 73 |
+
|
| 74 |
+
self.weight_lst = np.array(self.weight_lst)
|
| 75 |
+
self.weight_lst /= self.weight_lst.sum()
|
| 76 |
+
|
| 77 |
+
def __repr__(self):
|
| 78 |
+
return '\n'.join(
|
| 79 |
+
f' {self.ds_name_lst[i]:30s}: size={len(self.scene_names_lst[i]):6d} / prob={self.weight_lst[i]*100:.2f}% / dynamic={self.dynamic_lst[i]}'
|
| 80 |
+
for i in range(len(self.path_lst))
|
| 81 |
+
)
|
| 82 |
+
|
| 83 |
+
def __len__(self):
|
| 84 |
+
return self.total_scenes
|
| 85 |
+
|
| 86 |
+
def sample_frames(self, i_ds, i_scene, num_frames):
|
| 87 |
+
num_total = self.num_frames_lst[i_ds][i_scene]
|
| 88 |
+
min_frame_skip = self.min_frame_skip_lst[i_ds]
|
| 89 |
+
max_frame_skip = self.max_frame_skip_lst[i_ds]
|
| 90 |
+
|
| 91 |
+
frame_skip = np.random.randint(low=min_frame_skip, high=max_frame_skip + 1, size=[num_frames - 1])
|
| 92 |
+
frame_skip = np.concatenate([[0], frame_skip])
|
| 93 |
+
frame_offset = np.cumsum(frame_skip)
|
| 94 |
+
if num_total > frame_offset[-1]:
|
| 95 |
+
# Sample the subsequence
|
| 96 |
+
frame_start = np.random.randint(0, num_total - frame_offset[-1])
|
| 97 |
+
frame_indices = frame_start + frame_offset
|
| 98 |
+
elif num_total > num_frames:
|
| 99 |
+
# Randomly sample without replacement
|
| 100 |
+
frame_indices = np.random.choice(num_total, size=num_frames, replace=False)
|
| 101 |
+
elif num_total < 2:
|
| 102 |
+
frame_indices = np.zeros([num_frames], dtype=int)
|
| 103 |
+
else:
|
| 104 |
+
# Sample half of total frame and duplicate to target length
|
| 105 |
+
frame_indices = self.sample_frames(i_ds, i_scene, num_total//2)
|
| 106 |
+
frame_indices = np.resize(frame_indices, num_frames)
|
| 107 |
+
|
| 108 |
+
np.random.shuffle(frame_indices)
|
| 109 |
+
return frame_indices
|
| 110 |
+
|
| 111 |
+
def sample_target(self, i_ds, i_scene, frame_indices, num_target):
|
| 112 |
+
num_total = self.num_frames_lst[i_ds][i_scene]
|
| 113 |
+
extra_skip = self.extra_skip_lst[i_ds] # Allow target views outside the input subseq
|
| 114 |
+
frame_indices = set(frame_indices)
|
| 115 |
+
candidates = [
|
| 116 |
+
i
|
| 117 |
+
for i in range(min(frame_indices)-extra_skip, max(frame_indices)+extra_skip+1)
|
| 118 |
+
if i not in frame_indices and i >= 0 and i < num_total
|
| 119 |
+
]
|
| 120 |
+
if len(candidates) == 0:
|
| 121 |
+
candidates = list(frame_indices)
|
| 122 |
+
|
| 123 |
+
if num_target < len(candidates):
|
| 124 |
+
target_indices = np.random.choice(candidates, size=num_target, replace=False)
|
| 125 |
+
else:
|
| 126 |
+
target_indices = np.concatenate([
|
| 127 |
+
candidates,
|
| 128 |
+
np.random.choice(candidates, size=num_target-len(candidates), replace=True)])
|
| 129 |
+
return target_indices
|
| 130 |
+
|
| 131 |
+
@staticmethod
|
| 132 |
+
def aspect_ratio_crop(frame, ap_ratio, cp_ratio):
|
| 133 |
+
if ap_ratio <= 0 and cp_ratio <= 0:
|
| 134 |
+
return
|
| 135 |
+
|
| 136 |
+
# Find the maximum resolution ratio
|
| 137 |
+
H, W = frame['image'].shape[-2:]
|
| 138 |
+
ori_ap_ratio = W / H
|
| 139 |
+
if ap_ratio < ori_ap_ratio:
|
| 140 |
+
h_ratio = 1.
|
| 141 |
+
w_ratio = ap_ratio / ori_ap_ratio
|
| 142 |
+
else:
|
| 143 |
+
h_ratio = ori_ap_ratio / ap_ratio
|
| 144 |
+
w_ratio = 1.
|
| 145 |
+
|
| 146 |
+
# Zoom-in by cropping
|
| 147 |
+
if cp_ratio > 0:
|
| 148 |
+
h_ratio *= (1 - cp_ratio)
|
| 149 |
+
w_ratio *= (1 - cp_ratio)
|
| 150 |
+
|
| 151 |
+
# Derive the top-left corner position
|
| 152 |
+
ir = round((1 - h_ratio) * 0.5 * H)
|
| 153 |
+
ic = round((1 - w_ratio) * 0.5 * W)
|
| 154 |
+
h = round(h_ratio * H)
|
| 155 |
+
w = round(w_ratio * W)
|
| 156 |
+
|
| 157 |
+
# Crop image
|
| 158 |
+
frame['image'] = frame['image'][..., ir:ir+h, ic:ic+w]
|
| 159 |
+
|
| 160 |
+
# Apply the crop to intrinsics
|
| 161 |
+
fx, fy, cx, cy = frame['intr']
|
| 162 |
+
fx = fx * W / w
|
| 163 |
+
fy = fy * H / h
|
| 164 |
+
cx = (cx * W - ic) / w
|
| 165 |
+
cy = (cy * H - ir) / h
|
| 166 |
+
frame['intr'] = np.array([fx, fy, cx, cy], dtype=np.float32)
|
| 167 |
+
|
| 168 |
+
def __getitem__(self, samp):
|
| 169 |
+
'''
|
| 170 |
+
Read one frame specified by the batch sampler.
|
| 171 |
+
'''
|
| 172 |
+
|
| 173 |
+
# Unpack loading target setup
|
| 174 |
+
batch_id, is_heldout, i_ds, i_scene, i_frame, ap_ratio, cp_ratio, t_size = samp
|
| 175 |
+
ds_name = self.ds_name_lst[i_ds]
|
| 176 |
+
scene_name = self.scene_names_lst[i_ds][i_scene]
|
| 177 |
+
|
| 178 |
+
# Prepare the data reader
|
| 179 |
+
if self.reader_lst[i_ds] is None:
|
| 180 |
+
self.reader_lst[i_ds] = h5py.File(self.path_lst[i_ds], 'r')
|
| 181 |
+
data = self.reader_lst[i_ds][scene_name]
|
| 182 |
+
attrs = dict(data.attrs.items())
|
| 183 |
+
|
| 184 |
+
# Load the frame
|
| 185 |
+
image = torch.tensor(data['image'][i_frame], dtype=torch.uint8).moveaxis(-1, 0).contiguous()
|
| 186 |
+
image = image.float() / 255.
|
| 187 |
+
|
| 188 |
+
# Parse extrinsic
|
| 189 |
+
w2c = data['w2c'][i_frame]
|
| 190 |
+
w2c_r = w2c[:3, :3]
|
| 191 |
+
w2c_t = w2c[:3, 3]
|
| 192 |
+
c2w_r = np.copy(w2c_r.T)
|
| 193 |
+
c2w_t = -c2w_r @ w2c_t
|
| 194 |
+
|
| 195 |
+
# Parse intrinsic (normalized)
|
| 196 |
+
fovx = attrs['fovx'] if 'fovx' in attrs else data['fovx'][i_frame]
|
| 197 |
+
fovy = attrs['fovy'] if 'fovy' in attrs else data['fovy'][i_frame]
|
| 198 |
+
cx = attrs['cx_p'] if 'cx_p' in attrs else data['cx_p'][i_frame]
|
| 199 |
+
cy = attrs['cy_p'] if 'cy_p' in attrs else data['cy_p'][i_frame]
|
| 200 |
+
fx = 0.5 / np.tan(0.5*fovx)
|
| 201 |
+
fy = 0.5 / np.tan(0.5*fovy)
|
| 202 |
+
|
| 203 |
+
# Parse frame time
|
| 204 |
+
if self.dynamic_lst[i_ds]:
|
| 205 |
+
ts = i_frame / 5 # Assume 5FPS
|
| 206 |
+
elif self.randomize_static_time:
|
| 207 |
+
ts = np.random.uniform(0, self.max_ts)
|
| 208 |
+
else:
|
| 209 |
+
ts = 0
|
| 210 |
+
|
| 211 |
+
frame = dict(
|
| 212 |
+
batch_id=batch_id,
|
| 213 |
+
ts=ts,
|
| 214 |
+
is_heldout=is_heldout,
|
| 215 |
+
ds_name=ds_name,
|
| 216 |
+
scene_name=scene_name,
|
| 217 |
+
frame_name=data['name'][i_frame].decode(),
|
| 218 |
+
image=image,
|
| 219 |
+
c2w_r=c2w_r,
|
| 220 |
+
c2w_t=c2w_t,
|
| 221 |
+
intr=np.array([fx, fy, cx, cy], dtype=np.float32),
|
| 222 |
+
)
|
| 223 |
+
|
| 224 |
+
# Apply cropping
|
| 225 |
+
self.aspect_ratio_crop(frame, ap_ratio, cp_ratio)
|
| 226 |
+
|
| 227 |
+
# Resize to target resolution
|
| 228 |
+
if self.keep_unscaled_image:
|
| 229 |
+
frame['src_image'] = frame['image']
|
| 230 |
+
if frame['image'].shape[-2:] != t_size:
|
| 231 |
+
frame['image'] = torch.nn.functional.interpolate(
|
| 232 |
+
frame['image'].unsqueeze(0),
|
| 233 |
+
size=t_size, mode="bilinear", align_corners=False, antialias=True).squeeze(0)
|
| 234 |
+
|
| 235 |
+
return frame
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
class MixedBatchRandomSampler(torch.utils.data.Sampler):
|
| 239 |
+
'''
|
| 240 |
+
Sample a batch consisting of #seq x #img_per_seq.
|
| 241 |
+
'''
|
| 242 |
+
def __init__(self,
|
| 243 |
+
mixed_dataset,
|
| 244 |
+
min_img_per_seq,
|
| 245 |
+
max_img_per_seq,
|
| 246 |
+
num_target_per_gpu,
|
| 247 |
+
num_input_per_gpu,
|
| 248 |
+
num_pixels,
|
| 249 |
+
patch_size,
|
| 250 |
+
remain_iter,
|
| 251 |
+
divisible_only=False,
|
| 252 |
+
fixed_img_per_seq=False,
|
| 253 |
+
ap_ratio_lst="",
|
| 254 |
+
crop_pad_ratio=0.):
|
| 255 |
+
self.mixed_dataset = mixed_dataset
|
| 256 |
+
self.len_candidate = list(range(min_img_per_seq, max_img_per_seq+1))
|
| 257 |
+
if divisible_only and num_input_per_gpu > 0:
|
| 258 |
+
self.len_candidate = [v for v in self.len_candidate if (num_input_per_gpu%v==0)]
|
| 259 |
+
assert len(self.len_candidate) > 0
|
| 260 |
+
self.fixed_img_per_seq = fixed_img_per_seq
|
| 261 |
+
self.fixed_it = 0
|
| 262 |
+
self.num_target_per_gpu = num_target_per_gpu
|
| 263 |
+
self.num_input_per_gpu = num_input_per_gpu
|
| 264 |
+
self.num_pixels = num_pixels
|
| 265 |
+
self.patch_size = patch_size
|
| 266 |
+
self.remain_iter = remain_iter
|
| 267 |
+
|
| 268 |
+
# The image aspect ratios on a gpu should all be the same
|
| 269 |
+
if ap_ratio_lst != "":
|
| 270 |
+
self.ap_ratio_lst = [
|
| 271 |
+
float(r.split(':')[0]) / float(r.split(':')[1])
|
| 272 |
+
for r in ap_ratio_lst.split(',')]
|
| 273 |
+
else:
|
| 274 |
+
self.ap_ratio_lst = [0.]
|
| 275 |
+
|
| 276 |
+
# Zoom-in into the image central
|
| 277 |
+
self.crop_pad_ratio = crop_pad_ratio
|
| 278 |
+
|
| 279 |
+
def sample_batch_indices(self):
|
| 280 |
+
# Determine #seq x #imgs_per_seq on current gpu
|
| 281 |
+
if self.fixed_img_per_seq:
|
| 282 |
+
num_img_per_seq = self.len_candidate[self.fixed_it]
|
| 283 |
+
self.fixed_it = (self.fixed_it + 1) % len(self.len_candidate)
|
| 284 |
+
else:
|
| 285 |
+
num_img_per_seq = np.random.choice(self.len_candidate)
|
| 286 |
+
num_seq = max(1, self.num_input_per_gpu // num_img_per_seq)
|
| 287 |
+
num_target_per_seq = self.num_target_per_gpu // num_seq
|
| 288 |
+
|
| 289 |
+
# Sample aspect ratio
|
| 290 |
+
ap_ratio = np.random.choice(self.ap_ratio_lst)
|
| 291 |
+
|
| 292 |
+
# Sample padding crop
|
| 293 |
+
if self.crop_pad_ratio > 0:
|
| 294 |
+
cp_ratio = np.random.uniform(0, self.crop_pad_ratio)
|
| 295 |
+
else:
|
| 296 |
+
cp_ratio = 0
|
| 297 |
+
|
| 298 |
+
# Sample the batch from the mixed datasets
|
| 299 |
+
batch_indices = []
|
| 300 |
+
for batch_id in range(num_seq):
|
| 301 |
+
i_ds = np.random.choice(len(self.mixed_dataset.weight_lst), p=self.mixed_dataset.weight_lst)
|
| 302 |
+
i_scene = np.random.choice(len(self.mixed_dataset.scene_names_lst[i_ds]))
|
| 303 |
+
|
| 304 |
+
# Calculate target resolution
|
| 305 |
+
if ap_ratio > 0:
|
| 306 |
+
th = (self.num_pixels / (ap_ratio * self.patch_size * self.patch_size)) ** 0.5
|
| 307 |
+
tw = th * ap_ratio
|
| 308 |
+
else:
|
| 309 |
+
# The batch size should be 1 or all the datasets are in same aspect ratio
|
| 310 |
+
sh, sw = self.mixed_dataset.image_res_lst[i_ds][i_scene]
|
| 311 |
+
th = (self.num_pixels / (sw/sh * self.patch_size * self.patch_size)) ** 0.5
|
| 312 |
+
tw = th * sw/sh
|
| 313 |
+
th = round(th) * self.patch_size
|
| 314 |
+
tw = round(tw) * self.patch_size
|
| 315 |
+
|
| 316 |
+
# Sample frames
|
| 317 |
+
frame_indices = self.mixed_dataset.sample_frames(i_ds, i_scene, num_img_per_seq)
|
| 318 |
+
target_indices = self.mixed_dataset.sample_target(i_ds, i_scene, frame_indices, num_target_per_seq)
|
| 319 |
+
|
| 320 |
+
for i_frame in frame_indices:
|
| 321 |
+
batch_indices.append((
|
| 322 |
+
batch_id, False, i_ds, i_scene, i_frame,
|
| 323 |
+
ap_ratio, cp_ratio, (th, tw)))
|
| 324 |
+
for i_frame in target_indices:
|
| 325 |
+
batch_indices.append((
|
| 326 |
+
batch_id, True, i_ds, i_scene, i_frame,
|
| 327 |
+
ap_ratio, cp_ratio, (th, tw)))
|
| 328 |
+
|
| 329 |
+
return batch_indices
|
| 330 |
+
|
| 331 |
+
def __iter__(self):
|
| 332 |
+
for _ in range(self.remain_iter):
|
| 333 |
+
yield self.sample_batch_indices()
|
| 334 |
+
|
| 335 |
+
def __len__(self):
|
| 336 |
+
return self.remain_iter
|
| 337 |
+
|
| 338 |
+
@staticmethod
|
| 339 |
+
def batchify_sequences(batch):
|
| 340 |
+
# Recover batch dimension
|
| 341 |
+
num_img_per_seq = sum(int(item['batch_id']==0 and not item['is_heldout']) for item in batch)
|
| 342 |
+
num_target_per_seq = sum(int(item['batch_id']==0 and item['is_heldout']) for item in batch)
|
| 343 |
+
seq_size = num_img_per_seq + num_target_per_seq
|
| 344 |
+
num_seq = len(batch) // seq_size
|
| 345 |
+
|
| 346 |
+
# Sanity check
|
| 347 |
+
assert len(batch) == num_seq * seq_size
|
| 348 |
+
for i in range(len(batch)):
|
| 349 |
+
assert batch[i]['batch_id'] == batch[i - i%seq_size]['batch_id']
|
| 350 |
+
assert (not batch[i]['is_heldout']) == (i%seq_size < num_img_per_seq)
|
| 351 |
+
assert batch[i]['is_heldout'] == (i%seq_size >= num_img_per_seq)
|
| 352 |
+
|
| 353 |
+
# Batchify
|
| 354 |
+
ds_names = [batch[i*seq_size]['ds_name'] for i in range(num_seq)]
|
| 355 |
+
scene_names = [batch[i*seq_size]['scene_name'] for i in range(num_seq)]
|
| 356 |
+
frame_names = np.array([batch[i]['frame_name'] for i in range(len(batch))])
|
| 357 |
+
ts = torch.tensor([batch[i]['ts'] for i in range(len(batch))], dtype=torch.float32)
|
| 358 |
+
image = torch.stack([batch[i]['image'] for i in range(len(batch))])
|
| 359 |
+
c2w_r = torch.tensor(np.stack([batch[i]['c2w_r'] for i in range(len(batch))]), dtype=torch.float32)
|
| 360 |
+
c2w_t = torch.tensor(np.stack([batch[i]['c2w_t'] for i in range(len(batch))]), dtype=torch.float32)
|
| 361 |
+
intr = torch.tensor(np.stack([batch[i]['intr'] for i in range(len(batch))]), dtype=torch.float32)
|
| 362 |
+
|
| 363 |
+
# Reshape
|
| 364 |
+
frame_names = np.array(frame_names).reshape(num_seq, seq_size)
|
| 365 |
+
ts = ts.unflatten(0, (num_seq, seq_size))
|
| 366 |
+
image = image.unflatten(0, (num_seq, seq_size))
|
| 367 |
+
c2w_r = c2w_r.unflatten(0, (num_seq, seq_size))
|
| 368 |
+
c2w_t = c2w_t.unflatten(0, (num_seq, seq_size))
|
| 369 |
+
intr = intr.unflatten(0, (num_seq, seq_size))
|
| 370 |
+
|
| 371 |
+
# Split into input and heldout target views
|
| 372 |
+
input_views = dict(
|
| 373 |
+
frame_names=frame_names[:, :num_img_per_seq],
|
| 374 |
+
ts=ts[:, :num_img_per_seq].contiguous(),
|
| 375 |
+
image=image[:, :num_img_per_seq].contiguous(),
|
| 376 |
+
c2w_r=c2w_r[:, :num_img_per_seq].contiguous(),
|
| 377 |
+
c2w_t=c2w_t[:, :num_img_per_seq].contiguous(),
|
| 378 |
+
intr=intr[:, :num_img_per_seq].contiguous(),
|
| 379 |
+
)
|
| 380 |
+
target_views = dict(
|
| 381 |
+
frame_names=frame_names[:, num_img_per_seq:],
|
| 382 |
+
ts=ts[:, num_img_per_seq:].contiguous(),
|
| 383 |
+
image=image[:, num_img_per_seq:].contiguous(),
|
| 384 |
+
c2w_r=c2w_r[:, num_img_per_seq:].contiguous(),
|
| 385 |
+
c2w_t=c2w_t[:, num_img_per_seq:].contiguous(),
|
| 386 |
+
intr=intr[:, num_img_per_seq:].contiguous(),
|
| 387 |
+
)
|
| 388 |
+
|
| 389 |
+
# Batchify source frames if there is
|
| 390 |
+
if 'src_image' in batch[0]:
|
| 391 |
+
src_image = torch.stack([batch[i]['src_image'] for i in range(len(batch))])
|
| 392 |
+
src_image = src_image.unflatten(0, (num_seq, seq_size))
|
| 393 |
+
input_views['src_image'] = src_image[:, :num_img_per_seq].contiguous()
|
| 394 |
+
target_views['src_image'] = src_image[:, num_img_per_seq:].contiguous()
|
| 395 |
+
|
| 396 |
+
return dict(
|
| 397 |
+
ds_names=ds_names,
|
| 398 |
+
scene_names=scene_names,
|
| 399 |
+
frame_names=frame_names,
|
| 400 |
+
input_views=input_views,
|
| 401 |
+
target_views=target_views)
|
| 402 |
+
|
| 403 |
+
|
| 404 |
+
class PresampledTestingDataset(MixedTrainingDataset):
|
| 405 |
+
'''
|
| 406 |
+
Load one scene from the mixed dataset where the views are sampled and fixed.
|
| 407 |
+
'''
|
| 408 |
+
def __init__(self, datasets_json_path, num_pixels, patch_size, ap_ratio="16:9", cp_ratio=0., keep_unscaled_image=False):
|
| 409 |
+
# Parse datasets composition
|
| 410 |
+
if isinstance(datasets_json_path, dict):
|
| 411 |
+
datasets_json = datasets_json_path
|
| 412 |
+
else:
|
| 413 |
+
with open(datasets_json_path) as f:
|
| 414 |
+
datasets_json = json.load(f)
|
| 415 |
+
|
| 416 |
+
self.randomize_static_time = False
|
| 417 |
+
self.keep_unscaled_image = keep_unscaled_image
|
| 418 |
+
|
| 419 |
+
self.num_pixels = num_pixels
|
| 420 |
+
self.patch_size = patch_size
|
| 421 |
+
ap_ratio = ap_ratio.split(':')
|
| 422 |
+
self.ap_ratio = float(ap_ratio[0]) / float(ap_ratio[1])
|
| 423 |
+
self.cp_ratio = cp_ratio
|
| 424 |
+
|
| 425 |
+
# Init loaders
|
| 426 |
+
self.reader_lst = []
|
| 427 |
+
self.path_lst = []
|
| 428 |
+
self.ds_name_lst = []
|
| 429 |
+
self.ds_name_idx = {}
|
| 430 |
+
self.scene_names_lst = []
|
| 431 |
+
self.dynamic_lst = []
|
| 432 |
+
self.image_res_lst = []
|
| 433 |
+
self.input_frame_idx_lst = []
|
| 434 |
+
self.target_frame_idx_lst = []
|
| 435 |
+
self.remap_idx = []
|
| 436 |
+
self.total_scenes = 0
|
| 437 |
+
|
| 438 |
+
for i_ds, (ds_name, meta) in enumerate(datasets_json.items()):
|
| 439 |
+
self.reader_lst.append(None) # Should be init by each process
|
| 440 |
+
self.path_lst.append(meta["path"])
|
| 441 |
+
self.ds_name_lst.append(ds_name)
|
| 442 |
+
self.ds_name_idx[ds_name] = i_ds
|
| 443 |
+
self.dynamic_lst.append(meta.get("is_dynamic", False))
|
| 444 |
+
|
| 445 |
+
path = Path(meta["path"])
|
| 446 |
+
statpath = path.with_suffix('.json')
|
| 447 |
+
if statpath.exists():
|
| 448 |
+
with open(statpath) as f:
|
| 449 |
+
stat = json.load(f)
|
| 450 |
+
res_dict = {scene_name: res for scene_name, res in zip(stat['scene_names'], stat['resolutions'])}
|
| 451 |
+
else:
|
| 452 |
+
with h5py.File(path, 'r') as f:
|
| 453 |
+
res_dict = {}
|
| 454 |
+
for scene_name in f.keys():
|
| 455 |
+
res_dict[scene_name] = f[scene_name]['image'].shape[1:3]
|
| 456 |
+
|
| 457 |
+
with open(meta["samp"]) as f:
|
| 458 |
+
samp = json.load(f)
|
| 459 |
+
|
| 460 |
+
self.scene_names_lst.append([])
|
| 461 |
+
self.input_frame_idx_lst.append([])
|
| 462 |
+
self.target_frame_idx_lst.append([])
|
| 463 |
+
self.image_res_lst.append([])
|
| 464 |
+
self.total_scenes += len(samp)
|
| 465 |
+
|
| 466 |
+
for i_scene, (scene_name, split) in enumerate(samp.items()):
|
| 467 |
+
assert scene_name in res_dict
|
| 468 |
+
self.scene_names_lst[-1].append(scene_name)
|
| 469 |
+
self.image_res_lst[-1].append(res_dict[scene_name])
|
| 470 |
+
self.input_frame_idx_lst[-1].append(split['input'])
|
| 471 |
+
self.target_frame_idx_lst[-1].append(split['heldout'])
|
| 472 |
+
self.remap_idx.append((i_ds, i_scene))
|
| 473 |
+
|
| 474 |
+
def __repr__(self):
|
| 475 |
+
return '\n'.join(
|
| 476 |
+
f' {self.ds_name_lst[i]:30s}: size={len(self.scene_names_lst[i]):6d} / dynamic={self.dynamic_lst[i]}'
|
| 477 |
+
for i in range(len(self.path_lst))
|
| 478 |
+
)
|
| 479 |
+
|
| 480 |
+
def __len__(self):
|
| 481 |
+
return self.total_scenes
|
| 482 |
+
|
| 483 |
+
def __getitem__(self, idx):
|
| 484 |
+
'''
|
| 485 |
+
Read one sequence of a scene.
|
| 486 |
+
'''
|
| 487 |
+
# Remap the flatten index
|
| 488 |
+
i_ds, i_scene = self.remap_idx[idx]
|
| 489 |
+
|
| 490 |
+
# Caculate target resolution
|
| 491 |
+
if self.ap_ratio > 0:
|
| 492 |
+
th = (self.num_pixels / (self.ap_ratio * self.patch_size * self.patch_size)) ** 0.5
|
| 493 |
+
tw = th * self.ap_ratio
|
| 494 |
+
else:
|
| 495 |
+
# The batch size should be 1 or all the datasets are in same aspect ratio
|
| 496 |
+
sh, sw = self.image_res_lst[i_ds][i_scene]
|
| 497 |
+
s_ap_ratio = sw / sh
|
| 498 |
+
th = (self.num_pixels / (s_ap_ratio * self.patch_size * self.patch_size)) ** 0.5
|
| 499 |
+
tw = th * s_ap_ratio
|
| 500 |
+
|
| 501 |
+
th = round(th) * self.patch_size
|
| 502 |
+
tw = round(tw) * self.patch_size
|
| 503 |
+
|
| 504 |
+
# Read each frame
|
| 505 |
+
batch = [
|
| 506 |
+
*[
|
| 507 |
+
super().__getitem__((
|
| 508 |
+
0, False, i_ds, i_scene, i_frame,
|
| 509 |
+
self.ap_ratio, self.cp_ratio, (th, tw)))
|
| 510 |
+
for i_frame in self.input_frame_idx_lst[i_ds][i_scene]
|
| 511 |
+
],
|
| 512 |
+
*[
|
| 513 |
+
super().__getitem__((
|
| 514 |
+
0, True, i_ds, i_scene, i_frame,
|
| 515 |
+
self.ap_ratio, self.cp_ratio, (th, tw)))
|
| 516 |
+
for i_frame in self.target_frame_idx_lst[i_ds][i_scene]
|
| 517 |
+
],
|
| 518 |
+
]
|
| 519 |
+
|
| 520 |
+
# Batchify the scene
|
| 521 |
+
batch = MixedBatchRandomSampler.batchify_sequences(batch)
|
| 522 |
+
|
| 523 |
+
return batch
|
dvsm/loss.py
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import scipy.io
|
| 3 |
+
import numpy as np
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
import torch.nn as nn
|
| 8 |
+
from torchvision.models import vgg19
|
| 9 |
+
|
| 10 |
+
from lpips import LPIPS
|
| 11 |
+
from pytorch_msssim import SSIM
|
| 12 |
+
from skimage.metrics import structural_similarity
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
metric_networks = {}
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
# the perception loss code is modified from https://github.com/zhengqili/Crowdsampling-the-Plenoptic-Function/blob/f5216f312cf82d77f8d20454b5eeb3930324630a/models/networks.py#L1478
|
| 19 |
+
# and some parts are based on https://github.com/arthurhero/Long-LRM/blob/main/model/loss.py
|
| 20 |
+
class PerceptualLoss(nn.Module):
|
| 21 |
+
def __init__(self, device="cpu"):
|
| 22 |
+
super().__init__()
|
| 23 |
+
self.device = device
|
| 24 |
+
self.vgg = self._build_vgg()
|
| 25 |
+
self._load_weights()
|
| 26 |
+
self._setup_feature_blocks()
|
| 27 |
+
|
| 28 |
+
def _build_vgg(self):
|
| 29 |
+
"""Create VGG model with average pooling instead of max pooling."""
|
| 30 |
+
model = vgg19()
|
| 31 |
+
# Replace max pooling with average pooling
|
| 32 |
+
for i, layer in enumerate(model.features):
|
| 33 |
+
if isinstance(layer, nn.MaxPool2d):
|
| 34 |
+
model.features[i] = nn.AvgPool2d(kernel_size=2, stride=2)
|
| 35 |
+
|
| 36 |
+
return model.to(self.device).eval()
|
| 37 |
+
|
| 38 |
+
def _load_weights(self):
|
| 39 |
+
"""Load pre-trained VGG weights. """
|
| 40 |
+
weight_file = Path("./outputs/imagenet-vgg-verydeep-19.mat")
|
| 41 |
+
weight_file.parent.mkdir(exist_ok=True, parents=True)
|
| 42 |
+
|
| 43 |
+
if not weight_file.exists():
|
| 44 |
+
# Download weights if needed
|
| 45 |
+
if not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0:
|
| 46 |
+
os.system(f'wget https://www.vlfeat.org/matconvnet/models/imagenet-vgg-verydeep-19.mat -O {weight_file}')
|
| 47 |
+
if torch.distributed.is_initialized():
|
| 48 |
+
torch.distributed.barrier()
|
| 49 |
+
|
| 50 |
+
# Load MatConvNet weights
|
| 51 |
+
vgg_data = scipy.io.loadmat(weight_file)
|
| 52 |
+
vgg_layers = vgg_data["layers"][0]
|
| 53 |
+
|
| 54 |
+
# Layer indices and filter sizes
|
| 55 |
+
layer_indices = [0, 2, 5, 7, 10, 12, 14, 16, 19, 21, 23, 25, 28, 30, 32, 34]
|
| 56 |
+
filter_sizes = [64, 64, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512, 512, 512, 512, 512]
|
| 57 |
+
|
| 58 |
+
# Transfer weights to PyTorch model
|
| 59 |
+
with torch.no_grad():
|
| 60 |
+
for i, layer_idx in enumerate(layer_indices):
|
| 61 |
+
# Set weights
|
| 62 |
+
weights = torch.from_numpy(vgg_layers[layer_idx][0][0][2][0][0]).permute(3, 2, 0, 1)
|
| 63 |
+
self.vgg.features[layer_idx].weight = nn.Parameter(weights, requires_grad=False)
|
| 64 |
+
|
| 65 |
+
# Set biases
|
| 66 |
+
biases = torch.from_numpy(vgg_layers[layer_idx][0][0][2][0][1]).view(filter_sizes[i])
|
| 67 |
+
self.vgg.features[layer_idx].bias = nn.Parameter(biases, requires_grad=False)
|
| 68 |
+
|
| 69 |
+
def _setup_feature_blocks(self):
|
| 70 |
+
"""Create feature extraction blocks at different network depths."""
|
| 71 |
+
output_indices = [0, 4, 9, 14, 23, 32]
|
| 72 |
+
self.blocks = nn.ModuleList()
|
| 73 |
+
|
| 74 |
+
# Create sequential blocks
|
| 75 |
+
for i in range(len(output_indices) - 1):
|
| 76 |
+
block = nn.Sequential(*list(self.vgg.features[output_indices[i]:output_indices[i+1]]))
|
| 77 |
+
self.blocks.append(block.to(self.device).eval())
|
| 78 |
+
|
| 79 |
+
# Freeze all parameters
|
| 80 |
+
for param in self.vgg.parameters():
|
| 81 |
+
param.requires_grad = False
|
| 82 |
+
|
| 83 |
+
def _extract_features(self, x):
|
| 84 |
+
"""Extract features from each block."""
|
| 85 |
+
features = []
|
| 86 |
+
for block in self.blocks:
|
| 87 |
+
x = block(x)
|
| 88 |
+
features.append(x)
|
| 89 |
+
return features
|
| 90 |
+
|
| 91 |
+
def _preprocess_images(self, images):
|
| 92 |
+
"""Convert images to VGG input format."""
|
| 93 |
+
# VGG mean values for ImageNet
|
| 94 |
+
mean = torch.tensor([123.6800, 116.7790, 103.9390]).reshape(1, 3, 1, 1).to(images.device)
|
| 95 |
+
return images * 255.0 - mean
|
| 96 |
+
|
| 97 |
+
@staticmethod
|
| 98 |
+
def _compute_error(real, fake):
|
| 99 |
+
return torch.mean(torch.abs(real - fake))
|
| 100 |
+
|
| 101 |
+
def forward(self, pred_img, target_img):
|
| 102 |
+
"""Compute perceptual loss between prediction and target."""
|
| 103 |
+
# Preprocess images
|
| 104 |
+
target_img_p = self._preprocess_images(target_img)
|
| 105 |
+
pred_img_p = self._preprocess_images(pred_img)
|
| 106 |
+
|
| 107 |
+
# Extract features
|
| 108 |
+
target_features = self._extract_features(target_img_p)
|
| 109 |
+
pred_features = self._extract_features(pred_img_p)
|
| 110 |
+
|
| 111 |
+
# Pixel-level error
|
| 112 |
+
e0 = self._compute_error(target_img_p, pred_img_p)
|
| 113 |
+
|
| 114 |
+
# Feature-level errors with scaling factors
|
| 115 |
+
e1 = self._compute_error(target_features[0], pred_features[0]) / 2.6
|
| 116 |
+
e2 = self._compute_error(target_features[1], pred_features[1]) / 4.8
|
| 117 |
+
e3 = self._compute_error(target_features[2], pred_features[2]) / 3.7
|
| 118 |
+
e4 = self._compute_error(target_features[3], pred_features[3]) / 5.6
|
| 119 |
+
e5 = self._compute_error(target_features[4], pred_features[4]) * 10 / 1.5
|
| 120 |
+
|
| 121 |
+
# Combine all errors and normalize
|
| 122 |
+
total_loss = (e0 + e1 + e2 + e3 + e4 + e5) / 255.0
|
| 123 |
+
|
| 124 |
+
return total_loss
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def compute_ssim_score(x, y):
|
| 128 |
+
if 'SSIM' not in metric_networks:
|
| 129 |
+
metric_networks['SSIM'] = SSIM(data_range=1, win_size=11, win_sigma=1.5, channel=3).cuda()
|
| 130 |
+
if len(x.shape) == 3:
|
| 131 |
+
x = x.unsqueeze(0)
|
| 132 |
+
if len(y.shape) == 3:
|
| 133 |
+
y = y.unsqueeze(0)
|
| 134 |
+
return metric_networks['SSIM'](x, y)
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def compute_lpips_loss(x, y, net='vgg'):
|
| 138 |
+
key = f'LPIPS_{net}'
|
| 139 |
+
if key not in metric_networks:
|
| 140 |
+
metric_networks[key] = LPIPS(net=net, version='0.1').cuda()
|
| 141 |
+
if len(x.shape) == 3:
|
| 142 |
+
x = x.unsqueeze(0)
|
| 143 |
+
if len(y.shape) == 3:
|
| 144 |
+
y = y.unsqueeze(0)
|
| 145 |
+
return metric_networks[key](x*2-1, y*2-1).mean()
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
class LossComputer(nn.Module):
|
| 149 |
+
def __init__(self,
|
| 150 |
+
lambda_percep,
|
| 151 |
+
lambda_ssim=0.,
|
| 152 |
+
lambda_lpips=0.,
|
| 153 |
+
report_skimage_ssim=False):
|
| 154 |
+
super().__init__()
|
| 155 |
+
self.lambda_percep = lambda_percep
|
| 156 |
+
self.lambda_ssim = lambda_ssim
|
| 157 |
+
self.lambda_lpips = lambda_lpips
|
| 158 |
+
self.report_skimage_ssim = report_skimage_ssim
|
| 159 |
+
self.amp_dtype = torch.bfloat16 if torch.cuda.get_device_capability()[0] >= 8 else torch.float16
|
| 160 |
+
|
| 161 |
+
self.perceptual_loss_module = self._init_frozen_module(PerceptualLoss())
|
| 162 |
+
|
| 163 |
+
def _init_frozen_module(self, module):
|
| 164 |
+
"""Helper method to initialize and freeze a module's parameters."""
|
| 165 |
+
module.eval()
|
| 166 |
+
for param in module.parameters():
|
| 167 |
+
param.requires_grad = False
|
| 168 |
+
return module
|
| 169 |
+
|
| 170 |
+
def forward(self, rendering, target, cameras_pack=None):
|
| 171 |
+
"""
|
| 172 |
+
Calculate various losses between rendering and target images.
|
| 173 |
+
|
| 174 |
+
Args:
|
| 175 |
+
rendering: [b, v, 3, h, w], value range [0, 1]
|
| 176 |
+
target: [b, v, 3, h, w], value range [0, 1]
|
| 177 |
+
|
| 178 |
+
Returns:
|
| 179 |
+
Dictionary of loss metrics
|
| 180 |
+
"""
|
| 181 |
+
b, v, _, h, w = rendering.size()
|
| 182 |
+
rendering = rendering.reshape(b * v, -1, h, w)
|
| 183 |
+
target = target.reshape(b * v, -1, h, w)
|
| 184 |
+
assert target.size(1) == 3
|
| 185 |
+
|
| 186 |
+
loss_metrics = dict()
|
| 187 |
+
|
| 188 |
+
with torch.autocast(enabled=self.amp_dtype is not None, device_type="cuda", dtype=self.amp_dtype):
|
| 189 |
+
mse = (rendering - target).square().mean((1,2,3))
|
| 190 |
+
l2_loss = mse.mean()
|
| 191 |
+
loss = l2_loss
|
| 192 |
+
loss_metrics['psnr'] = -10.0 * torch.log10(mse.detach().float()).mean()
|
| 193 |
+
|
| 194 |
+
if self.lambda_percep > 0:
|
| 195 |
+
perceptual_loss = self.perceptual_loss_module(rendering, target)
|
| 196 |
+
loss = loss + self.lambda_percep * perceptual_loss
|
| 197 |
+
loss_metrics['percep'] = perceptual_loss.detach()
|
| 198 |
+
|
| 199 |
+
if self.lambda_lpips > 0:
|
| 200 |
+
lpips_loss = compute_lpips_loss(rendering, target)
|
| 201 |
+
loss = loss + self.lambda_lpips * lpips_loss
|
| 202 |
+
loss_metrics['lpips'] = lpips_loss.detach()
|
| 203 |
+
|
| 204 |
+
if self.lambda_ssim > 0:
|
| 205 |
+
ssim = compute_ssim_score(rendering.float(), target.float())
|
| 206 |
+
loss = loss + self.lambda_ssim * (1 - ssim)
|
| 207 |
+
loss_metrics['ssim'] = ssim.detach()
|
| 208 |
+
|
| 209 |
+
if self.report_skimage_ssim:
|
| 210 |
+
ski_ssim = []
|
| 211 |
+
for gt, pred in zip(target, rendering):
|
| 212 |
+
ski_ssim.append(structural_similarity(
|
| 213 |
+
gt.detach().float().cpu().numpy(),
|
| 214 |
+
pred.detach().float().cpu().numpy(),
|
| 215 |
+
win_size=11,
|
| 216 |
+
gaussian_weights=True,
|
| 217 |
+
channel_axis=0,
|
| 218 |
+
data_range=1.0,
|
| 219 |
+
))
|
| 220 |
+
ski_ssim = torch.tensor([np.mean(ski_ssim)], dtype=torch.float32, device=loss.device)
|
| 221 |
+
loss_metrics['skimage_ssim'] = ski_ssim.squeeze()
|
| 222 |
+
|
| 223 |
+
loss_metrics['loss'] = loss
|
| 224 |
+
|
| 225 |
+
return loss_metrics
|
dvsm/models/DVSM.py
ADDED
|
@@ -0,0 +1,436 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import math
|
| 3 |
+
from einops import rearrange
|
| 4 |
+
from einops.layers.torch import Rearrange
|
| 5 |
+
import torch
|
| 6 |
+
import torch.nn as nn
|
| 7 |
+
from huggingface_hub import PyTorchModelHubMixin, snapshot_download
|
| 8 |
+
|
| 9 |
+
from .pose_preprocessor import PosePreprocessor
|
| 10 |
+
from .plucker import PluckerEncoding
|
| 11 |
+
from .backbones import DINO, RADIO
|
| 12 |
+
from .blocks import BlockCM, BlockICM, BlockIMCM, BlockCMIM, BlockCMTB, BlockIMCMTB, BlockIMTB
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def _pose_features(c2w_r, c2w_t):
|
| 16 |
+
"""6D per-view feature: camera position β central-axis ray direction."""
|
| 17 |
+
forward_axis = c2w_r[..., :, 2] # camera z-axis in world coords
|
| 18 |
+
return torch.cat([c2w_t, forward_axis], dim=-1) # [B, V, 6]
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@torch.no_grad()
|
| 22 |
+
def _batched_kmeans(x, K, n_iter=20):
|
| 23 |
+
"""Per-batch k-means on points x: [B, V, D] β centroids [B, K, D], labels [B, V].
|
| 24 |
+
|
| 25 |
+
Initializes centroids by spreading them as `K` evenly-spaced views in the input
|
| 26 |
+
sequence (good enough proxy for farthest-point init at small K). Falls back to
|
| 27 |
+
closest-point assignment if a centroid drifts to an empty cluster.
|
| 28 |
+
"""
|
| 29 |
+
B, V, D = x.shape
|
| 30 |
+
if V <= K:
|
| 31 |
+
# Each view its own chunk; pad missing chunks by repeating the last view's centroid.
|
| 32 |
+
idx = torch.arange(K, device=x.device).clamp(max=V - 1)
|
| 33 |
+
centroids = x[:, idx, :].clone()
|
| 34 |
+
labels = torch.arange(V, device=x.device).unsqueeze(0).expand(B, -1).clamp(max=K - 1)
|
| 35 |
+
return centroids, labels
|
| 36 |
+
|
| 37 |
+
# Even-spaced init across the view sequence
|
| 38 |
+
init_idx = torch.linspace(0, V - 1, K, device=x.device).round().long()
|
| 39 |
+
centroids = x[:, init_idx, :].clone() # [B, K, D]
|
| 40 |
+
|
| 41 |
+
for _ in range(n_iter):
|
| 42 |
+
# Assign each view to nearest centroid
|
| 43 |
+
# dists: [B, V, K] = ||x - c||Β²
|
| 44 |
+
dists = (x.unsqueeze(2) - centroids.unsqueeze(1)).pow(2).sum(-1)
|
| 45 |
+
labels = dists.argmin(-1) # [B, V]
|
| 46 |
+
|
| 47 |
+
# Update: centroid_k = mean of points with label k (per batch)
|
| 48 |
+
one_hot = torch.nn.functional.one_hot(labels, num_classes=K).to(x.dtype) # [B, V, K]
|
| 49 |
+
counts = one_hot.sum(dim=1).clamp(min=1).unsqueeze(-1) # [B, K, 1]
|
| 50 |
+
new_centroids = torch.einsum('bvk,bvd->bkd', one_hot, x) / counts
|
| 51 |
+
|
| 52 |
+
# Empty-cluster guard: if a chunk got 0 points, keep its old centroid
|
| 53 |
+
empty = (one_hot.sum(dim=1) == 0).unsqueeze(-1) # [B, K, 1]
|
| 54 |
+
centroids = torch.where(empty, centroids, new_centroids)
|
| 55 |
+
|
| 56 |
+
# Final assignment with updated centroids
|
| 57 |
+
dists = (x.unsqueeze(2) - centroids.unsqueeze(1)).pow(2).sum(-1)
|
| 58 |
+
labels = dists.argmin(-1)
|
| 59 |
+
return centroids, labels
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class DecoderViewSynthesisModel(nn.Module, PyTorchModelHubMixin):
|
| 63 |
+
def __init__(self,
|
| 64 |
+
patch_size=8,
|
| 65 |
+
latent_dim=768,
|
| 66 |
+
pose_enc='plucker',
|
| 67 |
+
render_up_scale=0.,
|
| 68 |
+
recon_up_scale=0.,
|
| 69 |
+
auto_divisible_pad=False,
|
| 70 |
+
|
| 71 |
+
backbone='',
|
| 72 |
+
freeze_backbone=False,
|
| 73 |
+
|
| 74 |
+
num_register=0,
|
| 75 |
+
|
| 76 |
+
block_type='IMCM',
|
| 77 |
+
num_layers=12,
|
| 78 |
+
head_dim=64,
|
| 79 |
+
expand_ratio=4,
|
| 80 |
+
use_bias=False,
|
| 81 |
+
use_qk_norm=True,
|
| 82 |
+
use_gating=False,
|
| 83 |
+
use_swiglumlp=False,
|
| 84 |
+
use_sandwich=False,
|
| 85 |
+
ttb_feedback=True,
|
| 86 |
+
ttb_use_gating=True,
|
| 87 |
+
ttb_head_dim=0,
|
| 88 |
+
ttb_inter_multi=4.0,
|
| 89 |
+
ttb_base_lr=0.01,
|
| 90 |
+
ttb_muon_steps=5,
|
| 91 |
+
ttb_num_steps=3,
|
| 92 |
+
ttb_target_residual=True,
|
| 93 |
+
ttb_pre_cm_target=True,
|
| 94 |
+
ttb_use_mse_loss=True,
|
| 95 |
+
ttb_per_step_norm=False,
|
| 96 |
+
ttb_share_qk_proj=True,
|
| 97 |
+
ttb_per_cluster_K=0,
|
| 98 |
+
ttb_per_cluster_boundary_ratio=0.7,
|
| 99 |
+
use_amp=True,
|
| 100 |
+
use_compile=True,
|
| 101 |
+
grad_checkpoint=True,
|
| 102 |
+
**block_extra_kwargs):
|
| 103 |
+
super().__init__()
|
| 104 |
+
|
| 105 |
+
self.patch_size = patch_size
|
| 106 |
+
self.latent_dim = latent_dim
|
| 107 |
+
self.plucker_enc = PluckerEncoding(pose_enc)
|
| 108 |
+
self.render_up_scale = render_up_scale
|
| 109 |
+
self.recon_up_scale = recon_up_scale
|
| 110 |
+
self.auto_divisible_pad = auto_divisible_pad
|
| 111 |
+
self.ttb_per_cluster_K = int(ttb_per_cluster_K)
|
| 112 |
+
self.ttb_per_cluster_boundary_ratio = float(ttb_per_cluster_boundary_ratio)
|
| 113 |
+
self._in_per_cluster_call = False
|
| 114 |
+
|
| 115 |
+
if use_amp:
|
| 116 |
+
self.amp_dtype = torch.bfloat16 if torch.cuda.get_device_capability()[0] >= 8 else torch.float16
|
| 117 |
+
else:
|
| 118 |
+
self.amp_dtype = None
|
| 119 |
+
|
| 120 |
+
# Input layers
|
| 121 |
+
self.inp_cam_embd = nn.Sequential(
|
| 122 |
+
Rearrange("b v c (hh ph) (ww pw) -> b v (hh ww) (ph pw c)", ph=self.patch_size, pw=self.patch_size),
|
| 123 |
+
nn.Linear(self.plucker_enc.dim * (self.patch_size**2), latent_dim, bias=False))
|
| 124 |
+
self.inp_img_embd = nn.Sequential(
|
| 125 |
+
Rearrange("b v c (hh ph) (ww pw) -> b v (hh ww) (ph pw c)", ph=self.patch_size, pw=self.patch_size),
|
| 126 |
+
nn.Linear(3 * (self.patch_size**2), latent_dim, bias=False))
|
| 127 |
+
self.inp_norm = nn.LayerNorm(latent_dim, bias=False)
|
| 128 |
+
|
| 129 |
+
if num_register > 0:
|
| 130 |
+
self.register_tokens = torch.nn.Parameter(torch.randn(1,1,num_register,self.latent_dim))
|
| 131 |
+
nn.init.trunc_normal_(self.register_tokens, std=0.02)
|
| 132 |
+
else:
|
| 133 |
+
self.register_tokens = None
|
| 134 |
+
|
| 135 |
+
# Compose a series of repeated blocks
|
| 136 |
+
if block_type == 'CM':
|
| 137 |
+
BLOCK = BlockCM
|
| 138 |
+
elif block_type == 'ICM':
|
| 139 |
+
BLOCK = BlockICM
|
| 140 |
+
elif block_type == 'IMCM':
|
| 141 |
+
BLOCK = BlockIMCM
|
| 142 |
+
elif block_type == 'CMIM':
|
| 143 |
+
BLOCK = BlockCMIM
|
| 144 |
+
elif block_type == 'CMTB':
|
| 145 |
+
BLOCK = BlockCMTB
|
| 146 |
+
elif block_type == 'IMCMTB':
|
| 147 |
+
BLOCK = BlockIMCMTB
|
| 148 |
+
elif block_type == 'IMTB':
|
| 149 |
+
BLOCK = BlockIMTB
|
| 150 |
+
else:
|
| 151 |
+
raise NotImplementedError(f"Unknown block_type={block_type}")
|
| 152 |
+
|
| 153 |
+
block_kwargs = dict(
|
| 154 |
+
dim=latent_dim,
|
| 155 |
+
head_dim=head_dim,
|
| 156 |
+
bias=use_bias,
|
| 157 |
+
use_qk_norm=use_qk_norm,
|
| 158 |
+
use_gating=use_gating,
|
| 159 |
+
expand_ratio=expand_ratio,
|
| 160 |
+
use_swiglumlp=use_swiglumlp,
|
| 161 |
+
use_sandwich=use_sandwich,
|
| 162 |
+
ttb_feedback=ttb_feedback,
|
| 163 |
+
ttb_use_gating=ttb_use_gating,
|
| 164 |
+
ttb_head_dim=ttb_head_dim,
|
| 165 |
+
ttb_inter_multi=ttb_inter_multi,
|
| 166 |
+
ttb_base_lr=ttb_base_lr,
|
| 167 |
+
ttb_muon_steps=ttb_muon_steps,
|
| 168 |
+
ttb_num_steps=ttb_num_steps,
|
| 169 |
+
ttb_target_residual=ttb_target_residual,
|
| 170 |
+
ttb_pre_cm_target=ttb_pre_cm_target,
|
| 171 |
+
ttb_use_mse_loss=ttb_use_mse_loss,
|
| 172 |
+
ttb_per_step_norm=ttb_per_step_norm,
|
| 173 |
+
ttb_share_qk_proj=ttb_share_qk_proj,
|
| 174 |
+
**block_extra_kwargs,
|
| 175 |
+
)
|
| 176 |
+
self.blocks = nn.ModuleList([BLOCK(**block_kwargs) for _ in range(num_layers)])
|
| 177 |
+
self.use_dual_stream = block_type in ('CMTB', 'IMCMTB', 'IMTB')
|
| 178 |
+
|
| 179 |
+
# Output layers
|
| 180 |
+
self.rgb_head = nn.Sequential(
|
| 181 |
+
nn.LayerNorm(latent_dim, bias=False),
|
| 182 |
+
nn.Linear(latent_dim, (self.patch_size**2) * 3, bias=False),
|
| 183 |
+
nn.Sigmoid())
|
| 184 |
+
|
| 185 |
+
# Initialize all linear layers
|
| 186 |
+
for name, m in self.named_modules():
|
| 187 |
+
if isinstance(m, torch.nn.Linear):
|
| 188 |
+
torch.nn.init.normal_(m.weight, mean=0.0, std=0.02)
|
| 189 |
+
if m.bias is not None:
|
| 190 |
+
torch.nn.init.zeros_(m.bias)
|
| 191 |
+
|
| 192 |
+
# Initialize the output projection of a residual
|
| 193 |
+
special_std = 0.02 / math.sqrt(sum(blk.num_residual for blk in self.blocks))
|
| 194 |
+
for blk in self.blocks:
|
| 195 |
+
blk.init_residual(special_std)
|
| 196 |
+
|
| 197 |
+
# Apply torch.compile
|
| 198 |
+
if use_compile:
|
| 199 |
+
self.rgb_head.forward = torch.compile(self.rgb_head.forward, dynamic=True)
|
| 200 |
+
for blk in self.blocks:
|
| 201 |
+
blk.apply_compile()
|
| 202 |
+
|
| 203 |
+
# Apply activation checkpointing
|
| 204 |
+
if grad_checkpoint:
|
| 205 |
+
for blk in self.blocks:
|
| 206 |
+
blk.apply_checkpoint()
|
| 207 |
+
|
| 208 |
+
# Use pre-trained foundation feature extractor
|
| 209 |
+
if 'dino' in backbone:
|
| 210 |
+
self.backbone = DINO(backbone, latent_dim, grad_checkpoint, use_compile)
|
| 211 |
+
elif 'radio' in backbone:
|
| 212 |
+
self.backbone = RADIO(backbone, latent_dim, grad_checkpoint, use_compile)
|
| 213 |
+
else:
|
| 214 |
+
assert backbone == '', f"Unknown backbone model: {backbone}"
|
| 215 |
+
self.backbone = None
|
| 216 |
+
|
| 217 |
+
if self.backbone is not None:
|
| 218 |
+
assert self.patch_size == self.backbone.patch_size, "Inconsistent patch size"
|
| 219 |
+
for param in self.backbone.parameters():
|
| 220 |
+
if freeze_backbone:
|
| 221 |
+
param.requires_grad = False
|
| 222 |
+
else:
|
| 223 |
+
param.use_lr_scaler = True
|
| 224 |
+
|
| 225 |
+
def forward(self, image, intr, c2w_r, c2w_t, pose_preproc_done=False):
|
| 226 |
+
''' Process images and return a rendering function for the input scenes. '''
|
| 227 |
+
|
| 228 |
+
# Per-cluster boundary-adaptive short-circuit: K independent V_sub
|
| 229 |
+
# sub-forwards each running on its own k-means cluster (local pose
|
| 230 |
+
# norm + per-cluster cross-attn). See PerClusterRenderer for details.
|
| 231 |
+
if self.ttb_per_cluster_K > 0 and not self._in_per_cluster_call:
|
| 232 |
+
# K>0 enables eval-only per-cluster mode. Cap at V (more clusters
|
| 233 |
+
# than views is meaningless) and skip when K=1 (degenerate to
|
| 234 |
+
# single-cluster = baseline single forward).
|
| 235 |
+
K = min(self.ttb_per_cluster_K, image.shape[1])
|
| 236 |
+
if K > 1:
|
| 237 |
+
return PerClusterRenderer(
|
| 238 |
+
self, image, intr, c2w_r, c2w_t, K=K,
|
| 239 |
+
boundary_ratio=self.ttb_per_cluster_boundary_ratio,
|
| 240 |
+
)
|
| 241 |
+
|
| 242 |
+
# Normalize camera poses
|
| 243 |
+
if pose_preproc_done:
|
| 244 |
+
pose_preprocessor = None
|
| 245 |
+
else:
|
| 246 |
+
pose_preprocessor = PosePreprocessor(c2w_r, c2w_t)
|
| 247 |
+
c2w_r, c2w_t = pose_preprocessor(c2w_r, c2w_t)
|
| 248 |
+
|
| 249 |
+
# Process input images
|
| 250 |
+
if self.recon_up_scale > 1:
|
| 251 |
+
B, V = image.shape[:2]
|
| 252 |
+
image = torch.nn.functional.interpolate(
|
| 253 |
+
image.flatten(0,1),
|
| 254 |
+
scale_factor=self.recon_up_scale, mode='bilinear').unflatten(0,(B,V))
|
| 255 |
+
|
| 256 |
+
B, V, _, H, W = image.shape
|
| 257 |
+
plucker_map = self.plucker_enc(intr, c2w_r, c2w_t, H, W)
|
| 258 |
+
|
| 259 |
+
# TODO: camera param may need to update for PRope? Move it to data loader?
|
| 260 |
+
if self.auto_divisible_pad:
|
| 261 |
+
padH = (self.patch_size - (H % self.patch_size)) % self.patch_size
|
| 262 |
+
padW = (self.patch_size - (W % self.patch_size)) % self.patch_size
|
| 263 |
+
if padH > 0 or padW > 0:
|
| 264 |
+
image = torch.nn.functional.pad(image, (0, padW, 0, padH))
|
| 265 |
+
plucker_map = torch.nn.functional.pad(plucker_map, (0, padW, 0, padH))
|
| 266 |
+
|
| 267 |
+
# Running the model
|
| 268 |
+
with torch.autocast(enabled=self.amp_dtype is not None, device_type="cuda", dtype=self.amp_dtype):
|
| 269 |
+
|
| 270 |
+
# Input tokenization
|
| 271 |
+
x_cam_embd = self.inp_cam_embd(plucker_map)
|
| 272 |
+
x_img_embd = self.inp_img_embd(image)
|
| 273 |
+
x = x_cam_embd + x_img_embd
|
| 274 |
+
|
| 275 |
+
# Foundation prior feature injection
|
| 276 |
+
if self.backbone is not None:
|
| 277 |
+
x = x + self.backbone(image)
|
| 278 |
+
|
| 279 |
+
# Append register tokens
|
| 280 |
+
if self.register_tokens is not None:
|
| 281 |
+
register_tokens = self.register_tokens.repeat(B,V,1,1)
|
| 282 |
+
x = torch.cat([register_tokens, x], dim=2)
|
| 283 |
+
|
| 284 |
+
# Input token normalization
|
| 285 |
+
x = self.inp_norm(x)
|
| 286 |
+
|
| 287 |
+
# Transformer feed-forward
|
| 288 |
+
kv_lst = []
|
| 289 |
+
if self.use_dual_stream:
|
| 290 |
+
# Dual-stream: prepare cam-only tokens and concatenate along V
|
| 291 |
+
if self.register_tokens is not None:
|
| 292 |
+
x_cam = torch.cat([register_tokens, x_cam_embd], dim=2)
|
| 293 |
+
else:
|
| 294 |
+
x_cam = x_cam_embd
|
| 295 |
+
x_cam = self.inp_norm(x_cam)
|
| 296 |
+
x = torch.cat([x, x_cam], dim=1) # [B, 2V, L, D]
|
| 297 |
+
|
| 298 |
+
for block in self.blocks:
|
| 299 |
+
x, kv = block(x)
|
| 300 |
+
kv_lst.append(kv)
|
| 301 |
+
|
| 302 |
+
return NeuralRenderer(pose_preprocessor, self, kv_lst)
|
| 303 |
+
|
| 304 |
+
@classmethod
|
| 305 |
+
def from_hf_pretrained(cls, variant, repo_id="nvidia/DVSM"):
|
| 306 |
+
local_dir = snapshot_download(repo_id=repo_id, allow_patterns=os.path.join(variant, "*"))
|
| 307 |
+
model = cls.from_pretrained(os.path.join(local_dir, variant))
|
| 308 |
+
return model
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
class PerClusterRenderer:
|
| 312 |
+
def __init__(self, model, image, intr, c2w_r, c2w_t, K, boundary_ratio):
|
| 313 |
+
self.model = model
|
| 314 |
+
pose_feat = _pose_features(c2w_r, c2w_t) # [B, V, D]
|
| 315 |
+
B, V, _ = pose_feat.shape
|
| 316 |
+
K_eff = min(K, V)
|
| 317 |
+
centroids, _ = _batched_kmeans(pose_feat, K_eff) # [B, K, D]
|
| 318 |
+
dists_sq = (pose_feat.unsqueeze(2) - centroids.unsqueeze(1)).pow(2).sum(-1) # [B, V, K]
|
| 319 |
+
|
| 320 |
+
# Per-view membership: primary (always) + secondary when boundary.
|
| 321 |
+
top2 = dists_sq.topk(min(2, K_eff), dim=-1, largest=False)
|
| 322 |
+
primary = top2.indices[..., 0] # [B, V]
|
| 323 |
+
membership = torch.zeros(B, V, K_eff, dtype=torch.bool, device=pose_feat.device)
|
| 324 |
+
membership.scatter_(2, primary.unsqueeze(-1), True)
|
| 325 |
+
if K_eff >= 2 and boundary_ratio > 0:
|
| 326 |
+
d1_sq = top2.values[..., 0]
|
| 327 |
+
d2_sq = top2.values[..., 1]
|
| 328 |
+
secondary = top2.indices[..., 1]
|
| 329 |
+
# Squared form of d1/d2 > boundary_ratio.
|
| 330 |
+
is_boundary = (d1_sq / d2_sq.clamp(min=1e-12)) > (boundary_ratio ** 2)
|
| 331 |
+
sec_mask = torch.zeros_like(membership)
|
| 332 |
+
sec_mask.scatter_(2, secondary.unsqueeze(-1), True)
|
| 333 |
+
membership = membership | (sec_mask & is_boundary.unsqueeze(-1))
|
| 334 |
+
|
| 335 |
+
self.cluster_centroids = centroids
|
| 336 |
+
self.cluster_renderers = []
|
| 337 |
+
assert B == 1, "PerClusterRenderer is implemented for B=1 (use --rend_bs 1)"
|
| 338 |
+
was_in = model._in_per_cluster_call
|
| 339 |
+
model._in_per_cluster_call = True
|
| 340 |
+
try:
|
| 341 |
+
for k in range(K_eff):
|
| 342 |
+
idx_k = membership[0, :, k].nonzero(as_tuple=True)[0] # variable size
|
| 343 |
+
sub_renderer = model(
|
| 344 |
+
image[:, idx_k].to(intr.device),
|
| 345 |
+
intr[:, idx_k], c2w_r[:, idx_k], c2w_t[:, idx_k],
|
| 346 |
+
)
|
| 347 |
+
self.cluster_renderers.append(sub_renderer)
|
| 348 |
+
finally:
|
| 349 |
+
model._in_per_cluster_call = was_in
|
| 350 |
+
|
| 351 |
+
def __call__(self, intr, c2w_r, c2w_t, H, W, ts=None):
|
| 352 |
+
B, T = intr.shape[:2]
|
| 353 |
+
assert B == 1, "PerClusterRenderer expects B=1 (use --rend_bs 1)"
|
| 354 |
+
tgt_pose = _pose_features(c2w_r, c2w_t)
|
| 355 |
+
d = (tgt_pose.unsqueeze(2) - self.cluster_centroids.unsqueeze(1)).pow(2).sum(-1)
|
| 356 |
+
cluster_idx = d.argmin(dim=-1) # [B, T]
|
| 357 |
+
outputs = []
|
| 358 |
+
for t in range(T):
|
| 359 |
+
k = int(cluster_idx[0, t].item())
|
| 360 |
+
out_t = self.cluster_renderers[k](
|
| 361 |
+
intr[:, t:t+1], c2w_r[:, t:t+1], c2w_t[:, t:t+1], H, W, ts)
|
| 362 |
+
outputs.append(out_t)
|
| 363 |
+
return torch.cat(outputs, dim=1)
|
| 364 |
+
|
| 365 |
+
|
| 366 |
+
class NeuralRenderer:
|
| 367 |
+
def __init__(self, pose_preprocessor, model, kv_lst):
|
| 368 |
+
self.pose_preprocessor = pose_preprocessor
|
| 369 |
+
self.model = model
|
| 370 |
+
self.kv_lst = kv_lst
|
| 371 |
+
assert len(self.model.blocks) == len(self.kv_lst)
|
| 372 |
+
|
| 373 |
+
def __call__(self, intr, c2w_r, c2w_t, H, W, ts=None):
|
| 374 |
+
|
| 375 |
+
# Process poses
|
| 376 |
+
if self.pose_preprocessor is not None:
|
| 377 |
+
c2w_r, c2w_t = self.pose_preprocessor(c2w_r, c2w_t)
|
| 378 |
+
|
| 379 |
+
# Process input images
|
| 380 |
+
if self.model.render_up_scale > 1:
|
| 381 |
+
H_, W_ = H, W
|
| 382 |
+
H = round(H * self.model.render_up_scale)
|
| 383 |
+
W = round(W * self.model.render_up_scale)
|
| 384 |
+
|
| 385 |
+
B, V = intr.shape[:2]
|
| 386 |
+
plucker_map = self.model.plucker_enc(intr, c2w_r, c2w_t, H, W)
|
| 387 |
+
|
| 388 |
+
padH, padW = 0, 0
|
| 389 |
+
if self.model.auto_divisible_pad:
|
| 390 |
+
padH = (self.model.patch_size - (H % self.model.patch_size)) % self.model.patch_size
|
| 391 |
+
padW = (self.model.patch_size - (W % self.model.patch_size)) % self.model.patch_size
|
| 392 |
+
if padH > 0 or padW > 0:
|
| 393 |
+
plucker_map = torch.nn.functional.pad(plucker_map, (0, padW, 0, padH))
|
| 394 |
+
|
| 395 |
+
# Run renderer
|
| 396 |
+
with torch.autocast(enabled=self.model.amp_dtype is not None, device_type="cuda", dtype=self.model.amp_dtype):
|
| 397 |
+
|
| 398 |
+
# Input tokenization
|
| 399 |
+
x = self.model.inp_cam_embd(plucker_map)
|
| 400 |
+
|
| 401 |
+
# Append register tokens
|
| 402 |
+
if self.model.register_tokens is not None:
|
| 403 |
+
register_tokens = self.model.register_tokens.repeat(B,V,1,1)
|
| 404 |
+
x = torch.cat([register_tokens, x], dim=2)
|
| 405 |
+
|
| 406 |
+
# Input token normalization
|
| 407 |
+
x = self.model.inp_norm(x)
|
| 408 |
+
|
| 409 |
+
# Transformer feed-forward
|
| 410 |
+
for block, kv in zip(self.model.blocks, self.kv_lst):
|
| 411 |
+
x, _ = block(x, kv=kv)
|
| 412 |
+
|
| 413 |
+
# Remove register tokens
|
| 414 |
+
if self.model.register_tokens is not None:
|
| 415 |
+
x = x[:, :, register_tokens.shape[2]:]
|
| 416 |
+
|
| 417 |
+
# Predict rgb
|
| 418 |
+
image = self.model.rgb_head(x)
|
| 419 |
+
image = rearrange(
|
| 420 |
+
image,
|
| 421 |
+
"b v (hh ww) (ph pw c) -> b v c (hh ph) (ww pw)",
|
| 422 |
+
hh=(H+padH) // self.model.patch_size,
|
| 423 |
+
ww=(W+padW) // self.model.patch_size,
|
| 424 |
+
ph=self.model.patch_size,
|
| 425 |
+
pw=self.model.patch_size,
|
| 426 |
+
c=3)
|
| 427 |
+
|
| 428 |
+
if self.model.auto_divisible_pad and (padH > 0 or padW > 0):
|
| 429 |
+
image = image[..., :H, :W]
|
| 430 |
+
|
| 431 |
+
if self.model.render_up_scale > 1:
|
| 432 |
+
image = torch.nn.functional.interpolate(
|
| 433 |
+
image.flatten(0,1),
|
| 434 |
+
size=(H_, W_), mode='bilinear', antialias=True).unflatten(0,(B,V))
|
| 435 |
+
|
| 436 |
+
return image
|
dvsm/models/backbones.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from transformers import AutoModel
|
| 3 |
+
|
| 4 |
+
from .modules import checkpoint_wrapper
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class DINO(torch.nn.Module):
|
| 8 |
+
def __init__(self, version, latent_dim, grad_checkpoint=True, use_compile=True):
|
| 9 |
+
super().__init__()
|
| 10 |
+
if version == 'dino' or version == 'dino-large':
|
| 11 |
+
version = "facebook/dinov3-vitl16-pretrain-lvd1689m"
|
| 12 |
+
if version == 'dino-big':
|
| 13 |
+
version = "facebook/dinov3-vitb16-pretrain-lvd1689m"
|
| 14 |
+
if version == 'dino-huge':
|
| 15 |
+
version = "facebook/dinov3-vith16plus-pretrain-lvd1689m"
|
| 16 |
+
|
| 17 |
+
# Prepare the model
|
| 18 |
+
try:
|
| 19 |
+
self.prior = AutoModel.from_pretrained(version, local_files_only=True)
|
| 20 |
+
except:
|
| 21 |
+
print(f"Fail to load pretrained foundation model. May be due to missing local file.")
|
| 22 |
+
print(f"Open an python terminal and run the following to download to local first.")
|
| 23 |
+
print(f" from transformers import AutoModel; AutoModel.from_pretrained({version})")
|
| 24 |
+
raise Exception
|
| 25 |
+
|
| 26 |
+
self.patch_size = self.prior.config.patch_size
|
| 27 |
+
self.prior_prefix = self.prior.embeddings.cls_token.shape[1] + self.prior.embeddings.register_tokens.shape[1]
|
| 28 |
+
del self.prior.embeddings.mask_token
|
| 29 |
+
self.register_buffer("img_mean", torch.tensor([0.485, 0.456, 0.406]).view(3,1,1))
|
| 30 |
+
self.register_buffer("img_std", torch.tensor([0.229, 0.224, 0.225]).view(3,1,1))
|
| 31 |
+
|
| 32 |
+
# Zero-init channel projection layer
|
| 33 |
+
self.out_proj = torch.nn.Linear(self.prior.config.hidden_size, latent_dim, bias=False)
|
| 34 |
+
torch.nn.init.constant_(self.out_proj.weight, 0.)
|
| 35 |
+
|
| 36 |
+
# Apply grad checkpointing & compile
|
| 37 |
+
for i in range(len(self.prior.layer)):
|
| 38 |
+
if use_compile:
|
| 39 |
+
self.prior.layer[i].forward = torch.compile(self.prior.layer[i].forward)
|
| 40 |
+
if grad_checkpoint:
|
| 41 |
+
self.prior.layer[i].forward = checkpoint_wrapper(self.prior.layer[i])
|
| 42 |
+
|
| 43 |
+
def forward(self, image):
|
| 44 |
+
# image: [B V 3 H W] under intensity range [0, 1]
|
| 45 |
+
x = self.prior((image.flatten(0,1) - self.img_mean) / self.img_std)
|
| 46 |
+
x = x.last_hidden_state[:, self.prior_prefix:]
|
| 47 |
+
x = self.out_proj(x)
|
| 48 |
+
x = x.unflatten(0, image.shape[:2])
|
| 49 |
+
return x
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class RADIO(torch.nn.Module):
|
| 53 |
+
def __init__(self, version, latent_dim, grad_checkpoint=True, use_compile=True):
|
| 54 |
+
super().__init__()
|
| 55 |
+
if version == 'radio' or version == 'radio-large' or version == 'radio-so400m':
|
| 56 |
+
version = "c-radio_v4-so400m"
|
| 57 |
+
if version == 'radio-huge':
|
| 58 |
+
version = "c-radio_v4-h"
|
| 59 |
+
|
| 60 |
+
# Prepare the model
|
| 61 |
+
# TODO: may cause bug if weight is not downloaded with multi-gpu training
|
| 62 |
+
self.prior = torch.hub.load('NVlabs/RADIO', 'radio_model', version=version, skip_validation=True)
|
| 63 |
+
self.patch_size = self.prior._patch_size
|
| 64 |
+
|
| 65 |
+
# Zero-init channel projection layer
|
| 66 |
+
self.out_proj = torch.nn.Linear(self.prior.model.embed_dim, latent_dim, bias=False)
|
| 67 |
+
torch.nn.init.constant_(self.out_proj.weight, 0.)
|
| 68 |
+
|
| 69 |
+
# Apply grad checkpointing & compile
|
| 70 |
+
for i in range(len(self.prior.model.blocks)):
|
| 71 |
+
if use_compile:
|
| 72 |
+
self.prior.model.blocks[i].forward = torch.compile(self.prior.model.blocks[i].forward)
|
| 73 |
+
if grad_checkpoint:
|
| 74 |
+
self.prior.model.blocks[i].forward = checkpoint_wrapper(self.prior.model.blocks[i])
|
| 75 |
+
|
| 76 |
+
def forward(self, image):
|
| 77 |
+
# image: [B V 3 H W] under intensity range [0, 1]
|
| 78 |
+
x = self.prior(image.flatten(0,1))
|
| 79 |
+
x = x.features
|
| 80 |
+
x = self.out_proj(x)
|
| 81 |
+
x = x.unflatten(0, image.shape[:2])
|
| 82 |
+
return x
|
dvsm/models/blocks.py
ADDED
|
@@ -0,0 +1,403 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
from .modules import checkpoint_wrapper, MLP, SwiGluMLP, IntraViewAttention, CrossViewAttention
|
| 4 |
+
from .ttb import FastWeightGluMLPMultiheadTTB, TTBOperator
|
| 5 |
+
|
| 6 |
+
'''
|
| 7 |
+
Basic layers
|
| 8 |
+
I: IntraViewAttention
|
| 9 |
+
C: CrossViewAttention
|
| 10 |
+
M: MLP
|
| 11 |
+
D: TTB / TTB (Test-Time Baking)
|
| 12 |
+
|
| 13 |
+
Block types
|
| 14 |
+
IM
|
| 15 |
+
CM
|
| 16 |
+
ICM
|
| 17 |
+
IMCM (DVSM: IntraView + MLP + CrossView + MLP)
|
| 18 |
+
CMIM
|
| 19 |
+
CMTB (CrossView + MLP + TTB)
|
| 20 |
+
IMCMTB (TTB: IntraView + MLP + CrossView + MLP + TTB)
|
| 21 |
+
IMTB (TTB--: IntraView + MLP + TTB, dual-stream recon, ttb_feedback controls apply to x_img)
|
| 22 |
+
'''
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _build_ttb_op_order(total_tokens, update, apply, num_steps=1):
|
| 26 |
+
if num_steps > 1 and update:
|
| 27 |
+
# Multi-step: repeat full-batch update, apply only on last step
|
| 28 |
+
ops = [TTBOperator(start=0, end=total_tokens, update=True, apply=False)
|
| 29 |
+
for _ in range(num_steps - 1)]
|
| 30 |
+
ops.append(TTBOperator(start=0, end=total_tokens, update=True, apply=apply))
|
| 31 |
+
return ops
|
| 32 |
+
return [TTBOperator(start=0, end=total_tokens, update=update, apply=apply)]
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class BlockBase(torch.nn.Module):
|
| 36 |
+
def __init__(self,
|
| 37 |
+
dim,
|
| 38 |
+
head_dim,
|
| 39 |
+
bias=False,
|
| 40 |
+
use_qk_norm=True,
|
| 41 |
+
use_gating=False,
|
| 42 |
+
expand_ratio=4,
|
| 43 |
+
use_swiglumlp=False,
|
| 44 |
+
use_sandwich=False,
|
| 45 |
+
**kwargs, # ignore TTB/fast-weight kwargs forwarded to base blocks
|
| 46 |
+
):
|
| 47 |
+
super().__init__()
|
| 48 |
+
self.dim = dim
|
| 49 |
+
self.head_dim = head_dim
|
| 50 |
+
self.bias = bias
|
| 51 |
+
self.use_qk_norm = use_qk_norm
|
| 52 |
+
self.use_gating = use_gating
|
| 53 |
+
self.use_sandwich = use_sandwich
|
| 54 |
+
|
| 55 |
+
self.norm_mlp = torch.nn.LayerNorm(dim, bias=bias)
|
| 56 |
+
if use_swiglumlp:
|
| 57 |
+
self.mlp = SwiGluMLP(dim=dim, expand_ratio=expand_ratio, bias=bias)
|
| 58 |
+
else:
|
| 59 |
+
self.mlp = MLP(dim=dim, expand_ratio=expand_ratio, bias=bias)
|
| 60 |
+
|
| 61 |
+
@property
|
| 62 |
+
def num_residual(self):
|
| 63 |
+
return 1
|
| 64 |
+
|
| 65 |
+
def fw_attn(self, x, kv):
|
| 66 |
+
raise NotImplementedError
|
| 67 |
+
|
| 68 |
+
def fw_mlp(self, x):
|
| 69 |
+
x = x + self.mlp(self.norm_mlp(x))
|
| 70 |
+
return x
|
| 71 |
+
|
| 72 |
+
def apply_compile(self):
|
| 73 |
+
self.fw_attn = torch.compile(self.fw_attn, dynamic=True)
|
| 74 |
+
self.fw_mlp = torch.compile(self.fw_mlp, dynamic=True)
|
| 75 |
+
|
| 76 |
+
def apply_checkpoint(self):
|
| 77 |
+
self.forward = checkpoint_wrapper(self)
|
| 78 |
+
|
| 79 |
+
def forward(self, x, kv=None):
|
| 80 |
+
x, kv = self.fw_attn(x, kv)
|
| 81 |
+
x = self.fw_mlp(x)
|
| 82 |
+
return x, kv
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
class BlockIM(BlockBase):
|
| 86 |
+
def __init__(self, *args, **kwargs):
|
| 87 |
+
super().__init__(*args, **kwargs)
|
| 88 |
+
self.norm1 = torch.nn.LayerNorm(self.dim, bias=self.bias)
|
| 89 |
+
self.attn = IntraViewAttention(
|
| 90 |
+
dim=self.dim,
|
| 91 |
+
head_dim=self.head_dim,
|
| 92 |
+
bias=self.bias,
|
| 93 |
+
use_qk_norm=self.use_qk_norm,
|
| 94 |
+
use_gating=self.use_gating,
|
| 95 |
+
use_sandwich=self.use_sandwich)
|
| 96 |
+
|
| 97 |
+
@property
|
| 98 |
+
def num_residual(self):
|
| 99 |
+
return 2
|
| 100 |
+
|
| 101 |
+
def init_residual(self, std):
|
| 102 |
+
torch.nn.init.normal_(self.attn.fc.weight, std=std)
|
| 103 |
+
torch.nn.init.normal_(self.mlp.mlp[-1].weight, std=std)
|
| 104 |
+
|
| 105 |
+
def fw_attn(self, x, kv=None):
|
| 106 |
+
B, V, L, D = x.shape
|
| 107 |
+
x = x.view(B*V, L, D)
|
| 108 |
+
x = x + self.attn(self.norm1(x))
|
| 109 |
+
x = x.view(B, V, L, D)
|
| 110 |
+
return x, kv
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
class BlockCM(BlockBase):
|
| 114 |
+
def __init__(self, *args, **kwargs):
|
| 115 |
+
super().__init__(*args, **kwargs)
|
| 116 |
+
self.norm1 = torch.nn.LayerNorm(self.dim, bias=self.bias)
|
| 117 |
+
self.attn = CrossViewAttention(
|
| 118 |
+
dim=self.dim,
|
| 119 |
+
head_dim=self.head_dim,
|
| 120 |
+
bias=self.bias,
|
| 121 |
+
use_qk_norm=self.use_qk_norm,
|
| 122 |
+
use_gating=self.use_gating,
|
| 123 |
+
use_sandwich=self.use_sandwich)
|
| 124 |
+
|
| 125 |
+
@property
|
| 126 |
+
def num_residual(self):
|
| 127 |
+
return 2
|
| 128 |
+
|
| 129 |
+
def init_residual(self, std):
|
| 130 |
+
torch.nn.init.normal_(self.attn.fc.weight, std=std)
|
| 131 |
+
torch.nn.init.normal_(self.mlp.mlp[-1].weight, std=std)
|
| 132 |
+
|
| 133 |
+
def fw_attn(self, x, kv=None):
|
| 134 |
+
B, V, L, D = x.shape
|
| 135 |
+
x = x.view(B, V*L, D)
|
| 136 |
+
skip, kv = self.attn(self.norm1(x), kv=kv)
|
| 137 |
+
x = x + skip
|
| 138 |
+
x = x.view(B, V, L, D)
|
| 139 |
+
return x, kv
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
class BlockICM(BlockBase):
|
| 143 |
+
def __init__(self, *args, **kwargs):
|
| 144 |
+
super().__init__(*args, **kwargs)
|
| 145 |
+
self.norm1 = torch.nn.LayerNorm(self.dim, bias=self.bias)
|
| 146 |
+
self.attn_intra = IntraViewAttention(
|
| 147 |
+
dim=self.dim,
|
| 148 |
+
head_dim=self.head_dim,
|
| 149 |
+
bias=self.bias,
|
| 150 |
+
use_qk_norm=self.use_qk_norm,
|
| 151 |
+
use_gating=self.use_gating,
|
| 152 |
+
use_sandwich=self.use_sandwich)
|
| 153 |
+
|
| 154 |
+
self.norm2 = torch.nn.LayerNorm(self.dim, bias=self.bias)
|
| 155 |
+
self.attn_cross = CrossViewAttention(
|
| 156 |
+
dim=self.dim,
|
| 157 |
+
head_dim=self.head_dim,
|
| 158 |
+
bias=self.bias,
|
| 159 |
+
use_qk_norm=self.use_qk_norm,
|
| 160 |
+
use_gating=self.use_gating,
|
| 161 |
+
use_sandwich=self.use_sandwich)
|
| 162 |
+
|
| 163 |
+
@property
|
| 164 |
+
def num_residual(self):
|
| 165 |
+
return 3
|
| 166 |
+
|
| 167 |
+
def init_residual(self, std):
|
| 168 |
+
torch.nn.init.normal_(self.attn_intra.fc.weight, std=std)
|
| 169 |
+
torch.nn.init.normal_(self.attn_cross.fc.weight, std=std)
|
| 170 |
+
torch.nn.init.normal_(self.mlp.mlp[-1].weight, std=std)
|
| 171 |
+
|
| 172 |
+
def fw_attn(self, x, kv=None):
|
| 173 |
+
B, V, L, D = x.shape
|
| 174 |
+
x = x.view(B*V, L, D)
|
| 175 |
+
x = x + self.attn_intra(self.norm1(x))
|
| 176 |
+
x = x.view(B, V*L, D)
|
| 177 |
+
skip, kv = self.attn_cross(self.norm2(x), kv=kv)
|
| 178 |
+
x = x + skip
|
| 179 |
+
x = x.view(B, V, L, D)
|
| 180 |
+
return x, kv
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
class BlockSequence(torch.nn.Module):
|
| 184 |
+
def __init__(self, *args, **kwargs):
|
| 185 |
+
super().__init__()
|
| 186 |
+
self.blklst = None
|
| 187 |
+
|
| 188 |
+
@property
|
| 189 |
+
def num_residual(self):
|
| 190 |
+
return sum(blk.num_residual for blk in self.blklst)
|
| 191 |
+
|
| 192 |
+
def init_residual(self, std):
|
| 193 |
+
for blk in self.blklst:
|
| 194 |
+
blk.init_residual(std)
|
| 195 |
+
|
| 196 |
+
def apply_compile(self):
|
| 197 |
+
for blk in self.blklst:
|
| 198 |
+
blk.apply_compile()
|
| 199 |
+
|
| 200 |
+
def apply_checkpoint(self):
|
| 201 |
+
for blk in self.blklst:
|
| 202 |
+
blk.apply_checkpoint()
|
| 203 |
+
|
| 204 |
+
def forward(self, x, kv=None):
|
| 205 |
+
for blk in self.blklst:
|
| 206 |
+
x, kv = blk(x, kv)
|
| 207 |
+
return x, kv
|
| 208 |
+
|
| 209 |
+
class BlockIMCM(BlockSequence):
|
| 210 |
+
def __init__(self, *args, **kwargs):
|
| 211 |
+
super().__init__()
|
| 212 |
+
self.blklst = torch.nn.ModuleList([
|
| 213 |
+
BlockIM(*args, **kwargs),
|
| 214 |
+
BlockCM(*args, **kwargs)])
|
| 215 |
+
|
| 216 |
+
class BlockCMIM(BlockSequence):
|
| 217 |
+
def __init__(self, *args, **kwargs):
|
| 218 |
+
super().__init__()
|
| 219 |
+
self.blklst = torch.nn.ModuleList([
|
| 220 |
+
BlockCM(*args, **kwargs),
|
| 221 |
+
BlockIM(*args, **kwargs)])
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
class BlockTBBase(BlockBase):
|
| 225 |
+
"""Shared base for TTB (TTB) blocks.
|
| 226 |
+
|
| 227 |
+
Owns the TTB parameter signature, builds the SwiGLU fast-weight module from
|
| 228 |
+
the block-level ttb_*/ttb_* kwargs, and implements the (identical) rendering
|
| 229 |
+
branch. Subclasses implement only the reconstruction `forward`.
|
| 230 |
+
"""
|
| 231 |
+
def __init__(self, *args, ttb_feedback=False,
|
| 232 |
+
ttb_use_gating=False, ttb_head_dim=0, ttb_inter_multi=2,
|
| 233 |
+
ttb_base_lr=0.01, ttb_muon_steps=5, ttb_num_steps=1,
|
| 234 |
+
ttb_target_residual=True, ttb_pre_cm_target=False,
|
| 235 |
+
ttb_use_mse_loss=False,
|
| 236 |
+
ttb_share_qk_proj=False,
|
| 237 |
+
ttb_per_step_norm=False,
|
| 238 |
+
inner_grad_checkpoint=False,
|
| 239 |
+
**kwargs):
|
| 240 |
+
super().__init__(*args, **kwargs)
|
| 241 |
+
self.ttb_feedback = ttb_feedback
|
| 242 |
+
self.ttb_num_steps = ttb_num_steps
|
| 243 |
+
self.ttb_target_residual = ttb_target_residual
|
| 244 |
+
self.ttb_pre_cm_target = ttb_pre_cm_target
|
| 245 |
+
|
| 246 |
+
self.norm_ttb = torch.nn.LayerNorm(self.dim, bias=self.bias)
|
| 247 |
+
self.ttb = FastWeightGluMLPMultiheadTTB(
|
| 248 |
+
dim=self.dim, head_dim=ttb_head_dim or self.dim, inter_multi=ttb_inter_multi,
|
| 249 |
+
bias=self.bias, base_lr=ttb_base_lr,
|
| 250 |
+
muon_update_steps=ttb_muon_steps,
|
| 251 |
+
use_gating=ttb_use_gating,
|
| 252 |
+
use_mse_loss=ttb_use_mse_loss,
|
| 253 |
+
share_qk_proj=ttb_share_qk_proj,
|
| 254 |
+
per_step_norm=ttb_per_step_norm,
|
| 255 |
+
inner_grad_checkpoint=inner_grad_checkpoint)
|
| 256 |
+
|
| 257 |
+
def apply_compile(self):
|
| 258 |
+
self.forward = torch.compile(self.forward, dynamic=True)
|
| 259 |
+
|
| 260 |
+
def _ttb_recon(self, x_cam, x_img_target, n_tokens):
|
| 261 |
+
"""Reconstruction-stage inner-loop update + apply.
|
| 262 |
+
Bakes the mapping x_cam -> (target) into the fast weights and applies it
|
| 263 |
+
to x_cam. Returns (ttb_out, ttb_states)."""
|
| 264 |
+
x_cam_normed = self.norm_ttb(x_cam)
|
| 265 |
+
if self.ttb_target_residual:
|
| 266 |
+
ttb_v = self.norm_ttb(x_img_target) - x_cam_normed
|
| 267 |
+
else:
|
| 268 |
+
ttb_v = self.norm_ttb(x_img_target)
|
| 269 |
+
op_order = _build_ttb_op_order(
|
| 270 |
+
n_tokens, update=True, apply=True, num_steps=self.ttb_num_steps)
|
| 271 |
+
return self.ttb(
|
| 272 |
+
x_query=x_cam_normed, info={"ttb_op_order": op_order},
|
| 273 |
+
x_key=x_cam_normed, x_value=ttb_v)
|
| 274 |
+
|
| 275 |
+
def _render(self, x, kv):
|
| 276 |
+
"""Rendering-stage TTB apply only + MLP (shared by all TTB blocks)."""
|
| 277 |
+
B, V, L, D = x.shape
|
| 278 |
+
x = x.view(B, V*L, D)
|
| 279 |
+
info = {"ttb_op_order": [
|
| 280 |
+
TTBOperator(start=0, end=V*L, update=False, apply=True)
|
| 281 |
+
]}
|
| 282 |
+
info.update(kv)
|
| 283 |
+
ttb_out, _ = self.ttb(x_query=self.norm_ttb(x), info=info)
|
| 284 |
+
x = x + ttb_out
|
| 285 |
+
x = x + self.mlp(self.norm_mlp(x))
|
| 286 |
+
return x.view(B, V, L, D), kv
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
class BlockCMTB(BlockTBBase):
|
| 290 |
+
def __init__(self, *args, **kwargs):
|
| 291 |
+
super().__init__(*args, **kwargs)
|
| 292 |
+
# Cross-view attention (produces the bake target for the TTB module)
|
| 293 |
+
self.norm1 = torch.nn.LayerNorm(self.dim, bias=self.bias)
|
| 294 |
+
self.attn = CrossViewAttention(
|
| 295 |
+
dim=self.dim,
|
| 296 |
+
head_dim=self.head_dim,
|
| 297 |
+
bias=self.bias,
|
| 298 |
+
use_qk_norm=self.use_qk_norm,
|
| 299 |
+
use_gating=self.use_gating,
|
| 300 |
+
use_sandwich=self.use_sandwich)
|
| 301 |
+
|
| 302 |
+
@property
|
| 303 |
+
def num_residual(self):
|
| 304 |
+
return 2 # x_img: cross_attn + mlp; x_cam: ttb + mlp
|
| 305 |
+
|
| 306 |
+
def init_residual(self, std):
|
| 307 |
+
torch.nn.init.normal_(self.attn.fc.weight, std=std)
|
| 308 |
+
torch.nn.init.normal_(self.mlp.mlp[-1].weight, std=std)
|
| 309 |
+
torch.nn.init.normal_(self.ttb.c_proj.weight, std=std)
|
| 310 |
+
|
| 311 |
+
def forward(self, x, kv=None):
|
| 312 |
+
if kv is not None:
|
| 313 |
+
return self._render(x, kv)
|
| 314 |
+
|
| 315 |
+
# ===== Reconstruction: x = [x_img; x_cam] with 2V views =====
|
| 316 |
+
B, V, L, D = x.shape
|
| 317 |
+
assert V % 2 == 0
|
| 318 |
+
V2 = V // 2
|
| 319 |
+
x_img = x[:, :V2].reshape(B, V2*L, D)
|
| 320 |
+
x_cam = x[:, V2:].reshape(B, V2*L, D)
|
| 321 |
+
|
| 322 |
+
# Cross-view attention on x_img
|
| 323 |
+
attn_delta, _ = self.attn(self.norm1(x_img), kv=None)
|
| 324 |
+
x_img = x_img + attn_delta
|
| 325 |
+
|
| 326 |
+
# Optionally use post-attn pre-MLP x_img as the TTB target
|
| 327 |
+
x_img_target = x_img if self.ttb_pre_cm_target else None
|
| 328 |
+
|
| 329 |
+
# MLP on x_img
|
| 330 |
+
x_img = x_img + self.mlp(self.norm_mlp(x_img))
|
| 331 |
+
if x_img_target is None:
|
| 332 |
+
x_img_target = x_img
|
| 333 |
+
|
| 334 |
+
# TTB update + apply on x_cam
|
| 335 |
+
ttb_out, ttb_states = self._ttb_recon(x_cam, x_img_target, V2*L)
|
| 336 |
+
x_cam = x_cam + ttb_out
|
| 337 |
+
if self.ttb_feedback:
|
| 338 |
+
x_img = x_img + ttb_out
|
| 339 |
+
x_cam = x_cam + self.mlp(self.norm_mlp(x_cam))
|
| 340 |
+
|
| 341 |
+
x = torch.cat([x_img.view(B, V2, L, D),
|
| 342 |
+
x_cam.view(B, V2, L, D)], dim=1)
|
| 343 |
+
return x, ttb_states
|
| 344 |
+
|
| 345 |
+
|
| 346 |
+
class BlockIMCMTB(BlockSequence):
|
| 347 |
+
def __init__(self, *args, **kwargs):
|
| 348 |
+
super().__init__()
|
| 349 |
+
# BlockIM ignores the extra TTB kwargs; BlockCMTB consumes them.
|
| 350 |
+
self.blklst = torch.nn.ModuleList([
|
| 351 |
+
BlockIM(*args, **kwargs),
|
| 352 |
+
BlockCMTB(*args, **kwargs)])
|
| 353 |
+
|
| 354 |
+
def apply_checkpoint(self):
|
| 355 |
+
self.forward = checkpoint_wrapper(self)
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
class BlockTB(BlockTBBase):
|
| 359 |
+
@property
|
| 360 |
+
def num_residual(self):
|
| 361 |
+
return 2 # ttb + mlp
|
| 362 |
+
|
| 363 |
+
def init_residual(self, std):
|
| 364 |
+
torch.nn.init.normal_(self.ttb.c_proj.weight, std=std)
|
| 365 |
+
torch.nn.init.normal_(self.mlp.mlp[-1].weight, std=std)
|
| 366 |
+
|
| 367 |
+
def forward(self, x, kv=None):
|
| 368 |
+
if kv is not None:
|
| 369 |
+
return self._render(x, kv)
|
| 370 |
+
|
| 371 |
+
# ===== Reconstruction: x = [x_img; x_cam] with 2V views =====
|
| 372 |
+
B, V, L, D = x.shape
|
| 373 |
+
assert V % 2 == 0
|
| 374 |
+
V2 = V // 2
|
| 375 |
+
x_img = x[:, :V2].reshape(B, V2*L, D)
|
| 376 |
+
x_cam = x[:, V2:].reshape(B, V2*L, D)
|
| 377 |
+
|
| 378 |
+
# TTB update + apply (no cross-view attention; target is x_img directly)
|
| 379 |
+
ttb_out, ttb_states = self._ttb_recon(x_cam, x_img, V2*L)
|
| 380 |
+
x_cam = x_cam + ttb_out
|
| 381 |
+
if self.ttb_feedback:
|
| 382 |
+
x_img = x_img + ttb_out
|
| 383 |
+
|
| 384 |
+
# MLP
|
| 385 |
+
x_cam = x_cam + self.mlp(self.norm_mlp(x_cam))
|
| 386 |
+
x_img = x_img + self.mlp(self.norm_mlp(x_img))
|
| 387 |
+
|
| 388 |
+
# Re-concatenate [x_img; x_cam]
|
| 389 |
+
x = torch.cat([x_img.view(B, V2, L, D),
|
| 390 |
+
x_cam.view(B, V2, L, D)], dim=1)
|
| 391 |
+
return x, ttb_states
|
| 392 |
+
|
| 393 |
+
|
| 394 |
+
class BlockIMTB(BlockSequence):
|
| 395 |
+
def __init__(self, *args, **kwargs):
|
| 396 |
+
super().__init__()
|
| 397 |
+
# BlockIM ignores the extra TTB kwargs; BlockTB consumes them.
|
| 398 |
+
self.blklst = torch.nn.ModuleList([
|
| 399 |
+
BlockIM(*args, **kwargs),
|
| 400 |
+
BlockTB(*args, **kwargs)])
|
| 401 |
+
|
| 402 |
+
def apply_checkpoint(self):
|
| 403 |
+
self.forward = checkpoint_wrapper(self)
|
dvsm/models/modules.py
ADDED
|
@@ -0,0 +1,273 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn.functional as F
|
| 3 |
+
import torch.utils.checkpoint
|
| 4 |
+
|
| 5 |
+
# flash-attn is a hard dependency in the training / NVIDIA-container environment,
|
| 6 |
+
# but it is unavailable (or ABI/arch-incompatible) on some inference targets such
|
| 7 |
+
# as HF Spaces ZeroGPU (torch 2.8+, Blackwell sm_120). Soft-import it and fall
|
| 8 |
+
# back to torch SDPA, which is numerically equivalent for our usage (non-causal,
|
| 9 |
+
# no dropout, default scale = 1/sqrt(head_dim)).
|
| 10 |
+
import os
|
| 11 |
+
|
| 12 |
+
try:
|
| 13 |
+
import flash_attn
|
| 14 |
+
_HAS_FLASH = True
|
| 15 |
+
except Exception: # ImportError or runtime/arch incompatibility on import
|
| 16 |
+
flash_attn = None
|
| 17 |
+
_HAS_FLASH = False
|
| 18 |
+
|
| 19 |
+
# Allow forcing the SDPA path even when flash_attn is importable, so the
|
| 20 |
+
# ZeroGPU code path can be validated on a flash-attn-equipped GPU node.
|
| 21 |
+
if os.environ.get("DVSM_DISABLE_FLASH") == "1":
|
| 22 |
+
_HAS_FLASH = False
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _attn_func(q, k, v):
|
| 26 |
+
"""Drop-in replacement for ``flash_attn.flash_attn_func``.
|
| 27 |
+
|
| 28 |
+
flash_attn layout: q/k/v are ``[B, L, num_head, head_dim]``, non-causal,
|
| 29 |
+
softmax_scale = 1/sqrt(head_dim). torch SDPA expects ``[B, num_head, L, head_dim]``
|
| 30 |
+
and uses the same default scale + non-causal semantics, so we just transpose
|
| 31 |
+
in and out. Output dtype matches the inputs (bf16/fp16), same as flash_attn.
|
| 32 |
+
"""
|
| 33 |
+
if _HAS_FLASH:
|
| 34 |
+
return flash_attn.flash_attn_func(q, k, v)
|
| 35 |
+
qh = q.transpose(1, 2) # [B, H, Lq, D]
|
| 36 |
+
kh = k.transpose(1, 2) # [B, H, Lk, D]
|
| 37 |
+
vh = v.transpose(1, 2) # [B, H, Lk, D]
|
| 38 |
+
out = F.scaled_dot_product_attention(qh, kh, vh) # default scale, non-causal
|
| 39 |
+
return out.transpose(1, 2).contiguous() # back to [B, Lq, H, D]
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def checkpoint_wrapper(module):
|
| 43 |
+
# Wrap model forward function to automatically use activation checkpointing
|
| 44 |
+
ori_forward = module.forward
|
| 45 |
+
def new_forward(*args, **kwargs):
|
| 46 |
+
if module.training:
|
| 47 |
+
output = torch.utils.checkpoint.checkpoint(
|
| 48 |
+
ori_forward, *args, **kwargs, use_reentrant=False)
|
| 49 |
+
else:
|
| 50 |
+
output = ori_forward(*args, **kwargs)
|
| 51 |
+
return output
|
| 52 |
+
return new_forward
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class Contiguous(torch.nn.Module):
|
| 56 |
+
def forward(self, x):
|
| 57 |
+
return x.contiguous()
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
class MLP(torch.nn.Module):
|
| 61 |
+
def __init__(self, dim, expand_ratio=4, bias=False, activation=torch.nn.GELU):
|
| 62 |
+
"""
|
| 63 |
+
Args:
|
| 64 |
+
dim: Input dimension
|
| 65 |
+
expand_ratio: Multiplier for hidden dimension
|
| 66 |
+
bias: Whether to use bias in linear layers
|
| 67 |
+
activation: Activation function
|
| 68 |
+
"""
|
| 69 |
+
super().__init__()
|
| 70 |
+
hidden_dim = round(dim * expand_ratio)
|
| 71 |
+
self.mlp = torch.nn.Sequential(
|
| 72 |
+
torch.nn.Linear(dim, hidden_dim, bias=bias),
|
| 73 |
+
activation(),
|
| 74 |
+
torch.nn.Linear(hidden_dim, dim, bias=bias))
|
| 75 |
+
|
| 76 |
+
def forward(self, x):
|
| 77 |
+
return self.mlp(x)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
class SwiGluMLP(torch.nn.Module):
|
| 81 |
+
def __init__(self, dim, expand_ratio=4, bias=False):
|
| 82 |
+
"""
|
| 83 |
+
Args:
|
| 84 |
+
dim: Input dimension
|
| 85 |
+
expand_ratio: Multiplier for hidden dimension
|
| 86 |
+
bias: Whether to use bias in linear layers
|
| 87 |
+
activation: Activation function
|
| 88 |
+
"""
|
| 89 |
+
super().__init__()
|
| 90 |
+
hidden_dim = round(dim * expand_ratio)
|
| 91 |
+
self.mlp = torch.nn.Sequential(
|
| 92 |
+
torch.nn.Linear(dim, hidden_dim, bias=bias),
|
| 93 |
+
torch.nn.Linear(dim, hidden_dim, bias=bias),
|
| 94 |
+
torch.nn.Linear(hidden_dim, dim, bias=bias))
|
| 95 |
+
|
| 96 |
+
def forward(self, x):
|
| 97 |
+
gate = torch.nn.functional.silu(self.mlp[0](x), inplace=True)
|
| 98 |
+
return self.mlp[2](gate * self.mlp[1](x))
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
class IntraViewAttention(torch.nn.Module):
|
| 102 |
+
def __init__(self,
|
| 103 |
+
dim,
|
| 104 |
+
head_dim,
|
| 105 |
+
bias=False,
|
| 106 |
+
use_qk_norm=True,
|
| 107 |
+
use_gating=False,
|
| 108 |
+
use_sandwich=False,
|
| 109 |
+
out_dim=None,
|
| 110 |
+
):
|
| 111 |
+
"""
|
| 112 |
+
dim: Input dimension
|
| 113 |
+
head_dim: Dimension of each attention head
|
| 114 |
+
bias: Whether to use bias in QKV and output projection
|
| 115 |
+
use_qk_norm: Whether to use Q-K normalization
|
| 116 |
+
use_gating: Whether to apply gating before output ffn
|
| 117 |
+
"""
|
| 118 |
+
super().__init__()
|
| 119 |
+
assert dim % head_dim == 0, f"dim={dim} should be divisible by head_dim={head_dim}"
|
| 120 |
+
num_head = dim // head_dim
|
| 121 |
+
|
| 122 |
+
self.dim = dim
|
| 123 |
+
self.out_dim = dim if out_dim is None else out_dim
|
| 124 |
+
self.head_dim = head_dim
|
| 125 |
+
self.num_head = num_head
|
| 126 |
+
self.use_qk_norm = use_qk_norm
|
| 127 |
+
self.use_gating = use_gating
|
| 128 |
+
self.use_sandwich = use_sandwich
|
| 129 |
+
self.attn_dtype = torch.bfloat16 if torch.cuda.get_device_capability()[0] >= 8 else torch.float16
|
| 130 |
+
|
| 131 |
+
self.to_qkv = torch.nn.Linear(dim, 3 * dim, bias=bias)
|
| 132 |
+
self.fc = torch.nn.Linear(dim, self.out_dim, bias=bias)
|
| 133 |
+
|
| 134 |
+
# Optional Q-K normalization
|
| 135 |
+
if self.use_qk_norm:
|
| 136 |
+
self.q_norm = torch.nn.RMSNorm(head_dim, eps=1e-5)
|
| 137 |
+
self.k_norm = torch.nn.RMSNorm(head_dim, eps=1e-5)
|
| 138 |
+
else:
|
| 139 |
+
self.q_norm = torch.nn.Identity()
|
| 140 |
+
self.k_norm = torch.nn.Identity()
|
| 141 |
+
|
| 142 |
+
# Optional gating
|
| 143 |
+
if self.use_gating:
|
| 144 |
+
self.gate = torch.nn.Sequential(
|
| 145 |
+
torch.nn.Linear(dim, dim, bias=False),
|
| 146 |
+
torch.nn.Sigmoid())
|
| 147 |
+
# Optional normalization
|
| 148 |
+
if self.use_sandwich:
|
| 149 |
+
self.sandwich = torch.nn.LayerNorm(dim, bias=False)
|
| 150 |
+
|
| 151 |
+
def forward(self, x):
|
| 152 |
+
"""
|
| 153 |
+
x: [B, L, D]
|
| 154 |
+
"""
|
| 155 |
+
# Predict qkv packed as (batch, seq_len, 3, num_head, head_dim)
|
| 156 |
+
qkv = self.to_qkv(x).unflatten(2, (3, self.num_head, self.head_dim))
|
| 157 |
+
|
| 158 |
+
# Unpack (q, k, v) each as (batch, seq_len, num_head, head_dim)
|
| 159 |
+
q, k, v = qkv.moveaxis(2,0).contiguous()
|
| 160 |
+
|
| 161 |
+
# Apply qk normalization if enabled
|
| 162 |
+
q = self.q_norm(q)
|
| 163 |
+
k = self.k_norm(k)
|
| 164 |
+
|
| 165 |
+
# Attention
|
| 166 |
+
with torch.autocast(enabled=False, device_type="cuda"):
|
| 167 |
+
q = q.to(self.attn_dtype)
|
| 168 |
+
k = k.to(self.attn_dtype)
|
| 169 |
+
v = v.to(self.attn_dtype)
|
| 170 |
+
|
| 171 |
+
out = _attn_func(q, k, v)
|
| 172 |
+
|
| 173 |
+
# Output layer to yield (batch, seq_len, dim)
|
| 174 |
+
out = out.flatten(2,3)
|
| 175 |
+
if self.use_gating:
|
| 176 |
+
out = out * self.gate(x)
|
| 177 |
+
out = self.fc(out)
|
| 178 |
+
if self.use_sandwich:
|
| 179 |
+
out = self.sandwich(out)
|
| 180 |
+
|
| 181 |
+
return out
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
class CrossViewAttention(torch.nn.Module):
|
| 185 |
+
def __init__(self,
|
| 186 |
+
dim,
|
| 187 |
+
head_dim,
|
| 188 |
+
bias=False,
|
| 189 |
+
use_qk_norm=True,
|
| 190 |
+
use_gating=False,
|
| 191 |
+
use_sandwich=False,
|
| 192 |
+
out_dim=None,
|
| 193 |
+
):
|
| 194 |
+
"""
|
| 195 |
+
Args:
|
| 196 |
+
dim: Input dimension
|
| 197 |
+
head_dim: Dimension of each attention head
|
| 198 |
+
bias: Whether to use bias in QKV and output projection
|
| 199 |
+
use_qk_norm: Whether to use Q-K normalization
|
| 200 |
+
use_gating: Whether to apply gating before output ffn
|
| 201 |
+
"""
|
| 202 |
+
super().__init__()
|
| 203 |
+
assert dim % head_dim == 0, f"dim={dim} should be divisible by head_dim={head_dim}"
|
| 204 |
+
num_head = dim // head_dim
|
| 205 |
+
|
| 206 |
+
self.dim = dim
|
| 207 |
+
self.out_dim = dim if out_dim is None else out_dim
|
| 208 |
+
self.head_dim = head_dim
|
| 209 |
+
self.num_head = num_head
|
| 210 |
+
self.use_qk_norm = use_qk_norm
|
| 211 |
+
self.use_gating = use_gating
|
| 212 |
+
self.use_sandwich = use_sandwich
|
| 213 |
+
self.attn_dtype = torch.bfloat16 if torch.cuda.get_device_capability()[0] >= 8 else torch.float16
|
| 214 |
+
|
| 215 |
+
self.to_q = torch.nn.Linear(dim, dim, bias=bias)
|
| 216 |
+
self.to_kv = torch.nn.Linear(dim, 2 * dim, bias=bias)
|
| 217 |
+
self.fc = torch.nn.Linear(dim, self.out_dim, bias=bias)
|
| 218 |
+
|
| 219 |
+
# Optional Q-K normalization
|
| 220 |
+
if self.use_qk_norm:
|
| 221 |
+
self.q_norm = torch.nn.RMSNorm(head_dim, eps=1e-5)
|
| 222 |
+
self.k_norm = torch.nn.RMSNorm(head_dim, eps=1e-5)
|
| 223 |
+
else:
|
| 224 |
+
self.q_norm = torch.nn.Identity()
|
| 225 |
+
self.k_norm = torch.nn.Identity()
|
| 226 |
+
|
| 227 |
+
# Optional gating
|
| 228 |
+
if self.use_gating:
|
| 229 |
+
self.gate = torch.nn.Sequential(
|
| 230 |
+
torch.nn.Linear(dim, dim, bias=False),
|
| 231 |
+
torch.nn.Sigmoid())
|
| 232 |
+
# Optional normalization
|
| 233 |
+
if self.use_sandwich:
|
| 234 |
+
self.sandwich = torch.nn.LayerNorm(dim, bias=False)
|
| 235 |
+
|
| 236 |
+
def forward(self, x, kv=None):
|
| 237 |
+
"""
|
| 238 |
+
x: [B, L, D]
|
| 239 |
+
kv: ([B, M, D], [B, M, D]) If provided, x query on kv instead of doing self attention
|
| 240 |
+
"""
|
| 241 |
+
q = self.to_q(x)
|
| 242 |
+
|
| 243 |
+
# To shape of (batch, seq_len, num_head, head_dim)
|
| 244 |
+
q = q.unflatten(2, (self.num_head, self.head_dim))
|
| 245 |
+
q = self.q_norm(q)
|
| 246 |
+
with torch.autocast(enabled=False, device_type="cuda"):
|
| 247 |
+
q = q.to(self.attn_dtype)
|
| 248 |
+
|
| 249 |
+
# Prepare (k, v)
|
| 250 |
+
if kv is None:
|
| 251 |
+
kv = self.to_kv(x)
|
| 252 |
+
kv = kv.unflatten(2, (2, self.num_head, self.head_dim))
|
| 253 |
+
k, v = kv.moveaxis(2, 0).contiguous()
|
| 254 |
+
k = self.k_norm(k)
|
| 255 |
+
with torch.autocast(enabled=False, device_type="cuda"):
|
| 256 |
+
k = k.to(self.attn_dtype)
|
| 257 |
+
v = v.to(self.attn_dtype)
|
| 258 |
+
else:
|
| 259 |
+
k, v = kv
|
| 260 |
+
assert len(x) == len(k) and len(x) == len(v)
|
| 261 |
+
|
| 262 |
+
# Attention
|
| 263 |
+
out = _attn_func(q, k, v)
|
| 264 |
+
|
| 265 |
+
# Output layer into (batch, seq_len, dim)
|
| 266 |
+
out = out.flatten(2, 3)
|
| 267 |
+
if self.use_gating:
|
| 268 |
+
out = out * self.gate(x)
|
| 269 |
+
out = self.fc(out)
|
| 270 |
+
if self.use_sandwich:
|
| 271 |
+
out = self.sandwich(out)
|
| 272 |
+
|
| 273 |
+
return out, (k, v)
|
dvsm/models/plucker.py
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
class PluckerEncoding(torch.nn.Module):
|
| 5 |
+
def __init__(self, method):
|
| 6 |
+
super().__init__()
|
| 7 |
+
if method == 'plucker':
|
| 8 |
+
self.pose_enc = compute_plucker
|
| 9 |
+
self.dim = 6
|
| 10 |
+
elif method == 'plucker_aug':
|
| 11 |
+
self.pose_enc = compute_plucker_aug
|
| 12 |
+
self.dim = 9
|
| 13 |
+
elif method == 'plucker_custom':
|
| 14 |
+
self.pose_enc = compute_plucker_custom
|
| 15 |
+
self.dim = 6
|
| 16 |
+
elif method == 'concat':
|
| 17 |
+
self.pose_enc = simple_concat
|
| 18 |
+
self.dim = 6
|
| 19 |
+
else:
|
| 20 |
+
raise NotImplementedError
|
| 21 |
+
|
| 22 |
+
def forward(self, intr, c2w_r, c2w_t, h, w):
|
| 23 |
+
'''
|
| 24 |
+
Encode camera parameters.
|
| 25 |
+
Args:
|
| 26 |
+
intr (torch.tensor): [b, v, 4]
|
| 27 |
+
c2w_r (torch.tensor): [b, v, 3, 3]
|
| 28 |
+
c2w_t (torch.tensor): [b, v, 3]
|
| 29 |
+
(h, w): size of the ray map
|
| 30 |
+
'''
|
| 31 |
+
# Sanity check
|
| 32 |
+
assert len(intr.shape) == 3 and intr.shape[2] == 4
|
| 33 |
+
assert len(c2w_r.shape) == 4 and c2w_r.shape[-2:] == (3, 3)
|
| 34 |
+
assert len(c2w_t.shape) == 3 and c2w_t.shape[2] == 3
|
| 35 |
+
assert intr.shape[:2] == c2w_r.shape[:2]
|
| 36 |
+
assert intr.shape[:2] == c2w_t.shape[:2]
|
| 37 |
+
|
| 38 |
+
# Encode camera parameters
|
| 39 |
+
ray_o, ray_d = compute_rays(intr, c2w_r, c2w_t, h, w)
|
| 40 |
+
plucker = self.pose_enc(ray_o, ray_d)
|
| 41 |
+
|
| 42 |
+
return plucker
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def compute_rays(intr, c2w_r, c2w_t, h, w):
|
| 47 |
+
'''
|
| 48 |
+
Args:
|
| 49 |
+
intr (torch.tensor): [b, v, 4]
|
| 50 |
+
c2w_r (torch.tensor): [b, v, 3, 3]
|
| 51 |
+
c2w_t (torch.tensor): [b, v, 3]
|
| 52 |
+
Returns:
|
| 53 |
+
ray_o: (b, v, 3, h, w)
|
| 54 |
+
ray_d: (b, v, 3, h, w)
|
| 55 |
+
'''
|
| 56 |
+
b, v = intr.shape[:2]
|
| 57 |
+
|
| 58 |
+
# Unpack intrinsics into [b*v, 1] and denormalized
|
| 59 |
+
fx, fy, cx, cy = intr.flatten(0, 1).T.unsqueeze(-1).contiguous()
|
| 60 |
+
fx = fx * w
|
| 61 |
+
fy = fy * h
|
| 62 |
+
cx = cx * w
|
| 63 |
+
cy = cy * h
|
| 64 |
+
|
| 65 |
+
# Efficient meshgrid equivalent using broadcasting
|
| 66 |
+
idx_x = torch.arange(w, device=intr.device) + 0.5
|
| 67 |
+
idx_y = torch.arange(h, device=intr.device) + 0.5
|
| 68 |
+
idx_x = idx_x[None, :].expand(h, -1) # [h, w]
|
| 69 |
+
idx_y = idx_y[:, None].expand(-1, w) # [h, w]
|
| 70 |
+
|
| 71 |
+
# Reshape for batched matrix multiplication
|
| 72 |
+
idx_x = idx_x.flatten().expand(b * v, -1) # [b*v, h*w]
|
| 73 |
+
idx_y = idx_y.flatten().expand(b * v, -1) # [b*v, h*w]
|
| 74 |
+
|
| 75 |
+
x = (idx_x - cx) / fx # [b*v, h*w]
|
| 76 |
+
y = (idx_y - cy) / fy # [b*v, h*w]
|
| 77 |
+
z = torch.ones_like(x) # [b*v, h*w]
|
| 78 |
+
|
| 79 |
+
# Transform to world coordinate
|
| 80 |
+
c2w_r = c2w_r.flatten(0, 1) # [b*v, 3, 3]
|
| 81 |
+
ray_d = torch.stack([x, y, z], dim=1) # [b*v, 3, h*w]
|
| 82 |
+
ray_d = torch.bmm(c2w_r, ray_d) # [b*v, 3, h*w]
|
| 83 |
+
ray_d = torch.nn.functional.normalize(ray_d, dim=1) # [b*v, 3, h*w]
|
| 84 |
+
|
| 85 |
+
ray_d = ray_d.reshape(b,v,3,h,w) # [b, v, 3, h, w]
|
| 86 |
+
ray_o = c2w_t.view(b,v,3,1,1).expand(b,v,3,h,w) # [b, v, 3, h, w]
|
| 87 |
+
|
| 88 |
+
return ray_o, ray_d
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def simple_concat(ray_o, ray_d):
|
| 92 |
+
return torch.cat([ray_o, ray_d], dim=2)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def compute_plucker(ray_o, ray_d):
|
| 96 |
+
o_cross_d = torch.cross(ray_o, ray_d, dim=2)
|
| 97 |
+
plucker = torch.cat([o_cross_d, ray_d], dim=2)
|
| 98 |
+
return plucker
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def compute_plucker_aug(ray_o, ray_d):
|
| 102 |
+
o_dot_d = torch.sum(-ray_o * ray_d, dim=2, keepdim=True)
|
| 103 |
+
nearest_pts = ray_o + o_dot_d * ray_d
|
| 104 |
+
o_cross_d = torch.cross(ray_o, ray_d, dim=2)
|
| 105 |
+
plucker = torch.cat([o_cross_d, ray_d, nearest_pts], dim=2)
|
| 106 |
+
return plucker
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def compute_plucker_custom(ray_o, ray_d):
|
| 110 |
+
o_dot_d = torch.sum(-ray_o * ray_d, dim=2, keepdim=True)
|
| 111 |
+
nearest_pts = ray_o + o_dot_d * ray_d
|
| 112 |
+
plucker = torch.cat([ray_d, nearest_pts], dim=2)
|
| 113 |
+
return plucker
|
dvsm/models/pose_preprocessor.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
class PosePreprocessor:
|
| 5 |
+
def __init__(self, c2w_r, c2w_t, subset_idx=None, scale_factor=0.75):
|
| 6 |
+
"""
|
| 7 |
+
Compute a series of [offset -> scale -> rotation] for normalizing poses.
|
| 8 |
+
c2w_r: [b, v, 3, 3]
|
| 9 |
+
c2w_t: [b, v, 3]
|
| 10 |
+
subset_idx: [v] or None. Use a subset of view to compute normalization.
|
| 11 |
+
scale_factor: Rescale factor for the normalized camera
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
assert len(c2w_r.shape) == 4 and c2w_r.shape[-2:] == (3, 3)
|
| 15 |
+
assert len(c2w_t.shape) == 3 and c2w_t.shape[2] == 3
|
| 16 |
+
assert c2w_r.shape[:2] == c2w_t.shape[:2]
|
| 17 |
+
|
| 18 |
+
# Compute SE(3) normalizing factor
|
| 19 |
+
if subset_idx is not None:
|
| 20 |
+
assert len(subset_idx.shape) == 1 and len(subset_idx) == c2w_r.shape[1]
|
| 21 |
+
avg_t = c2w_t[:, subset_idx].mean(1)
|
| 22 |
+
avg_r = compute_avg_rotation(c2w_r[:, subset_idx])
|
| 23 |
+
else:
|
| 24 |
+
avg_t = c2w_t.mean(1)
|
| 25 |
+
avg_r = compute_avg_rotation(c2w_r)
|
| 26 |
+
|
| 27 |
+
# Target transformation for normalization
|
| 28 |
+
self.r = avg_r.moveaxis(2,1).contiguous()
|
| 29 |
+
self.t = (-self.r @ avg_t.unsqueeze(-1)).squeeze(-1).unsqueeze(1).contiguous()
|
| 30 |
+
|
| 31 |
+
# Target scene rescaling
|
| 32 |
+
new_t = torch.einsum('bij, bvj -> bvi', self.r, c2w_t) + self.t
|
| 33 |
+
post_scale = new_t.abs().amax((1,2), keepdim=True)
|
| 34 |
+
self.s = scale_factor / post_scale
|
| 35 |
+
|
| 36 |
+
def __call__(self, c2w_r, c2w_t):
|
| 37 |
+
c2w_r = torch.einsum('bij, bvjk -> bvik', self.r, c2w_r)
|
| 38 |
+
c2w_t = torch.einsum('bij, bvj -> bvi', self.r, c2w_t) + self.t
|
| 39 |
+
c2w_t = c2w_t * self.s
|
| 40 |
+
return c2w_r, c2w_t
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def compute_avg_rotation(c2w_r):
|
| 44 |
+
# c2w_r: [b, v, 3, 3]
|
| 45 |
+
avg_down = torch.nn.functional.normalize(c2w_r[..., 1].mean(1), dim=-1)
|
| 46 |
+
avg_forward = c2w_r[..., 2].mean(1)
|
| 47 |
+
avg_right = torch.nn.functional.normalize(torch.cross(avg_down, avg_forward, dim=-1), dim=-1)
|
| 48 |
+
avg_forward = torch.nn.functional.normalize(torch.cross(avg_right, avg_down, dim=-1), dim=-1)
|
| 49 |
+
avg_r = torch.stack([avg_right, avg_down, avg_forward], dim=-1)
|
| 50 |
+
return avg_r
|
dvsm/models/ttb.py
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import collections
|
| 2 |
+
import math
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
from torch import nn
|
| 6 |
+
|
| 7 |
+
import torch.nn.functional as F
|
| 8 |
+
from einops import rearrange
|
| 9 |
+
|
| 10 |
+
TTBOperator = collections.namedtuple("TTBOperator", ["start", "end", "update", "apply"])
|
| 11 |
+
|
| 12 |
+
@torch.compile
|
| 13 |
+
def inv_softplus(x):
|
| 14 |
+
y = x + math.log(-math.expm1(-x))
|
| 15 |
+
return y
|
| 16 |
+
|
| 17 |
+
@torch.compile
|
| 18 |
+
def silu_backprop(dy: torch.Tensor, x: torch.Tensor):
|
| 19 |
+
"""
|
| 20 |
+
Args:
|
| 21 |
+
dy: [b, d, l], gradient of the outer loss wrt the y
|
| 22 |
+
x: [b, d, l], input of the silu activation
|
| 23 |
+
outs:
|
| 24 |
+
dx: [b, d, l], gradient of the outer loss wrt the x
|
| 25 |
+
dx = dy * sigma * (1 + x * (1 - sigma))
|
| 26 |
+
"""
|
| 27 |
+
sigma = torch.sigmoid(x)
|
| 28 |
+
dx = dy * sigma * (1 + x * (1 - sigma))
|
| 29 |
+
return dx
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
@torch.compile
|
| 33 |
+
def zeropower_via_newtonschulz5(G, steps):
|
| 34 |
+
"""
|
| 35 |
+
modified from https://github.com/MoonshotAI/Moonlight/blob/master/examples/toy_train.py#L49
|
| 36 |
+
Major change: G is [b, d, d] rather than [d, d]
|
| 37 |
+
Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
|
| 38 |
+
quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
|
| 39 |
+
of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
|
| 40 |
+
zero even beyond the point where the iteration no longer converges all the way to one everywhere
|
| 41 |
+
on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
|
| 42 |
+
where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
|
| 43 |
+
performance at all relative to UV^T, where USV^T = G is the SVD.
|
| 44 |
+
Args:
|
| 45 |
+
G: [b, d, d]
|
| 46 |
+
steps: int
|
| 47 |
+
Returns:
|
| 48 |
+
X: [b, d, d]
|
| 49 |
+
"""
|
| 50 |
+
assert len(G.shape) == 3
|
| 51 |
+
a, b, c = (3.4445, -4.7750, 2.0315)
|
| 52 |
+
X = G.bfloat16()
|
| 53 |
+
if G.size(1) > G.size(2):
|
| 54 |
+
X = X.transpose(1, 2)
|
| 55 |
+
# Ensure spectral norm is at most 1
|
| 56 |
+
X = X / (X.norm(dim=(1, 2), keepdim=True) + 1e-7)
|
| 57 |
+
# Perform the NS iterations
|
| 58 |
+
for _ in range(steps):
|
| 59 |
+
A = X @ X.transpose(1, 2)
|
| 60 |
+
B = (
|
| 61 |
+
b * A + c * A @ A
|
| 62 |
+
) # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
|
| 63 |
+
X = a * X + B @ X
|
| 64 |
+
|
| 65 |
+
if G.size(1) > G.size(2):
|
| 66 |
+
X = X.transpose(1, 2)
|
| 67 |
+
return X
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def inner_loop(w0_now, w1_now, w2_now, ki, vi, lr0i, lr1i, lr2i,
|
| 71 |
+
use_mse,
|
| 72 |
+
muon_update_steps, n_tokens,
|
| 73 |
+
use_weight_norm, w0_norm, w1_norm, w2_norm):
|
| 74 |
+
|
| 75 |
+
gate_before_act = ki @ w0_now # [b, l, dh] = [b, l, d] @ [b, d, dh]
|
| 76 |
+
hidden_before_mul = ki @ w2_now # [b, l, dh] = [b, l, d] @ [b, d, dh]
|
| 77 |
+
hidden = F.silu(gate_before_act, inplace=False) * hidden_before_mul
|
| 78 |
+
|
| 79 |
+
if use_mse:
|
| 80 |
+
# MSE: gradient descent on ||f_W(k) - v||Β²
|
| 81 |
+
output_i = hidden @ w1_now # [b, l, d] β forward prediction
|
| 82 |
+
error = output_i - vi # [b, l, d] β prediction error
|
| 83 |
+
|
| 84 |
+
dhidden = error @ w1_now.transpose(-1, -2)
|
| 85 |
+
dhidden_before_mul = dhidden * F.silu(gate_before_act, inplace=False)
|
| 86 |
+
dgate = dhidden * hidden_before_mul
|
| 87 |
+
dgate_before_act = silu_backprop(dgate, gate_before_act)
|
| 88 |
+
|
| 89 |
+
w1_raw = -(hidden * lr1i).transpose(-1, -2) @ error
|
| 90 |
+
w0_raw = -(ki * lr0i).transpose(-1, -2) @ dgate_before_act
|
| 91 |
+
w2_raw = -(ki * lr2i).transpose(-1, -2) @ dhidden_before_mul
|
| 92 |
+
else:
|
| 93 |
+
# Hebbian: gradient ascent on <f_W(k), v>
|
| 94 |
+
dhidden = vi @ w1_now.transpose(-1, -2)
|
| 95 |
+
dhidden_before_mul = dhidden * F.silu(gate_before_act, inplace=False)
|
| 96 |
+
dgate = dhidden * hidden_before_mul
|
| 97 |
+
dgate_before_act = silu_backprop(dgate, gate_before_act)
|
| 98 |
+
|
| 99 |
+
w1_raw = (hidden * lr1i).transpose(-1, -2) @ vi
|
| 100 |
+
w0_raw = (ki * lr0i).transpose(-1, -2) @ dgate_before_act
|
| 101 |
+
w2_raw = (ki * lr2i).transpose(-1, -2) @ dhidden_before_mul
|
| 102 |
+
|
| 103 |
+
if muon_update_steps > 0:
|
| 104 |
+
w0_grad = zeropower_via_newtonschulz5(w0_raw, muon_update_steps)
|
| 105 |
+
w1_grad = zeropower_via_newtonschulz5(w1_raw, muon_update_steps)
|
| 106 |
+
w2_grad = zeropower_via_newtonschulz5(w2_raw, muon_update_steps)
|
| 107 |
+
else:
|
| 108 |
+
w0_grad = w0_raw / n_tokens
|
| 109 |
+
w1_grad = w1_raw / n_tokens
|
| 110 |
+
w2_grad = w2_raw / n_tokens
|
| 111 |
+
|
| 112 |
+
w0_now = w0_now + w0_grad
|
| 113 |
+
w1_now = w1_now + w1_grad
|
| 114 |
+
w2_now = w2_now + w2_grad
|
| 115 |
+
|
| 116 |
+
if use_weight_norm:
|
| 117 |
+
w0_now = w0_now / (w0_now.norm(dim=1, keepdim=True) + 1e-5) * w0_norm
|
| 118 |
+
w1_now = w1_now / (w1_now.norm(dim=1, keepdim=True) + 1e-5) * w1_norm
|
| 119 |
+
w2_now = w2_now / (w2_now.norm(dim=1, keepdim=True) + 1e-5) * w2_norm
|
| 120 |
+
|
| 121 |
+
return w0_now, w1_now, w2_now
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def _fast_weight_kernel_body(
|
| 125 |
+
w0: torch.Tensor,
|
| 126 |
+
w1: torch.Tensor,
|
| 127 |
+
w2: torch.Tensor,
|
| 128 |
+
q: torch.Tensor,
|
| 129 |
+
k: torch.Tensor,
|
| 130 |
+
v: torch.Tensor,
|
| 131 |
+
lr0: torch.Tensor,
|
| 132 |
+
lr1: torch.Tensor,
|
| 133 |
+
lr2: torch.Tensor,
|
| 134 |
+
ttb_ua_order: list,
|
| 135 |
+
muon_update_steps: int = 0,
|
| 136 |
+
use_mse: bool = False,
|
| 137 |
+
per_step_norm: bool = False,
|
| 138 |
+
inner_grad_checkpoint: bool = False,
|
| 139 |
+
):
|
| 140 |
+
"""
|
| 141 |
+
Forward: (silu(x @ w0) * (x @ w2)) @ w1
|
| 142 |
+
|
| 143 |
+
w0, w2: [b, d, dh]
|
| 144 |
+
w1: [b, dh, d]
|
| 145 |
+
q: [b, l, d]
|
| 146 |
+
k: [b, l, d]
|
| 147 |
+
v: [b, l, d]
|
| 148 |
+
lr0, lr1, lr2: [b, l, 1]
|
| 149 |
+
"""
|
| 150 |
+
w0_norm = w0.detach().norm(dim=1, keepdim=True)
|
| 151 |
+
w1_norm = w1.detach().norm(dim=1, keepdim=True)
|
| 152 |
+
w2_norm = w2.detach().norm(dim=1, keepdim=True)
|
| 153 |
+
|
| 154 |
+
# Sanity check: all entries use all tokens, update all-or-none, apply last only
|
| 155 |
+
num_update = 0
|
| 156 |
+
for idx, (start, end, update, apply) in enumerate(ttb_ua_order):
|
| 157 |
+
assert start == 0 and end == q.shape[1]
|
| 158 |
+
num_update += int(update)
|
| 159 |
+
if idx == len(ttb_ua_order) - 1:
|
| 160 |
+
assert apply
|
| 161 |
+
else:
|
| 162 |
+
assert not apply
|
| 163 |
+
assert num_update == 0 or num_update == len(ttb_ua_order)
|
| 164 |
+
|
| 165 |
+
n_tokens = q.shape[1]
|
| 166 |
+
|
| 167 |
+
# Inner loop fast weight update
|
| 168 |
+
for idx in range(len(ttb_ua_order)):
|
| 169 |
+
if ttb_ua_order[idx].update:
|
| 170 |
+
loop_kwargs = dict(
|
| 171 |
+
w0_now=w0, w1_now=w1, w2_now=w2,
|
| 172 |
+
ki=k, vi=v, lr0i=lr0, lr1i=lr1, lr2i=lr2,
|
| 173 |
+
use_mse=use_mse,
|
| 174 |
+
muon_update_steps=muon_update_steps, n_tokens=n_tokens,
|
| 175 |
+
use_weight_norm=per_step_norm,
|
| 176 |
+
w0_norm=w0_norm, w1_norm=w1_norm, w2_norm=w2_norm)
|
| 177 |
+
if inner_grad_checkpoint:
|
| 178 |
+
w0, w1, w2 = torch.utils.checkpoint.checkpoint(
|
| 179 |
+
inner_loop, **loop_kwargs, use_reentrant=False)
|
| 180 |
+
else:
|
| 181 |
+
w0, w1, w2 = inner_loop(**loop_kwargs)
|
| 182 |
+
|
| 183 |
+
# Apply with updated weights
|
| 184 |
+
output = (F.silu(q @ w0, inplace=True) * (q @ w2)) @ w1
|
| 185 |
+
|
| 186 |
+
return output, w0, w1, w2
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
fast_weight_swish_glu_weight_norm_mini_batch_apply = torch.compile(_fast_weight_kernel_body)
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
class FastWeightGluMLPMultiheadTTB(nn.Module):
|
| 193 |
+
"""TTB (Test-Time Distillation) module.
|
| 194 |
+
|
| 195 |
+
Distills cross-view attention into fast weights using separate q/k/v sources:
|
| 196 |
+
- q: from post-MLP tokens (for apply/query)
|
| 197 |
+
- k: from pre-cross-attn tokens (update input)
|
| 198 |
+
- v: from cross-attn delta (update target)
|
| 199 |
+
|
| 200 |
+
The fast weights learn: f_W(input) β CrossViewAttn(input)
|
| 201 |
+
"""
|
| 202 |
+
|
| 203 |
+
def __init__(
|
| 204 |
+
self,
|
| 205 |
+
dim: int,
|
| 206 |
+
head_dim: int,
|
| 207 |
+
inter_multi: int = 1,
|
| 208 |
+
bias: bool = False,
|
| 209 |
+
base_lr=0.01,
|
| 210 |
+
muon_update_steps=0,
|
| 211 |
+
use_gating=False,
|
| 212 |
+
use_mse_loss=False,
|
| 213 |
+
share_qk_proj=False,
|
| 214 |
+
per_step_norm=False,
|
| 215 |
+
inner_grad_checkpoint=False,
|
| 216 |
+
):
|
| 217 |
+
super().__init__()
|
| 218 |
+
self.dim = dim
|
| 219 |
+
assert dim % head_dim == 0
|
| 220 |
+
self.num_heads = dim // head_dim
|
| 221 |
+
self.muon_update_steps = muon_update_steps
|
| 222 |
+
self.use_gating = use_gating
|
| 223 |
+
self.per_step_norm = per_step_norm
|
| 224 |
+
self.inner_grad_checkpoint = inner_grad_checkpoint
|
| 225 |
+
self.use_mse_loss = use_mse_loss
|
| 226 |
+
self.share_qk_proj = share_qk_proj
|
| 227 |
+
d_in = d_out = head_dim
|
| 228 |
+
d_h = int(head_dim * inter_multi)
|
| 229 |
+
|
| 230 |
+
gain = math.sqrt(2) # for relu activations
|
| 231 |
+
self.w0 = nn.Parameter(
|
| 232 |
+
torch.randn(self.num_heads, d_in, d_h) * gain / math.sqrt(d_in)
|
| 233 |
+
)
|
| 234 |
+
self.w1 = nn.Parameter(
|
| 235 |
+
torch.randn(self.num_heads, d_h, d_out) * gain / math.sqrt(d_h)
|
| 236 |
+
)
|
| 237 |
+
self.w2 = nn.Parameter(
|
| 238 |
+
torch.randn(self.num_heads, d_in, d_h) * gain / math.sqrt(d_in)
|
| 239 |
+
)
|
| 240 |
+
|
| 241 |
+
# Q/K/V projections: shared or separate
|
| 242 |
+
if share_qk_proj:
|
| 243 |
+
self.to_qk = nn.Linear(dim, dim, bias=bias)
|
| 244 |
+
else:
|
| 245 |
+
self.to_q = nn.Linear(dim, dim, bias=bias)
|
| 246 |
+
self.to_k = nn.Linear(dim, dim, bias=bias)
|
| 247 |
+
self.to_v = nn.Linear(dim, dim, bias=bias)
|
| 248 |
+
self.c_proj = nn.Linear(dim, dim, bias=bias)
|
| 249 |
+
|
| 250 |
+
self.lr_dim = self.num_heads
|
| 251 |
+
self.lr_fc = nn.Linear(dim, self.lr_dim * 3)
|
| 252 |
+
self.base_lr_inv = inv_softplus(base_lr)
|
| 253 |
+
|
| 254 |
+
self.o_norm = torch.nn.RMSNorm(self.dim, eps=1e-5, elementwise_affine=True)
|
| 255 |
+
|
| 256 |
+
if use_gating:
|
| 257 |
+
self.gate_proj = nn.Linear(dim, dim, bias=bias)
|
| 258 |
+
|
| 259 |
+
def forward(self, x_query: torch.Tensor, info={}, x_key=None, x_value=None):
|
| 260 |
+
"""
|
| 261 |
+
Args:
|
| 262 |
+
x_query: (b, l, d) β tokens to query fast weights (post-MLP, normed)
|
| 263 |
+
info: dict with "ttb_op_order" and optionally "w0", "w1", "w2" states
|
| 264 |
+
x_key: (b, l, d) β pre-cross-attn tokens for update (None during rendering)
|
| 265 |
+
x_value: (b, l, d) β cross-attn delta for update target (None during rendering)
|
| 266 |
+
"""
|
| 267 |
+
B = x_query.shape[0]
|
| 268 |
+
H = self.num_heads
|
| 269 |
+
|
| 270 |
+
# Project query (always needed for apply)
|
| 271 |
+
qk_proj = self.to_qk if self.share_qk_proj else self.to_q
|
| 272 |
+
q = F.silu(qk_proj(x_query))
|
| 273 |
+
q = rearrange(q, "b l (h d) -> (b h) l d", h=H)
|
| 274 |
+
q = q / (q.norm(dim=2, keepdim=True) + 1e-5).to(x_query.dtype)
|
| 275 |
+
|
| 276 |
+
if x_key is not None:
|
| 277 |
+
# Reconstruction: project key and value from their respective sources
|
| 278 |
+
k_proj = self.to_qk if self.share_qk_proj else self.to_k
|
| 279 |
+
k = F.silu(k_proj(x_key))
|
| 280 |
+
v = F.silu(self.to_v(x_value))
|
| 281 |
+
k = rearrange(k, "b l (h d) -> (b h) l d", h=H)
|
| 282 |
+
v = rearrange(v, "b l (h d) -> (b h) l d", h=H)
|
| 283 |
+
k = k / (k.norm(dim=2, keepdim=True) + 1e-5).to(x_key.dtype)
|
| 284 |
+
v = v / (v.norm(dim=2, keepdim=True) + 1e-5).to(x_value.dtype)
|
| 285 |
+
|
| 286 |
+
# Learning rate computed from key tokens
|
| 287 |
+
with torch.autocast(device_type="cuda", enabled=False):
|
| 288 |
+
lr = self.lr_fc(x_key.float())
|
| 289 |
+
|
| 290 |
+
lr = torch.nn.functional.softplus(lr.float() + self.base_lr_inv)
|
| 291 |
+
|
| 292 |
+
lr0, lr1, lr2 = rearrange(
|
| 293 |
+
lr, "b l (lrs h d) -> lrs (b h) l d",
|
| 294 |
+
lrs=3, h=H
|
| 295 |
+
)
|
| 296 |
+
else:
|
| 297 |
+
# Rendering: dummy k/v/lr (update=False, never accessed by inner loop)
|
| 298 |
+
k = q.new_zeros(q.shape[0], 0, q.shape[2])
|
| 299 |
+
v = k
|
| 300 |
+
lr0 = lr1 = lr2 = q.new_zeros(q.shape[0], 0, 1)
|
| 301 |
+
|
| 302 |
+
if "w0" in info:
|
| 303 |
+
assert "w1" in info and "w2" in info
|
| 304 |
+
sw0 = info["w0"]
|
| 305 |
+
sw1 = info["w1"]
|
| 306 |
+
sw2 = info["w2"]
|
| 307 |
+
else:
|
| 308 |
+
sw0 = self.w0.repeat(B, 1, 1)
|
| 309 |
+
sw1 = self.w1.repeat(B, 1, 1)
|
| 310 |
+
sw2 = self.w2.repeat(B, 1, 1)
|
| 311 |
+
|
| 312 |
+
shared_out, sw0, sw1, sw2 = fast_weight_swish_glu_weight_norm_mini_batch_apply(
|
| 313 |
+
sw0, sw1, sw2, q, k, v, lr0, lr1, lr2, info["ttb_op_order"],
|
| 314 |
+
muon_update_steps=self.muon_update_steps,
|
| 315 |
+
use_mse=self.use_mse_loss,
|
| 316 |
+
per_step_norm=self.per_step_norm,
|
| 317 |
+
inner_grad_checkpoint=self.inner_grad_checkpoint,
|
| 318 |
+
)
|
| 319 |
+
|
| 320 |
+
output = shared_out
|
| 321 |
+
|
| 322 |
+
output = rearrange(
|
| 323 |
+
output, "(b h) l d -> b l (h d)", h=H, b=B
|
| 324 |
+
)
|
| 325 |
+
output = self.o_norm(output) # (b, l, dim)
|
| 326 |
+
|
| 327 |
+
if self.use_gating:
|
| 328 |
+
output = output * F.silu(self.gate_proj(output))
|
| 329 |
+
|
| 330 |
+
out = self.c_proj(output)
|
| 331 |
+
|
| 332 |
+
state = {"w0": sw0, "w1": sw1, "w2": sw2}
|
| 333 |
+
return out, state
|
| 334 |
+
|
| 335 |
+
def extra_repr(self) -> str:
|
| 336 |
+
return (f"w0 shape: {self.w0.shape}, w1 shape: {self.w1.shape}, w2 shape: {self.w2.shape}, "
|
| 337 |
+
f"Muon update steps: {self.muon_update_steps}, "
|
| 338 |
+
f"Base lr: {math.log(1 + math.exp(self.base_lr_inv))}, ")
|
| 339 |
+
|
| 340 |
+
|
packages.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
ffmpeg
|
requirements.txt
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# torch / torchvision are provided by the ZeroGPU runtime (torch 2.8+, Blackwell
|
| 2 |
+
# sm_120). Do NOT pin them here β pinning torch==2.7.0+cu128 breaks ZeroGPU.
|
| 3 |
+
#
|
| 4 |
+
# flash-attn is intentionally omitted: dvsm/models/modules.py falls back to torch
|
| 5 |
+
# SDPA when flash_attn is unavailable (the 2.7.3 torch2.6-ABI wheel does not work
|
| 6 |
+
# on Blackwell). See README for the second-phase FA3 option.
|
| 7 |
+
spaces
|
| 8 |
+
huggingface_hub
|
| 9 |
+
viser
|
| 10 |
+
numpy==1.26.4
|
| 11 |
+
einops
|
| 12 |
+
imageio[pyav]
|
| 13 |
+
opencv-python-headless
|
| 14 |
+
Pillow
|
| 15 |
+
transformers
|
| 16 |
+
yacs
|
| 17 |
+
scipy
|
| 18 |
+
scikit-image
|
| 19 |
+
scikit-learn
|
| 20 |
+
natsort
|
| 21 |
+
tqdm
|
| 22 |
+
httpx
|
| 23 |
+
websockets
|
| 24 |
+
git+https://github.com/yyfz/Pi3.git
|
viser_app/__init__.py
ADDED
|
File without changes
|
viser_app/cam_convention.py
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Camera convention conversion between viser and DVSM.
|
| 2 |
+
|
| 3 |
+
Both use OpenCV camera-local convention (+X right, +Y down, +Z forward) per
|
| 4 |
+
viser's API docs ("Like all cameras in the viser Python API, frustums follow
|
| 5 |
+
the OpenCV [+Z forward, +X right, +Y down] convention"). So the wxyz/position
|
| 6 |
+
mapping is direct -- no camera-local axis flip.
|
| 7 |
+
|
| 8 |
+
The viewer applies an additional WORLD rotation outside of this module to
|
| 9 |
+
align DVSM's normalized world (with -Y up) to viser/three.js native (+Y up).
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import numpy as np
|
| 14 |
+
import torch
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def wxyz_to_matrix(wxyz: np.ndarray) -> np.ndarray:
|
| 18 |
+
"""Quaternion (w, x, y, z) -> 3x3 rotation matrix."""
|
| 19 |
+
w, x, y, z = wxyz
|
| 20 |
+
n = w * w + x * x + y * y + z * z
|
| 21 |
+
s = 0.0 if n < 1e-12 else 2.0 / n
|
| 22 |
+
wx, wy, wz = s * w * x, s * w * y, s * w * z
|
| 23 |
+
xx, xy, xz = s * x * x, s * x * y, s * x * z
|
| 24 |
+
yy, yz, zz = s * y * y, s * y * z, s * z * z
|
| 25 |
+
return np.array([
|
| 26 |
+
[1.0 - (yy + zz), xy - wz, xz + wy ],
|
| 27 |
+
[xy + wz, 1.0 - (xx + zz), yz - wx ],
|
| 28 |
+
[xz - wy, yz + wx, 1.0 - (xx + yy)],
|
| 29 |
+
], dtype=np.float32)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def matrix_to_wxyz(R: np.ndarray) -> np.ndarray:
|
| 33 |
+
"""3x3 rotation matrix -> quaternion (w, x, y, z). Stable branch selection."""
|
| 34 |
+
R = R.astype(np.float64)
|
| 35 |
+
m00, m01, m02 = R[0, 0], R[0, 1], R[0, 2]
|
| 36 |
+
m10, m11, m12 = R[1, 0], R[1, 1], R[1, 2]
|
| 37 |
+
m20, m21, m22 = R[2, 0], R[2, 1], R[2, 2]
|
| 38 |
+
tr = m00 + m11 + m22
|
| 39 |
+
if tr > 0:
|
| 40 |
+
s = np.sqrt(tr + 1.0) * 2
|
| 41 |
+
w = 0.25 * s
|
| 42 |
+
x = (m21 - m12) / s
|
| 43 |
+
y = (m02 - m20) / s
|
| 44 |
+
z = (m10 - m01) / s
|
| 45 |
+
elif (m00 > m11) and (m00 > m22):
|
| 46 |
+
s = np.sqrt(1.0 + m00 - m11 - m22) * 2
|
| 47 |
+
w = (m21 - m12) / s
|
| 48 |
+
x = 0.25 * s
|
| 49 |
+
y = (m01 + m10) / s
|
| 50 |
+
z = (m02 + m20) / s
|
| 51 |
+
elif m11 > m22:
|
| 52 |
+
s = np.sqrt(1.0 + m11 - m00 - m22) * 2
|
| 53 |
+
w = (m02 - m20) / s
|
| 54 |
+
x = (m01 + m10) / s
|
| 55 |
+
y = 0.25 * s
|
| 56 |
+
z = (m12 + m21) / s
|
| 57 |
+
else:
|
| 58 |
+
s = np.sqrt(1.0 + m22 - m00 - m11) * 2
|
| 59 |
+
w = (m10 - m01) / s
|
| 60 |
+
x = (m02 + m20) / s
|
| 61 |
+
y = (m12 + m21) / s
|
| 62 |
+
z = 0.25 * s
|
| 63 |
+
q = np.array([w, x, y, z], dtype=np.float32)
|
| 64 |
+
return q / np.linalg.norm(q)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def fov_to_normalized_intrinsics(fov_y: float, aspect: float) -> np.ndarray:
|
| 68 |
+
"""Vertical FoV (rad) and aspect (W/H) -> DVSM-normalized intrinsics [fx, fy, cx, cy].
|
| 69 |
+
|
| 70 |
+
DVSM normalization: fx*W = focal_x_pixels, fy*H = focal_y_pixels.
|
| 71 |
+
Pinhole with square pixels: tan(fov_x/2) = aspect * tan(fov_y/2).
|
| 72 |
+
"""
|
| 73 |
+
tan_half = np.tan(0.5 * fov_y)
|
| 74 |
+
fy = 0.5 / tan_half
|
| 75 |
+
fx = fy / aspect
|
| 76 |
+
return np.array([fx, fy, 0.5, 0.5], dtype=np.float32)
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def normalized_intrinsics_to_fov(intr: np.ndarray) -> tuple[float, float]:
|
| 80 |
+
"""DVSM intrinsics [fx, fy, cx, cy] -> (fov_y in radians, aspect W/H)."""
|
| 81 |
+
fx, fy = float(intr[0]), float(intr[1])
|
| 82 |
+
fov_y = 2.0 * np.arctan(0.5 / fy)
|
| 83 |
+
aspect = fy / fx
|
| 84 |
+
return fov_y, aspect
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def viser_cam_to_dvsm(
|
| 88 |
+
position: np.ndarray,
|
| 89 |
+
wxyz: np.ndarray,
|
| 90 |
+
fov_y: float,
|
| 91 |
+
aspect: float,
|
| 92 |
+
device: torch.device | str = "cpu",
|
| 93 |
+
dtype: torch.dtype = torch.float32,
|
| 94 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 95 |
+
"""Convert viser ClientHandle.camera state to DVSM render inputs.
|
| 96 |
+
|
| 97 |
+
Returns:
|
| 98 |
+
intr: [1, 1, 4] normalized [fx, fy, cx, cy]
|
| 99 |
+
c2w_r: [1, 1, 3, 3]
|
| 100 |
+
c2w_t: [1, 1, 3]
|
| 101 |
+
"""
|
| 102 |
+
R_cv = wxyz_to_matrix(np.asarray(wxyz, dtype=np.float32))
|
| 103 |
+
t = np.asarray(position, dtype=np.float32)
|
| 104 |
+
intr_np = fov_to_normalized_intrinsics(fov_y, aspect)
|
| 105 |
+
intr = torch.from_numpy(intr_np).to(device=device, dtype=dtype).view(1, 1, 4)
|
| 106 |
+
c2w_r = torch.from_numpy(R_cv).to(device=device, dtype=dtype).view(1, 1, 3, 3)
|
| 107 |
+
c2w_t = torch.from_numpy(t).to(device=device, dtype=dtype).view(1, 1, 3)
|
| 108 |
+
return intr, c2w_r, c2w_t
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def dvsm_cam_to_viser(
|
| 112 |
+
intr: torch.Tensor | np.ndarray,
|
| 113 |
+
c2w_r: torch.Tensor | np.ndarray,
|
| 114 |
+
c2w_t: torch.Tensor | np.ndarray,
|
| 115 |
+
) -> tuple[np.ndarray, np.ndarray, float, float]:
|
| 116 |
+
"""Inverse of viser_cam_to_dvsm. Accepts unbatched or [1,1,...] shapes.
|
| 117 |
+
|
| 118 |
+
Returns:
|
| 119 |
+
position: (3,) numpy
|
| 120 |
+
wxyz: (4,) numpy, w-first
|
| 121 |
+
fov_y: float, radians
|
| 122 |
+
aspect: float, W/H
|
| 123 |
+
"""
|
| 124 |
+
def _to_np(t):
|
| 125 |
+
if isinstance(t, torch.Tensor):
|
| 126 |
+
return t.detach().cpu().numpy()
|
| 127 |
+
return np.asarray(t)
|
| 128 |
+
intr_np = _to_np(intr).reshape(-1)[:4]
|
| 129 |
+
R_cv = _to_np(c2w_r).reshape(3, 3)
|
| 130 |
+
t = _to_np(c2w_t).reshape(3).astype(np.float32)
|
| 131 |
+
wxyz = matrix_to_wxyz(R_cv)
|
| 132 |
+
fov_y, aspect = normalized_intrinsics_to_fov(intr_np.astype(np.float32))
|
| 133 |
+
return t, wxyz, fov_y, aspect
|
viser_app/pi3_estimator.py
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ΟΒ³ (Pi3X) front-end: estimate camera poses + intrinsics from unposed images.
|
| 2 |
+
|
| 3 |
+
Pi3X (yyfz233/Pi3X, ICLR 2026) is a feed-forward Permutation-Equivariant Visual
|
| 4 |
+
Geometry Learning model. Given a batch of images, it returns:
|
| 5 |
+
- camera_poses : (B, N, 4, 4) camera-to-world in OpenCV convention
|
| 6 |
+
- points : (B, N, H, W, 3) global point cloud
|
| 7 |
+
- local_points : (B, N, H, W, 3) per-view points in CAMERA frame
|
| 8 |
+
- conf : (B, N, H, W, 1) confidence logits
|
| 9 |
+
|
| 10 |
+
Pi3X does NOT output camera intrinsics. We estimate them from ``local_points``:
|
| 11 |
+
for a pinhole camera with principal point at the image center,
|
| 12 |
+
X / Z = (u - cx) / fx β fx = (u - cx) * Z / X
|
| 13 |
+
We solve the same for fy and take a confidence-weighted median across all valid
|
| 14 |
+
pixels of all views. Far from perfect, but good enough as an init for DVSM.
|
| 15 |
+
"""
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
from typing import Optional
|
| 20 |
+
|
| 21 |
+
import numpy as np
|
| 22 |
+
import torch
|
| 23 |
+
from PIL import Image
|
| 24 |
+
|
| 25 |
+
from viser_app.scene_loaders import pack_estimator_scene_data
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def _list_images(d: Path) -> list[Path]:
|
| 29 |
+
exts = {".jpg", ".jpeg", ".png", ".bmp", ".JPG", ".PNG", ".JPEG"}
|
| 30 |
+
return sorted([p for p in d.iterdir() if p.suffix in exts])
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
# ---------------------------------------------------------------------------
|
| 34 |
+
# Pi3X loading + inference
|
| 35 |
+
# ---------------------------------------------------------------------------
|
| 36 |
+
|
| 37 |
+
def load_pi3_model(flavor: str = "pi3x", device: str = "cuda"):
|
| 38 |
+
"""Load Pi3 or Pi3X via the GitHub package (huggingface checkpoint)."""
|
| 39 |
+
if flavor == "pi3x":
|
| 40 |
+
from pi3.models.pi3x import Pi3X
|
| 41 |
+
model = Pi3X.from_pretrained("yyfz233/Pi3X")
|
| 42 |
+
elif flavor == "pi3":
|
| 43 |
+
from pi3.models.pi3 import Pi3
|
| 44 |
+
model = Pi3.from_pretrained("yyfz233/Pi3")
|
| 45 |
+
else:
|
| 46 |
+
raise ValueError(f"unknown flavor {flavor!r}; expected 'pi3' or 'pi3x'")
|
| 47 |
+
return model.to(device).eval()
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _resize_for_pi3(img_pil: Image.Image, image_size: int = 518, patch: int = 14) -> Image.Image:
|
| 51 |
+
"""Aspect-preserving resize so longest edge β€ image_size, both dims multiple of `patch`."""
|
| 52 |
+
w, h = img_pil.size
|
| 53 |
+
if max(w, h) > image_size:
|
| 54 |
+
s = image_size / max(w, h)
|
| 55 |
+
w = int(round(w * s))
|
| 56 |
+
h = int(round(h * s))
|
| 57 |
+
w = max(patch, (w // patch) * patch)
|
| 58 |
+
h = max(patch, (h // patch) * patch)
|
| 59 |
+
return img_pil.resize((w, h), Image.LANCZOS)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def run_pi3(model, image_paths: list[Path], device: str = "cuda",
|
| 63 |
+
image_size: int = 518) -> dict:
|
| 64 |
+
"""Returns:
|
| 65 |
+
raw_images : list of native HxWx3 uint8 ndarrays
|
| 66 |
+
c2w : (N, 4, 4) float32 numpy camera-to-world (OpenCV)
|
| 67 |
+
intr : (fx, fy, cx, cy) pixels, valid for image_size
|
| 68 |
+
image_size : (h, w) of the (uniform) image batch fed to Pi3X
|
| 69 |
+
"""
|
| 70 |
+
raw_imgs: list[np.ndarray] = []
|
| 71 |
+
resized_imgs: list[np.ndarray] = []
|
| 72 |
+
for p in image_paths:
|
| 73 |
+
im = Image.open(p).convert("RGB")
|
| 74 |
+
raw_imgs.append(np.asarray(im))
|
| 75 |
+
resized_imgs.append(np.asarray(_resize_for_pi3(im, image_size=image_size)))
|
| 76 |
+
|
| 77 |
+
# Batched Pi3 input must be uniform-size. Use the smallest dims (snapped to 14).
|
| 78 |
+
h_b = min(im.shape[0] for im in resized_imgs)
|
| 79 |
+
w_b = min(im.shape[1] for im in resized_imgs)
|
| 80 |
+
h_b = (h_b // 14) * 14
|
| 81 |
+
w_b = (w_b // 14) * 14
|
| 82 |
+
batch = np.stack([
|
| 83 |
+
np.asarray(Image.fromarray(im).resize((w_b, h_b), Image.LANCZOS))
|
| 84 |
+
for im in resized_imgs
|
| 85 |
+
])
|
| 86 |
+
print(f"[pi3] feeding {len(batch)} images at {h_b}x{w_b}")
|
| 87 |
+
|
| 88 |
+
imgs_t = torch.from_numpy(batch).permute(0, 3, 1, 2).float() / 255.0
|
| 89 |
+
imgs_t = imgs_t.to(device)
|
| 90 |
+
with torch.no_grad():
|
| 91 |
+
with torch.amp.autocast("cuda", dtype=torch.bfloat16):
|
| 92 |
+
out = model(imgs_t[None]) # (1, N, ...)
|
| 93 |
+
|
| 94 |
+
c2w = out["camera_poses"][0].float().cpu().numpy() # (N, 4, 4)
|
| 95 |
+
local_pts = out["local_points"][0].float().cpu().numpy() # (N, H, W, 3)
|
| 96 |
+
conf = out["conf"][0].float().cpu().numpy() if "conf" in out else None
|
| 97 |
+
|
| 98 |
+
fx, fy, cx, cy = _estimate_intrinsics_from_local_points(local_pts, conf)
|
| 99 |
+
print(f"[pi3] estimated intr (px @ {h_b}x{w_b}): fx={fx:.1f} fy={fy:.1f} "
|
| 100 |
+
f"cx={cx:.1f} cy={cy:.1f}")
|
| 101 |
+
|
| 102 |
+
return dict(
|
| 103 |
+
raw_images=raw_imgs,
|
| 104 |
+
c2w=c2w,
|
| 105 |
+
intr=(fx, fy, cx, cy),
|
| 106 |
+
image_size=(h_b, w_b),
|
| 107 |
+
)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def _estimate_intrinsics_from_local_points(local_pts: np.ndarray,
|
| 111 |
+
conf: Optional[np.ndarray] = None,
|
| 112 |
+
) -> tuple[float, float, float, float]:
|
| 113 |
+
"""Assume centered principal point; solve fx, fy by robust median.
|
| 114 |
+
|
| 115 |
+
local_pts: (N, H, W, 3) β camera-frame XYZ for each pixel
|
| 116 |
+
conf: (N, H, W, 1) sigmoid logits (optional)
|
| 117 |
+
"""
|
| 118 |
+
N, H, W, _ = local_pts.shape
|
| 119 |
+
cx = W / 2.0
|
| 120 |
+
cy = H / 2.0
|
| 121 |
+
u, v = np.meshgrid(np.arange(W), np.arange(H), indexing="xy")
|
| 122 |
+
ux = (u - cx).astype(np.float64)[None] # (1, H, W)
|
| 123 |
+
vy = (v - cy).astype(np.float64)[None]
|
| 124 |
+
X = local_pts[..., 0].astype(np.float64)
|
| 125 |
+
Y = local_pts[..., 1].astype(np.float64)
|
| 126 |
+
Z = local_pts[..., 2].astype(np.float64)
|
| 127 |
+
|
| 128 |
+
# X/Z = (u-cx)/fx β fx = (u-cx) Z / X
|
| 129 |
+
eps = 1e-3
|
| 130 |
+
valid_fx = (Z > 0.05) & (np.abs(X) > eps) & (np.abs(ux) > 0.5)
|
| 131 |
+
valid_fy = (Z > 0.05) & (np.abs(Y) > eps) & (np.abs(vy) > 0.5)
|
| 132 |
+
if conf is not None:
|
| 133 |
+
c = 1.0 / (1.0 + np.exp(-conf.squeeze(-1))) # sigmoid
|
| 134 |
+
valid_fx &= (c > 0.5)
|
| 135 |
+
valid_fy &= (c > 0.5)
|
| 136 |
+
|
| 137 |
+
fx_pix = ux * Z / X
|
| 138 |
+
fy_pix = vy * Z / Y
|
| 139 |
+
fx = float(np.median(fx_pix[valid_fx])) if valid_fx.any() else max(W, H) * 1.0
|
| 140 |
+
fy = float(np.median(fy_pix[valid_fy])) if valid_fy.any() else max(W, H) * 1.0
|
| 141 |
+
# fx/fy can come out negative if Pi3's convention has flipped sign on X; take abs.
|
| 142 |
+
return abs(fx), abs(fy), cx, cy
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
# ---------------------------------------------------------------------------
|
| 146 |
+
# Pi3 result β scene_data (same format scene_loaders returns)
|
| 147 |
+
# ---------------------------------------------------------------------------
|
| 148 |
+
|
| 149 |
+
def pi3_to_scene_data(pi3_result: dict, target_h: int, target_w: int) -> dict:
|
| 150 |
+
return pack_estimator_scene_data(
|
| 151 |
+
pi3_result, target_h, target_w,
|
| 152 |
+
scene_name="pi3_scene", source="pi3_estimated",
|
| 153 |
+
)
|
viser_app/recon_scene.py
ADDED
|
@@ -0,0 +1,303 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Stage A: reconstruct one external scene -> self-contained .recon.pt artifact.
|
| 2 |
+
|
| 3 |
+
Loads a trained DVSM/TTB-DVSM checkpoint, builds the per-block fast weights
|
| 4 |
+
(kv_lst) + per-scene PosePreprocessor state for one scene, and saves a
|
| 5 |
+
self-contained artifact for viser_app/viewer.py (no input images needed after).
|
| 6 |
+
|
| 7 |
+
Pose source (--pose):
|
| 8 |
+
- auto (default): use COLMAP / transforms.json found under --input, otherwise
|
| 9 |
+
estimate poses with ΟΒ³ (a flat image folder needs no poses).
|
| 10 |
+
- colmap: read COLMAP ``sparse/*/cameras.{bin,txt}`` or nerfstudio
|
| 11 |
+
``transforms.json`` from --input (a directory or a .json file).
|
| 12 |
+
- pi3: estimate camera-to-world poses + intrinsics with ΟΒ³ from a
|
| 13 |
+
flat folder of images OR a video file (no cameras.bin, no transforms.json).
|
| 14 |
+
A video is decoded to --max_views evenly-spaced frames first.
|
| 15 |
+
|
| 16 |
+
Recon size defaults to: keep the input's aspect ratio, resize the long edge to
|
| 17 |
+
--long_edge (960), snapped to patch_size. Override with --target_h/--target_w.
|
| 18 |
+
|
| 19 |
+
Usage:
|
| 20 |
+
# auto: COLMAP/transforms if present, else ΟΒ³ (image folder / video)
|
| 21 |
+
python -m viser_app.recon_scene \\
|
| 22 |
+
--ckpt_root outputs/dl3dv/960p_ttb_dvsm_minus \\
|
| 23 |
+
--input /path/to/scene_or_video --max_views 32 \\
|
| 24 |
+
--out outputs/recon/scene.recon.pt
|
| 25 |
+
|
| 26 |
+
# force ΟΒ³ on a flat image folder, long edge 720
|
| 27 |
+
python -m viser_app.recon_scene --pose pi3 --long_edge 720 \\
|
| 28 |
+
--ckpt_root outputs/dl3dv/960p_ttb_dvsm_minus \\
|
| 29 |
+
--input /path/to/images --max_views 32 \\
|
| 30 |
+
--out outputs/recon/scene.recon.pt
|
| 31 |
+
"""
|
| 32 |
+
from __future__ import annotations
|
| 33 |
+
|
| 34 |
+
import argparse
|
| 35 |
+
import sys
|
| 36 |
+
import tempfile
|
| 37 |
+
from pathlib import Path
|
| 38 |
+
|
| 39 |
+
import numpy as np
|
| 40 |
+
import torch
|
| 41 |
+
|
| 42 |
+
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
| 43 |
+
sys.path.insert(0, str(PROJECT_ROOT))
|
| 44 |
+
|
| 45 |
+
from dvsm.config import cfg, update_config
|
| 46 |
+
from dvsm.models.DVSM import DecoderViewSynthesisModel
|
| 47 |
+
from viser_app.scene_loaders import (
|
| 48 |
+
VIDEO_EXTS, _find_colmap_dir, extract_video_frames, load_external_scene,
|
| 49 |
+
)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
# ---------------------------------------------------------------------------
|
| 53 |
+
# Pose source
|
| 54 |
+
# ---------------------------------------------------------------------------
|
| 55 |
+
def detect_pose_source(input_path: Path) -> str:
|
| 56 |
+
"""auto: 'colmap' if --input has COLMAP sparse / transforms.json, else 'pi3'.
|
| 57 |
+
|
| 58 |
+
A video file or a flat image folder has no poses -> 'pi3'.
|
| 59 |
+
"""
|
| 60 |
+
p = Path(input_path)
|
| 61 |
+
if p.is_file() and p.suffix.lower() in VIDEO_EXTS:
|
| 62 |
+
return "pi3"
|
| 63 |
+
if p.is_file() and p.suffix == ".json":
|
| 64 |
+
return "colmap"
|
| 65 |
+
if p.is_dir():
|
| 66 |
+
if (p / "transforms.json").exists():
|
| 67 |
+
return "colmap"
|
| 68 |
+
try:
|
| 69 |
+
_find_colmap_dir(p)
|
| 70 |
+
return "colmap"
|
| 71 |
+
except Exception:
|
| 72 |
+
pass
|
| 73 |
+
return "pi3"
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _snap(v: float, patch: int) -> int:
|
| 77 |
+
return max(patch, (int(round(v)) // patch) * patch)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def derive_target_hw(sample_img: Path, long_edge: int, patch: int) -> tuple:
|
| 81 |
+
"""Keep the input's aspect ratio, resize the LONG edge to ``long_edge``, and
|
| 82 |
+
snap both axes to multiples of ``patch`` (mirrors the gradio demo's logic)."""
|
| 83 |
+
from PIL import Image
|
| 84 |
+
with Image.open(sample_img) as im:
|
| 85 |
+
iw, ih = im.size
|
| 86 |
+
aspect = iw / ih # >1 landscape, <1 portrait
|
| 87 |
+
le = _snap(long_edge, patch)
|
| 88 |
+
if aspect >= 1.0:
|
| 89 |
+
w, h = le, _snap(le / aspect, patch)
|
| 90 |
+
else:
|
| 91 |
+
h, w = le, _snap(le * aspect, patch)
|
| 92 |
+
return h, w
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _peek_image(input_path: Path) -> Path:
|
| 96 |
+
"""First image under --input (a dir) or its parent (a .json), to infer aspect."""
|
| 97 |
+
from viser_app.scene_loaders import IMAGE_EXTS
|
| 98 |
+
p = Path(input_path)
|
| 99 |
+
root = p.parent if p.is_file() else p
|
| 100 |
+
cands = sorted(f for f in root.rglob("*") if f.suffix.lower() in IMAGE_EXTS)
|
| 101 |
+
if not cands:
|
| 102 |
+
raise FileNotFoundError(f"no image under {root} to infer aspect ratio")
|
| 103 |
+
return cands[0]
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def _resolve_target(args, sample_img: Path, patch: int) -> tuple:
|
| 107 |
+
"""--target_h/--target_w if both given, else derive from aspect + --long_edge."""
|
| 108 |
+
if args.target_h and args.target_w:
|
| 109 |
+
return args.target_h, args.target_w
|
| 110 |
+
th, tw = derive_target_hw(sample_img, args.long_edge, patch)
|
| 111 |
+
print(f"[recon] target derived from aspect: {tw}x{th} "
|
| 112 |
+
f"(long_edge={args.long_edge}, patch={patch})", flush=True)
|
| 113 |
+
return th, tw
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def _get_patch_size(args) -> int:
|
| 117 |
+
"""Return patch_size from local config or HF model config."""
|
| 118 |
+
if getattr(args, 'hf_variant', None):
|
| 119 |
+
import json
|
| 120 |
+
from huggingface_hub import snapshot_download
|
| 121 |
+
local_dir = snapshot_download(repo_id="nvidia/DVSM",
|
| 122 |
+
allow_patterns=f"{args.hf_variant}/*")
|
| 123 |
+
cfg_path = Path(local_dir) / args.hf_variant / "config.json"
|
| 124 |
+
return json.loads(cfg_path.read_text()).get("patch_size", 8)
|
| 125 |
+
update_config([args.ckpt_root / "config.yaml"])
|
| 126 |
+
return cfg.model.patch_size
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def load_scene_data(args, pose: str) -> dict:
|
| 130 |
+
"""Build scene_data {image, intr, c2w_r, c2w_t, scene_name, ...} for recon."""
|
| 131 |
+
patch = _get_patch_size(args)
|
| 132 |
+
if pose == "colmap":
|
| 133 |
+
target_h, target_w = _resolve_target(args, _peek_image(args.input), patch)
|
| 134 |
+
print(f"[recon] pose=colmap loading {args.input}", flush=True)
|
| 135 |
+
scene_data = load_external_scene(
|
| 136 |
+
args.input, target_h=target_h, target_w=target_w,
|
| 137 |
+
max_views=args.max_views, stride=args.stride,
|
| 138 |
+
convention=args.convention, images_subdir=args.images_subdir,
|
| 139 |
+
)
|
| 140 |
+
elif pose == "pi3":
|
| 141 |
+
from viser_app.pi3_estimator import (
|
| 142 |
+
_list_images, load_pi3_model, pi3_to_scene_data, run_pi3,
|
| 143 |
+
)
|
| 144 |
+
# A video file -> extract evenly-spaced frames to a temp folder first.
|
| 145 |
+
image_root = Path(args.input)
|
| 146 |
+
if image_root.is_file() and image_root.suffix.lower() in VIDEO_EXTS:
|
| 147 |
+
n_take = args.max_views or 64
|
| 148 |
+
tmp = Path(tempfile.mkdtemp(prefix="recon_video_"))
|
| 149 |
+
n = extract_video_frames(image_root, tmp, n_take)
|
| 150 |
+
print(f"[recon] extracted {n} frames from video {image_root.name} -> {tmp}", flush=True)
|
| 151 |
+
image_root = tmp
|
| 152 |
+
image_paths = _list_images(image_root)[::args.stride]
|
| 153 |
+
if args.max_views and len(image_paths) > args.max_views:
|
| 154 |
+
idxs = np.linspace(0, len(image_paths) - 1, args.max_views).round().astype(int)
|
| 155 |
+
image_paths = [image_paths[i] for i in idxs]
|
| 156 |
+
target_h, target_w = _resolve_target(args, image_paths[0], patch)
|
| 157 |
+
print(f"[recon] pose=pi3 {len(image_paths)} images from {args.input}", flush=True)
|
| 158 |
+
pi3 = load_pi3_model(flavor=args.pi3_flavor)
|
| 159 |
+
est = run_pi3(pi3, image_paths, image_size=args.pi3_image_size)
|
| 160 |
+
del pi3; torch.cuda.empty_cache()
|
| 161 |
+
scene_data = pi3_to_scene_data(est, target_h, target_w)
|
| 162 |
+
else:
|
| 163 |
+
raise ValueError(f"unknown pose source: {pose}")
|
| 164 |
+
print(f"[recon] scene packed: V={scene_data['n_input_views']} "
|
| 165 |
+
f"{scene_data['H']}x{scene_data['W']} source={scene_data.get('source', pose)}", flush=True)
|
| 166 |
+
return scene_data
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
# ---------------------------------------------------------------------------
|
| 170 |
+
# Reconstruction core (shared by every pose source)
|
| 171 |
+
# ---------------------------------------------------------------------------
|
| 172 |
+
def reconstruct(ckpt_root: Path | None, ckpt_name: str, scene_data: dict,
|
| 173 |
+
cmd_lst: list[str], hf_variant: str | None = None,
|
| 174 |
+
model: DecoderViewSynthesisModel | None = None):
|
| 175 |
+
if model is not None:
|
| 176 |
+
# Caller passed an already-loaded model (e.g. a module-level CPU model
|
| 177 |
+
# moved to cuda inside a @spaces.GPU worker). Skip download / ckpt load.
|
| 178 |
+
model = model.cuda().eval()
|
| 179 |
+
elif hf_variant is not None:
|
| 180 |
+
model = DecoderViewSynthesisModel.from_hf_pretrained(hf_variant)
|
| 181 |
+
model = model.cuda().eval()
|
| 182 |
+
else:
|
| 183 |
+
update_config([ckpt_root / "config.yaml"], cmd_lst)
|
| 184 |
+
cfg.model.use_compile = False # avoid first-call compile cost during recon
|
| 185 |
+
ckpt_path = ckpt_root / ckpt_name
|
| 186 |
+
assert ckpt_path.exists(), ckpt_path
|
| 187 |
+
model = DecoderViewSynthesisModel(**cfg.model).cuda().eval()
|
| 188 |
+
ckpt = torch.load(ckpt_path, weights_only=False, map_location="cpu")
|
| 189 |
+
model.load_state_dict(ckpt["state_dict"])
|
| 190 |
+
del ckpt
|
| 191 |
+
print(f"[recon] patch={model.patch_size}")
|
| 192 |
+
|
| 193 |
+
image = scene_data["image"].cuda().unsqueeze(0)
|
| 194 |
+
intr = scene_data["intr"].cuda().unsqueeze(0)
|
| 195 |
+
c2w_r = scene_data["c2w_r"].cuda().unsqueeze(0)
|
| 196 |
+
c2w_t = scene_data["c2w_t"].cuda().unsqueeze(0)
|
| 197 |
+
|
| 198 |
+
H, W = image.shape[-2:]
|
| 199 |
+
assert H % model.patch_size == 0 and W % model.patch_size == 0, (
|
| 200 |
+
f"target {H}x{W} not divisible by patch_size {model.patch_size}")
|
| 201 |
+
|
| 202 |
+
print(f"[recon] input image shape {tuple(image.shape)}")
|
| 203 |
+
print(f"[recon] running reconstruction...")
|
| 204 |
+
with torch.no_grad():
|
| 205 |
+
renderer = model(image, intr, c2w_r, c2w_t)
|
| 206 |
+
print(f"[recon] kv_lst length {len(renderer.kv_lst)}")
|
| 207 |
+
|
| 208 |
+
batch = {"ds_names": [scene_data["scene_name"]]}
|
| 209 |
+
return renderer, batch, image.shape, c2w_r, c2w_t, intr
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
def _move_kv_to_cpu(kv):
|
| 213 |
+
if kv is None:
|
| 214 |
+
return None
|
| 215 |
+
if isinstance(kv, dict):
|
| 216 |
+
return {k: (v.cpu() if isinstance(v, torch.Tensor) else v) for k, v in kv.items()}
|
| 217 |
+
if isinstance(kv, (tuple, list)):
|
| 218 |
+
return type(kv)(_move_kv_to_cpu(v) for v in kv)
|
| 219 |
+
if isinstance(kv, torch.Tensor):
|
| 220 |
+
return kv.cpu()
|
| 221 |
+
return kv
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def save_artifact(renderer, batch, image_shape, c2w_r, c2w_t, intr,
|
| 225 |
+
out_path: Path, ckpt_root: Path | None = None,
|
| 226 |
+
ckpt_name: str = "latest.pt", hf_variant: str | None = None):
|
| 227 |
+
pp = renderer.pose_preprocessor
|
| 228 |
+
model_cfg = dict(cfg.model) if hf_variant is None else {}
|
| 229 |
+
artifact = {
|
| 230 |
+
"kv_lst": [_move_kv_to_cpu(kv) for kv in renderer.kv_lst],
|
| 231 |
+
"pose_preproc": {"r": pp.r.cpu(), "t": pp.t.cpu(), "s": pp.s.cpu()},
|
| 232 |
+
"model_cfg": model_cfg,
|
| 233 |
+
"ckpt_root": str(ckpt_root) if ckpt_root is not None else None,
|
| 234 |
+
"ckpt_name": ckpt_name,
|
| 235 |
+
"hf_variant": hf_variant,
|
| 236 |
+
"scene_meta": {
|
| 237 |
+
"ds_name": batch.get("ds_names", ["unknown"])[0],
|
| 238 |
+
"n_input_views": image_shape[1],
|
| 239 |
+
"input_res": (int(image_shape[-2]), int(image_shape[-1])),
|
| 240 |
+
"input_c2w_r": c2w_r[0].cpu(),
|
| 241 |
+
"input_c2w_t": c2w_t[0].cpu(),
|
| 242 |
+
"input_intr": intr[0].cpu(),
|
| 243 |
+
},
|
| 244 |
+
}
|
| 245 |
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
| 246 |
+
torch.save(artifact, out_path)
|
| 247 |
+
sz = out_path.stat().st_size / (1024 ** 2)
|
| 248 |
+
print(f"[recon] saved {out_path} ({sz:.1f} MB)")
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
def main():
|
| 252 |
+
p = argparse.ArgumentParser()
|
| 253 |
+
p.add_argument("--ckpt_root", type=Path, default=None)
|
| 254 |
+
p.add_argument("--hf", dest="hf_variant", default=None,
|
| 255 |
+
help="HuggingFace variant, e.g. DVSM-TTB-p8-DL3DV-960p")
|
| 256 |
+
p.add_argument("--ckpt_name", default="latest.pt")
|
| 257 |
+
p.add_argument("--input", type=Path, required=True,
|
| 258 |
+
help="Scene dir / transforms.json (COLMAP/nerfstudio), a flat image folder, "
|
| 259 |
+
"or a video file (image folder / video -> ΟΒ³).")
|
| 260 |
+
p.add_argument("--pose", default="auto", choices=("auto", "colmap", "pi3"),
|
| 261 |
+
help="Pose source. auto = COLMAP/transforms if present, else ΟΒ³.")
|
| 262 |
+
p.add_argument("--long_edge", type=int, default=960,
|
| 263 |
+
help="Default recon size: keep input aspect, resize long edge to this "
|
| 264 |
+
"(snapped to patch_size). Ignored if --target_h/--target_w are both set.")
|
| 265 |
+
p.add_argument("--target_h", type=int, default=None,
|
| 266 |
+
help="Override recon height (else derived from aspect + --long_edge).")
|
| 267 |
+
p.add_argument("--target_w", type=int, default=None,
|
| 268 |
+
help="Override recon width (else derived from aspect + --long_edge).")
|
| 269 |
+
p.add_argument("--max_views", type=int, default=None,
|
| 270 |
+
help="Evenly sub-sample this many views (after --stride).")
|
| 271 |
+
p.add_argument("--stride", type=int, default=1,
|
| 272 |
+
help="Sample every Nth frame BEFORE max_views subsampling.")
|
| 273 |
+
p.add_argument("--out", type=Path, required=True)
|
| 274 |
+
# COLMAP / transforms.json options
|
| 275 |
+
p.add_argument("--convention", default="opengl", choices=("opengl", "opencv"),
|
| 276 |
+
help="transforms.json camera-local convention (ignored for COLMAP).")
|
| 277 |
+
p.add_argument("--images_subdir", default=None,
|
| 278 |
+
help="COLMAP: image sub-folder (e.g. 'images_4'); intrinsics are pre-scaled.")
|
| 279 |
+
# ΟΒ³ options
|
| 280 |
+
p.add_argument("--pi3_flavor", default="pi3x", choices=("pi3", "pi3x"))
|
| 281 |
+
p.add_argument("--pi3_image_size", type=int, default=518)
|
| 282 |
+
args, cmd_lst = p.parse_known_args()
|
| 283 |
+
|
| 284 |
+
if args.ckpt_root is None and args.hf_variant is None:
|
| 285 |
+
p.error("Provide either --ckpt_root or --hf <variant>")
|
| 286 |
+
if args.ckpt_root is not None and args.hf_variant is not None:
|
| 287 |
+
p.error("--ckpt_root and --hf are mutually exclusive")
|
| 288 |
+
|
| 289 |
+
pose = detect_pose_source(args.input) if args.pose == "auto" else args.pose
|
| 290 |
+
if args.pose == "auto":
|
| 291 |
+
print(f"[recon] auto-detected pose source: {pose}", flush=True)
|
| 292 |
+
|
| 293 |
+
scene_data = load_scene_data(args, pose)
|
| 294 |
+
renderer, batch, image_shape, c2w_r, c2w_t, intr = reconstruct(
|
| 295 |
+
args.ckpt_root, args.ckpt_name, scene_data, cmd_lst,
|
| 296 |
+
hf_variant=args.hf_variant)
|
| 297 |
+
save_artifact(renderer, batch, image_shape, c2w_r, c2w_t, intr,
|
| 298 |
+
args.out, ckpt_root=args.ckpt_root, ckpt_name=args.ckpt_name,
|
| 299 |
+
hf_variant=args.hf_variant)
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
if __name__ == "__main__":
|
| 303 |
+
main()
|
viser_app/render.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pure-DVSM render utilities (no viser dependency).
|
| 2 |
+
|
| 3 |
+
Splits the model-loading and NeuralRenderer reconstruction code out of
|
| 4 |
+
viewer.py so that headless tests and offline rendering scripts can use it
|
| 5 |
+
without pulling in viser.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
|
| 13 |
+
from dvsm.config import cfg, update_config
|
| 14 |
+
from dvsm.models.DVSM import DecoderViewSynthesisModel, NeuralRenderer
|
| 15 |
+
from dvsm.models.pose_preprocessor import PosePreprocessor
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
# Rotates DVSM normalized world (X right, Y down, Z forward; OpenCV) into
|
| 19 |
+
# viser/three.js native world (X right, Y up, Z toward viewer; OpenGL). Self-
|
| 20 |
+
# inverse since it is diag(1, -1, -1) (an involution).
|
| 21 |
+
WORLD_DVSM_TO_VISER = torch.tensor(
|
| 22 |
+
[[1.0, 0.0, 0.0], [0.0, -1.0, 0.0], [0.0, 0.0, -1.0]], dtype=torch.float32,
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def apply_pp_single(pp: PosePreprocessor, c2w_r, c2w_t, apply_scale: bool = False):
|
| 27 |
+
"""DVSM raw pose -> viser world pose: rotate + translate via pp, then Rw.
|
| 28 |
+
|
| 29 |
+
By default we OMIT pp.s. Reason: pp.s shrinks the scene to a unit sphere,
|
| 30 |
+
which makes viser's orbit/pan/zoom feel too fast (controls are tuned for
|
| 31 |
+
a "natural" metric scale). Keeping the original metric for viser and
|
| 32 |
+
applying pp.s only inside the renderer gives natural control feel.
|
| 33 |
+
|
| 34 |
+
c2w_r_n = pp.r @ c2w_r
|
| 35 |
+
c2w_t_n = pp.r @ c2w_t + pp.t (pp.s applied iff apply_scale)
|
| 36 |
+
c2w_r_viser = Rw @ c2w_r_n
|
| 37 |
+
c2w_t_viser = Rw @ c2w_t_n
|
| 38 |
+
"""
|
| 39 |
+
r = pp.r[0] # [3, 3]
|
| 40 |
+
t = pp.t[0, 0] # [3]
|
| 41 |
+
r_n = r @ c2w_r
|
| 42 |
+
t_n = r @ c2w_t + t
|
| 43 |
+
if apply_scale:
|
| 44 |
+
s = float(pp.s.reshape(-1)[0])
|
| 45 |
+
t_n = t_n * s
|
| 46 |
+
Rw = WORLD_DVSM_TO_VISER.to(r_n.dtype)
|
| 47 |
+
r_v = Rw @ r_n
|
| 48 |
+
t_v = Rw @ t_n
|
| 49 |
+
return r_v, t_v
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def load_model(ckpt_root: Path | None = None, ckpt_name: str = "latest.pt",
|
| 53 |
+
cmd_lst: list[str] | None = None,
|
| 54 |
+
hf_variant: str | None = None) -> DecoderViewSynthesisModel:
|
| 55 |
+
if hf_variant is not None:
|
| 56 |
+
model = DecoderViewSynthesisModel.from_hf_pretrained(hf_variant)
|
| 57 |
+
return model.cuda().eval()
|
| 58 |
+
assert ckpt_root is not None, "Provide either ckpt_root or hf_variant"
|
| 59 |
+
update_config([ckpt_root / "config.yaml"], cmd_lst or [])
|
| 60 |
+
cfg.model.use_compile = True # try compile + CUDA Graph together
|
| 61 |
+
ckpt_path = ckpt_root / ckpt_name
|
| 62 |
+
assert ckpt_path.exists(), ckpt_path
|
| 63 |
+
model = DecoderViewSynthesisModel(**cfg.model).cuda().eval()
|
| 64 |
+
ckpt = torch.load(ckpt_path, weights_only=False, map_location="cpu")
|
| 65 |
+
model.load_state_dict(ckpt["state_dict"])
|
| 66 |
+
del ckpt
|
| 67 |
+
return model
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def _kv_to_cuda(kv):
|
| 71 |
+
if kv is None:
|
| 72 |
+
return None
|
| 73 |
+
if isinstance(kv, dict):
|
| 74 |
+
return {k: (v.cuda() if isinstance(v, torch.Tensor) else v) for k, v in kv.items()}
|
| 75 |
+
if isinstance(kv, (tuple, list)):
|
| 76 |
+
return type(kv)(_kv_to_cuda(v) for v in kv)
|
| 77 |
+
if isinstance(kv, torch.Tensor):
|
| 78 |
+
return kv.cuda()
|
| 79 |
+
return kv
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def build_renderer(model: DecoderViewSynthesisModel, artifact: dict) -> NeuralRenderer:
|
| 83 |
+
"""Re-materialize a NeuralRenderer from an artifact saved by recon_scene.py."""
|
| 84 |
+
pp = PosePreprocessor.__new__(PosePreprocessor)
|
| 85 |
+
pp.r = artifact["pose_preproc"]["r"].cuda()
|
| 86 |
+
pp.t = artifact["pose_preproc"]["t"].cuda()
|
| 87 |
+
pp.s = artifact["pose_preproc"]["s"].cuda()
|
| 88 |
+
kv_lst = [_kv_to_cuda(kv) for kv in artifact["kv_lst"]]
|
| 89 |
+
# NeuralRenderer no longer accepts chunk_centroids / chunk_K (k-means
|
| 90 |
+
# routing is now baked into the kv_lst itself). We silently drop them from
|
| 91 |
+
# older artifacts.
|
| 92 |
+
return NeuralRenderer(
|
| 93 |
+
pose_preprocessor=pp,
|
| 94 |
+
model=model,
|
| 95 |
+
kv_lst=kv_lst,
|
| 96 |
+
)
|
viser_app/scene_loaders.py
ADDED
|
@@ -0,0 +1,622 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""External-scene loaders for the viser viewer.
|
| 2 |
+
|
| 3 |
+
Supports two input formats (auto-detected):
|
| 4 |
+
|
| 5 |
+
1. **COLMAP text**: a directory containing ``sparse/0/cameras.txt`` and
|
| 6 |
+
``sparse/0/images.txt`` plus an ``images/`` folder. Standard COLMAP output.
|
| 7 |
+
Convention: ``qvec``+``tvec`` are world-to-camera in OpenCV convention.
|
| 8 |
+
|
| 9 |
+
2. **nerfstudio transforms.json**: a JSON file with a ``frames`` array, each
|
| 10 |
+
frame having ``file_path`` and ``transform_matrix``. Camera intrinsics may
|
| 11 |
+
be at the JSON root or per-frame. Convention: ``transform_matrix`` is
|
| 12 |
+
camera-to-world in **OpenGL** convention (Y up, Z back); we flip Y and Z
|
| 13 |
+
columns to convert to DVSM's OpenCV convention.
|
| 14 |
+
|
| 15 |
+
The loaders return tensors shaped exactly like ``PresampledTestingDataset``
|
| 16 |
+
emits, so ``recon_scene.py`` can reuse the same downstream code.
|
| 17 |
+
"""
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import json
|
| 21 |
+
import re
|
| 22 |
+
import struct
|
| 23 |
+
from pathlib import Path
|
| 24 |
+
from typing import Optional
|
| 25 |
+
|
| 26 |
+
import numpy as np
|
| 27 |
+
import torch
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
# COLMAP binary model IDs and their (num_params, [(SIMPLE_)PINHOLE-equivalent param indices])
|
| 31 |
+
# https://colmap.github.io/format.html#camera-models
|
| 32 |
+
_COLMAP_MODEL_BY_ID = {
|
| 33 |
+
0: ("SIMPLE_PINHOLE", 3), # f, cx, cy
|
| 34 |
+
1: ("PINHOLE", 4), # fx, fy, cx, cy
|
| 35 |
+
2: ("SIMPLE_RADIAL", 4), # f, cx, cy, k
|
| 36 |
+
3: ("RADIAL", 5), # f, cx, cy, k1, k2
|
| 37 |
+
4: ("OPENCV", 8), # fx, fy, cx, cy, k1, k2, p1, p2
|
| 38 |
+
5: ("OPENCV_FISHEYE", 8),
|
| 39 |
+
6: ("FULL_OPENCV", 12),
|
| 40 |
+
7: ("FOV", 5),
|
| 41 |
+
8: ("SIMPLE_RADIAL_FISHEYE", 4),
|
| 42 |
+
9: ("RADIAL_FISHEYE", 5),
|
| 43 |
+
10: ("THIN_PRISM_FISHEYE", 12),
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
# ---------------------------------------------------------------------------
|
| 48 |
+
# COLMAP text parser
|
| 49 |
+
# ---------------------------------------------------------------------------
|
| 50 |
+
|
| 51 |
+
def _parse_colmap_cameras_txt(path: Path) -> dict[int, dict]:
|
| 52 |
+
"""Returns {camera_id: {model, width, height, fx, fy, cx, cy}}.
|
| 53 |
+
|
| 54 |
+
Supports SIMPLE_PINHOLE / PINHOLE / SIMPLE_RADIAL / RADIAL / OPENCV.
|
| 55 |
+
Distortion coefficients are ignored.
|
| 56 |
+
"""
|
| 57 |
+
cameras = {}
|
| 58 |
+
for line in path.read_text().splitlines():
|
| 59 |
+
line = line.strip()
|
| 60 |
+
if not line or line.startswith("#"):
|
| 61 |
+
continue
|
| 62 |
+
parts = line.split()
|
| 63 |
+
cam_id = int(parts[0])
|
| 64 |
+
model = parts[1]
|
| 65 |
+
width = int(parts[2])
|
| 66 |
+
height = int(parts[3])
|
| 67 |
+
params = [float(x) for x in parts[4:]]
|
| 68 |
+
if model == "SIMPLE_PINHOLE":
|
| 69 |
+
f, cx, cy = params[:3]
|
| 70 |
+
fx = fy = f
|
| 71 |
+
elif model in ("PINHOLE", "OPENCV", "FULL_OPENCV", "OPENCV_FISHEYE"):
|
| 72 |
+
fx, fy, cx, cy = params[:4]
|
| 73 |
+
elif model in ("SIMPLE_RADIAL", "RADIAL"):
|
| 74 |
+
f, cx, cy = params[:3]
|
| 75 |
+
fx = fy = f
|
| 76 |
+
else:
|
| 77 |
+
raise ValueError(
|
| 78 |
+
f"COLMAP camera model {model!r} not supported (camera_id={cam_id})"
|
| 79 |
+
)
|
| 80 |
+
cameras[cam_id] = dict(
|
| 81 |
+
model=model, width=width, height=height,
|
| 82 |
+
fx=fx, fy=fy, cx=cx, cy=cy,
|
| 83 |
+
)
|
| 84 |
+
return cameras
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _qvec_to_rotmat(qvec: np.ndarray) -> np.ndarray:
|
| 88 |
+
"""COLMAP-convention quaternion (w, x, y, z) -> 3x3 rotation matrix."""
|
| 89 |
+
qw, qx, qy, qz = qvec
|
| 90 |
+
return np.array([
|
| 91 |
+
[1 - 2 * (qy * qy + qz * qz), 2 * (qx * qy - qw * qz), 2 * (qx * qz + qw * qy)],
|
| 92 |
+
[ 2 * (qx * qy + qw * qz), 1 - 2 * (qx * qx + qz * qz), 2 * (qy * qz - qw * qx)],
|
| 93 |
+
[ 2 * (qx * qz - qw * qy), 2 * (qy * qz + qw * qx), 1 - 2 * (qx * qx + qy * qy)],
|
| 94 |
+
], dtype=np.float64)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def _parse_colmap_images_txt(path: Path) -> list[dict]:
|
| 98 |
+
"""Returns list of {image_id, name, camera_id, c2w_r [3,3], c2w_t [3]}.
|
| 99 |
+
|
| 100 |
+
Each image in images.txt occupies TWO lines; we only care about the first.
|
| 101 |
+
qvec+tvec are w2c (OpenCV); we convert to c2w.
|
| 102 |
+
"""
|
| 103 |
+
entries = []
|
| 104 |
+
lines = path.read_text().splitlines()
|
| 105 |
+
i = 0
|
| 106 |
+
while i < len(lines):
|
| 107 |
+
line = lines[i].strip()
|
| 108 |
+
if not line or line.startswith("#"):
|
| 109 |
+
i += 1
|
| 110 |
+
continue
|
| 111 |
+
# First line: IMAGE_ID QW QX QY QZ TX TY TZ CAMERA_ID NAME
|
| 112 |
+
parts = line.split()
|
| 113 |
+
image_id = int(parts[0])
|
| 114 |
+
qvec = np.array([float(x) for x in parts[1:5]]) # w, x, y, z
|
| 115 |
+
tvec = np.array([float(x) for x in parts[5:8]])
|
| 116 |
+
cam_id = int(parts[8])
|
| 117 |
+
name = " ".join(parts[9:])
|
| 118 |
+
# Second line: 2D feature observations β skip
|
| 119 |
+
i += 2
|
| 120 |
+
|
| 121 |
+
w2c_r = _qvec_to_rotmat(qvec)
|
| 122 |
+
w2c_t = tvec
|
| 123 |
+
c2w_r = w2c_r.T
|
| 124 |
+
c2w_t = -c2w_r @ w2c_t
|
| 125 |
+
entries.append(dict(
|
| 126 |
+
image_id=image_id, name=name, camera_id=cam_id,
|
| 127 |
+
c2w_r=c2w_r.astype(np.float32),
|
| 128 |
+
c2w_t=c2w_t.astype(np.float32),
|
| 129 |
+
))
|
| 130 |
+
return entries
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def _parse_colmap_cameras_bin(path: Path) -> dict[int, dict]:
|
| 134 |
+
"""Binary cameras.bin parser (Mip-NeRF 360 et al. ship binary)."""
|
| 135 |
+
out = {}
|
| 136 |
+
with open(path, "rb") as f:
|
| 137 |
+
(n_cams,) = struct.unpack("<Q", f.read(8))
|
| 138 |
+
for _ in range(n_cams):
|
| 139 |
+
cam_id, model_id, w, h = struct.unpack("<IiQQ", f.read(24))
|
| 140 |
+
if model_id not in _COLMAP_MODEL_BY_ID:
|
| 141 |
+
raise ValueError(f"unknown COLMAP model id {model_id}")
|
| 142 |
+
model_name, n_params = _COLMAP_MODEL_BY_ID[model_id]
|
| 143 |
+
params = struct.unpack(f"<{n_params}d", f.read(8 * n_params))
|
| 144 |
+
if model_name == "SIMPLE_PINHOLE":
|
| 145 |
+
fc, cx, cy = params[:3]
|
| 146 |
+
fx = fy = fc
|
| 147 |
+
elif model_name in ("PINHOLE", "OPENCV", "FULL_OPENCV", "OPENCV_FISHEYE",
|
| 148 |
+
"THIN_PRISM_FISHEYE"):
|
| 149 |
+
fx, fy, cx, cy = params[:4]
|
| 150 |
+
elif model_name in ("SIMPLE_RADIAL", "RADIAL",
|
| 151 |
+
"SIMPLE_RADIAL_FISHEYE", "RADIAL_FISHEYE"):
|
| 152 |
+
fc, cx, cy = params[:3]
|
| 153 |
+
fx = fy = fc
|
| 154 |
+
elif model_name == "FOV":
|
| 155 |
+
fc, cx, cy = params[:3]
|
| 156 |
+
fx = fy = fc
|
| 157 |
+
else:
|
| 158 |
+
raise ValueError(f"unhandled model {model_name}")
|
| 159 |
+
out[cam_id] = dict(
|
| 160 |
+
model=model_name, width=int(w), height=int(h),
|
| 161 |
+
fx=float(fx), fy=float(fy), cx=float(cx), cy=float(cy),
|
| 162 |
+
)
|
| 163 |
+
return out
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def _parse_colmap_images_bin(path: Path) -> list[dict]:
|
| 167 |
+
"""Binary images.bin parser."""
|
| 168 |
+
entries = []
|
| 169 |
+
with open(path, "rb") as f:
|
| 170 |
+
(n_imgs,) = struct.unpack("<Q", f.read(8))
|
| 171 |
+
for _ in range(n_imgs):
|
| 172 |
+
image_id, qw, qx, qy, qz, tx, ty, tz, cam_id = struct.unpack(
|
| 173 |
+
"<IdddddddI", f.read(4 + 8 * 7 + 4),
|
| 174 |
+
)
|
| 175 |
+
# Null-terminated name
|
| 176 |
+
name_bytes = bytearray()
|
| 177 |
+
while True:
|
| 178 |
+
c = f.read(1)
|
| 179 |
+
if not c or c == b"\x00":
|
| 180 |
+
break
|
| 181 |
+
name_bytes += c
|
| 182 |
+
name = name_bytes.decode("utf-8")
|
| 183 |
+
# Skip the 2D points block: uint64 num_points then 24 bytes per point.
|
| 184 |
+
(n_pts,) = struct.unpack("<Q", f.read(8))
|
| 185 |
+
f.read(24 * n_pts)
|
| 186 |
+
|
| 187 |
+
w2c_r = _qvec_to_rotmat(np.array([qw, qx, qy, qz]))
|
| 188 |
+
w2c_t = np.array([tx, ty, tz])
|
| 189 |
+
c2w_r = w2c_r.T
|
| 190 |
+
c2w_t = -c2w_r @ w2c_t
|
| 191 |
+
entries.append(dict(
|
| 192 |
+
image_id=image_id, name=name, camera_id=cam_id,
|
| 193 |
+
c2w_r=c2w_r.astype(np.float32),
|
| 194 |
+
c2w_t=c2w_t.astype(np.float32),
|
| 195 |
+
))
|
| 196 |
+
return entries
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def _find_colmap_dir(scene_dir: Path) -> tuple[Path, Path, str]:
|
| 200 |
+
"""Locate cameras + images files (binary or text) inside a scene dir.
|
| 201 |
+
|
| 202 |
+
Returns (cams_path, imgs_path, format) where format β {"bin", "txt"}.
|
| 203 |
+
"""
|
| 204 |
+
candidates = [
|
| 205 |
+
scene_dir / "sparse" / "0",
|
| 206 |
+
scene_dir / "sparse",
|
| 207 |
+
scene_dir / "colmap" / "sparse" / "0",
|
| 208 |
+
scene_dir,
|
| 209 |
+
]
|
| 210 |
+
for d in candidates:
|
| 211 |
+
for ext, fmt in (("bin", "bin"), ("txt", "txt")):
|
| 212 |
+
cams = d / f"cameras.{ext}"
|
| 213 |
+
imgs = d / f"images.{ext}"
|
| 214 |
+
if cams.exists() and imgs.exists():
|
| 215 |
+
return cams, imgs, fmt
|
| 216 |
+
raise FileNotFoundError(
|
| 217 |
+
f"could not find cameras.{{bin,txt}} + images.{{bin,txt}} under {scene_dir}"
|
| 218 |
+
)
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def _find_images_dir(scene_dir: Path, subdir: Optional[str] = None) -> Path:
|
| 222 |
+
if subdir is not None:
|
| 223 |
+
d = scene_dir / subdir
|
| 224 |
+
if d.is_dir():
|
| 225 |
+
return d
|
| 226 |
+
raise FileNotFoundError(f"{d} does not exist")
|
| 227 |
+
for sub in ("images", "image", "imgs"):
|
| 228 |
+
d = scene_dir / sub
|
| 229 |
+
if d.is_dir():
|
| 230 |
+
return d
|
| 231 |
+
raise FileNotFoundError(f"no images/ directory under {scene_dir}")
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
# ---------------------------------------------------------------------------
|
| 235 |
+
# Image loading + resize/crop with intrinsics adjustment
|
| 236 |
+
# ---------------------------------------------------------------------------
|
| 237 |
+
|
| 238 |
+
def _resize_and_crop(
|
| 239 |
+
img: np.ndarray, intr_px: dict, target_h: int, target_w: int,
|
| 240 |
+
) -> tuple[np.ndarray, dict]:
|
| 241 |
+
"""Aspect-preserving resize then center-crop to (target_h, target_w).
|
| 242 |
+
Returns the cropped image and the adjusted pixel intrinsics."""
|
| 243 |
+
from PIL import Image
|
| 244 |
+
|
| 245 |
+
src_h, src_w = img.shape[:2]
|
| 246 |
+
scale = max(target_w / src_w, target_h / src_h)
|
| 247 |
+
new_w = int(round(src_w * scale))
|
| 248 |
+
new_h = int(round(src_h * scale))
|
| 249 |
+
|
| 250 |
+
img_pil = Image.fromarray(img).resize((new_w, new_h), Image.LANCZOS)
|
| 251 |
+
crop_l = (new_w - target_w) // 2
|
| 252 |
+
crop_t = (new_h - target_h) // 2
|
| 253 |
+
img_pil = img_pil.crop((crop_l, crop_t, crop_l + target_w, crop_t + target_h))
|
| 254 |
+
|
| 255 |
+
intr_new = dict(
|
| 256 |
+
fx=intr_px["fx"] * scale,
|
| 257 |
+
fy=intr_px["fy"] * scale,
|
| 258 |
+
cx=intr_px["cx"] * scale - crop_l,
|
| 259 |
+
cy=intr_px["cy"] * scale - crop_t,
|
| 260 |
+
)
|
| 261 |
+
return np.asarray(img_pil), intr_new
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
def pack_estimator_scene_data(est_result: dict, target_h: int, target_w: int,
|
| 265 |
+
scene_name: str, source: str) -> dict:
|
| 266 |
+
"""Generic packer: take a pose-estimator result dict and produce the
|
| 267 |
+
scene_data shape consumed by build_renderer.
|
| 268 |
+
|
| 269 |
+
`est_result` must contain:
|
| 270 |
+
raw_images : list[HxWx3 uint8 ndarray] in native resolution
|
| 271 |
+
c2w : (V, 4, 4) camera-to-world (OpenCV)
|
| 272 |
+
intr : (fx, fy, cx, cy) in pixels, valid at `image_size`
|
| 273 |
+
image_size : (H, W) of the input fed to the estimator
|
| 274 |
+
"""
|
| 275 |
+
raw_imgs = est_result["raw_images"]
|
| 276 |
+
c2w = est_result["c2w"]
|
| 277 |
+
fx, fy, cx, cy = est_result["intr"]
|
| 278 |
+
h_src, w_src = est_result["image_size"]
|
| 279 |
+
V = len(raw_imgs)
|
| 280 |
+
|
| 281 |
+
img_arr = np.zeros((V, target_h, target_w, 3), dtype=np.uint8)
|
| 282 |
+
intr_arr = np.zeros((V, 4), dtype=np.float32)
|
| 283 |
+
c2w_r_arr = np.zeros((V, 3, 3), dtype=np.float32)
|
| 284 |
+
c2w_t_arr = np.zeros((V, 3), dtype=np.float32)
|
| 285 |
+
|
| 286 |
+
for i, raw in enumerate(raw_imgs):
|
| 287 |
+
# Estimator's intrinsics live at h_src Γ w_src. The native image is
|
| 288 |
+
# bigger (or differently sized), so scale to native size first, then
|
| 289 |
+
# let _resize_and_crop scale them again to the DVSM target.
|
| 290 |
+
native_h, native_w = raw.shape[:2]
|
| 291 |
+
sx = native_w / w_src
|
| 292 |
+
sy = native_h / h_src
|
| 293 |
+
intr_native = dict(
|
| 294 |
+
fx=fx * sx, fy=fy * sy,
|
| 295 |
+
cx=cx * sx, cy=cy * sy,
|
| 296 |
+
)
|
| 297 |
+
img_resized, intr_new = _resize_and_crop(raw, intr_native, target_h, target_w)
|
| 298 |
+
img_arr[i] = img_resized
|
| 299 |
+
intr_arr[i] = (
|
| 300 |
+
intr_new["fx"] / target_w, intr_new["fy"] / target_h,
|
| 301 |
+
intr_new["cx"] / target_w, intr_new["cy"] / target_h,
|
| 302 |
+
)
|
| 303 |
+
c2w_r_arr[i] = c2w[i, :3, :3]
|
| 304 |
+
c2w_t_arr[i] = c2w[i, :3, 3]
|
| 305 |
+
|
| 306 |
+
image_t = torch.from_numpy(img_arr).permute(0, 3, 1, 2).float() / 255.0
|
| 307 |
+
return dict(
|
| 308 |
+
image=image_t,
|
| 309 |
+
intr=torch.from_numpy(intr_arr),
|
| 310 |
+
c2w_r=torch.from_numpy(c2w_r_arr),
|
| 311 |
+
c2w_t=torch.from_numpy(c2w_t_arr),
|
| 312 |
+
H=target_h, W=target_w,
|
| 313 |
+
scene_name=scene_name,
|
| 314 |
+
source=source,
|
| 315 |
+
n_input_views=V,
|
| 316 |
+
)
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
# ---------------------------------------------------------------------------
|
| 320 |
+
# COLMAP scene loader
|
| 321 |
+
# ---------------------------------------------------------------------------
|
| 322 |
+
|
| 323 |
+
def load_colmap_scene(
|
| 324 |
+
scene_dir: Path,
|
| 325 |
+
target_h: int,
|
| 326 |
+
target_w: int,
|
| 327 |
+
max_views: Optional[int] = None,
|
| 328 |
+
stride: int = 1,
|
| 329 |
+
sort_by_name: bool = True,
|
| 330 |
+
images_subdir: Optional[str] = None,
|
| 331 |
+
) -> dict:
|
| 332 |
+
"""Load a COLMAP scene.
|
| 333 |
+
|
| 334 |
+
Args:
|
| 335 |
+
images_subdir: e.g. "images_4" to use Mip-NeRF 360 downscaled images.
|
| 336 |
+
cameras.bin intrinsics are pre-scaled to match the actual image
|
| 337 |
+
resolution we end up reading from disk.
|
| 338 |
+
"""
|
| 339 |
+
import imageio.v3 as iio
|
| 340 |
+
|
| 341 |
+
cams_path, imgs_path, fmt = _find_colmap_dir(scene_dir)
|
| 342 |
+
images_dir = _find_images_dir(scene_dir, subdir=images_subdir)
|
| 343 |
+
|
| 344 |
+
print(f"[colmap] format: {fmt}")
|
| 345 |
+
print(f"[colmap] cameras: {cams_path}")
|
| 346 |
+
print(f"[colmap] images: {imgs_path}")
|
| 347 |
+
print(f"[colmap] images dir: {images_dir}")
|
| 348 |
+
|
| 349 |
+
if fmt == "bin":
|
| 350 |
+
cameras = _parse_colmap_cameras_bin(cams_path)
|
| 351 |
+
image_entries = _parse_colmap_images_bin(imgs_path)
|
| 352 |
+
else:
|
| 353 |
+
cameras = _parse_colmap_cameras_txt(cams_path)
|
| 354 |
+
image_entries = _parse_colmap_images_txt(imgs_path)
|
| 355 |
+
if sort_by_name:
|
| 356 |
+
image_entries.sort(key=lambda e: _natural_key(e["name"]))
|
| 357 |
+
image_entries = image_entries[::stride]
|
| 358 |
+
if max_views is not None:
|
| 359 |
+
# Evenly subsample to N views.
|
| 360 |
+
if len(image_entries) > max_views:
|
| 361 |
+
idxs = np.linspace(0, len(image_entries) - 1, max_views).round().astype(int)
|
| 362 |
+
image_entries = [image_entries[i] for i in idxs]
|
| 363 |
+
V = len(image_entries)
|
| 364 |
+
print(f"[colmap] loading {V} views at {target_h}x{target_w} ...")
|
| 365 |
+
|
| 366 |
+
img_arr = np.zeros((V, target_h, target_w, 3), dtype=np.uint8)
|
| 367 |
+
intr_arr = np.zeros((V, 4), dtype=np.float32)
|
| 368 |
+
c2w_r_arr = np.zeros((V, 3, 3), dtype=np.float32)
|
| 369 |
+
c2w_t_arr = np.zeros((V, 3), dtype=np.float32)
|
| 370 |
+
|
| 371 |
+
for i, entry in enumerate(image_entries):
|
| 372 |
+
cam = cameras[entry["camera_id"]]
|
| 373 |
+
img_path = images_dir / entry["name"]
|
| 374 |
+
if not img_path.exists():
|
| 375 |
+
raise FileNotFoundError(img_path)
|
| 376 |
+
img = iio.imread(img_path)
|
| 377 |
+
if img.ndim == 2:
|
| 378 |
+
img = np.stack([img, img, img], axis=-1)
|
| 379 |
+
if img.shape[-1] == 4:
|
| 380 |
+
img = img[..., :3]
|
| 381 |
+
# Pre-scale intrinsics if we're reading a downscaled image variant
|
| 382 |
+
# (e.g. Mip-NeRF 360's images_2/_4/_8 vs the native cameras.bin width/height).
|
| 383 |
+
img_h, img_w = img.shape[:2]
|
| 384 |
+
if img_w != cam["width"] or img_h != cam["height"]:
|
| 385 |
+
sx = img_w / cam["width"]
|
| 386 |
+
sy = img_h / cam["height"]
|
| 387 |
+
intr_px = dict(fx=cam["fx"] * sx, fy=cam["fy"] * sy,
|
| 388 |
+
cx=cam["cx"] * sx, cy=cam["cy"] * sy)
|
| 389 |
+
else:
|
| 390 |
+
intr_px = dict(fx=cam["fx"], fy=cam["fy"], cx=cam["cx"], cy=cam["cy"])
|
| 391 |
+
img_resized, intr_new = _resize_and_crop(img, intr_px, target_h, target_w)
|
| 392 |
+
img_arr[i] = img_resized
|
| 393 |
+
intr_arr[i] = (
|
| 394 |
+
intr_new["fx"] / target_w,
|
| 395 |
+
intr_new["fy"] / target_h,
|
| 396 |
+
intr_new["cx"] / target_w,
|
| 397 |
+
intr_new["cy"] / target_h,
|
| 398 |
+
)
|
| 399 |
+
c2w_r_arr[i] = entry["c2w_r"]
|
| 400 |
+
c2w_t_arr[i] = entry["c2w_t"]
|
| 401 |
+
|
| 402 |
+
image_t = torch.from_numpy(img_arr).permute(0, 3, 1, 2).float() / 255.0
|
| 403 |
+
return dict(
|
| 404 |
+
image=image_t,
|
| 405 |
+
intr=torch.from_numpy(intr_arr),
|
| 406 |
+
c2w_r=torch.from_numpy(c2w_r_arr),
|
| 407 |
+
c2w_t=torch.from_numpy(c2w_t_arr),
|
| 408 |
+
H=target_h, W=target_w,
|
| 409 |
+
scene_name=scene_dir.name,
|
| 410 |
+
source="colmap_text",
|
| 411 |
+
n_input_views=V,
|
| 412 |
+
)
|
| 413 |
+
|
| 414 |
+
|
| 415 |
+
# ---------------------------------------------------------------------------
|
| 416 |
+
# nerfstudio transforms.json loader
|
| 417 |
+
# ---------------------------------------------------------------------------
|
| 418 |
+
|
| 419 |
+
# OpenGL β OpenCV: flip Y and Z columns of the camera-to-world rotation.
|
| 420 |
+
_GL_TO_CV = np.diag([1.0, -1.0, -1.0]).astype(np.float32)
|
| 421 |
+
|
| 422 |
+
|
| 423 |
+
def load_transforms_json(
|
| 424 |
+
json_path: Path,
|
| 425 |
+
target_h: int,
|
| 426 |
+
target_w: int,
|
| 427 |
+
max_views: Optional[int] = None,
|
| 428 |
+
stride: int = 1,
|
| 429 |
+
convention: str = "opengl",
|
| 430 |
+
) -> dict:
|
| 431 |
+
"""Load a nerfstudio-style transforms.json.
|
| 432 |
+
|
| 433 |
+
Args:
|
| 434 |
+
convention: "opengl" (default β nerfstudio's convention; we flip to
|
| 435 |
+
OpenCV) or "opencv" (already OpenCV β no flip).
|
| 436 |
+
"""
|
| 437 |
+
import imageio.v3 as iio
|
| 438 |
+
|
| 439 |
+
json_path = Path(json_path)
|
| 440 |
+
with open(json_path) as f:
|
| 441 |
+
data = json.load(f)
|
| 442 |
+
base_dir = json_path.parent
|
| 443 |
+
|
| 444 |
+
frames = data["frames"][::stride]
|
| 445 |
+
if max_views is not None and len(frames) > max_views:
|
| 446 |
+
idxs = np.linspace(0, len(frames) - 1, max_views).round().astype(int)
|
| 447 |
+
frames = [frames[i] for i in idxs]
|
| 448 |
+
V = len(frames)
|
| 449 |
+
print(f"[transforms_json] {V} frames; convention={convention}; "
|
| 450 |
+
f"target {target_h}x{target_w}")
|
| 451 |
+
|
| 452 |
+
def _intr_for_frame(frame):
|
| 453 |
+
fx = frame.get("fl_x", data.get("fl_x"))
|
| 454 |
+
fy = frame.get("fl_y", data.get("fl_y", fx))
|
| 455 |
+
cx = frame.get("cx", data.get("cx"))
|
| 456 |
+
cy = frame.get("cy", data.get("cy"))
|
| 457 |
+
w = frame.get("w", data.get("w"))
|
| 458 |
+
h = frame.get("h", data.get("h"))
|
| 459 |
+
return fx, fy, cx, cy, w, h
|
| 460 |
+
|
| 461 |
+
img_arr = np.zeros((V, target_h, target_w, 3), dtype=np.uint8)
|
| 462 |
+
intr_arr = np.zeros((V, 4), dtype=np.float32)
|
| 463 |
+
c2w_r_arr = np.zeros((V, 3, 3), dtype=np.float32)
|
| 464 |
+
c2w_t_arr = np.zeros((V, 3), dtype=np.float32)
|
| 465 |
+
|
| 466 |
+
for i, fr in enumerate(frames):
|
| 467 |
+
img_path = base_dir / fr["file_path"]
|
| 468 |
+
if not img_path.exists():
|
| 469 |
+
# nerfstudio sometimes drops the file extension; try a few.
|
| 470 |
+
for ext in (".png", ".jpg", ".jpeg", ".JPG", ".PNG"):
|
| 471 |
+
p2 = img_path.with_suffix(ext)
|
| 472 |
+
if p2.exists():
|
| 473 |
+
img_path = p2
|
| 474 |
+
break
|
| 475 |
+
else:
|
| 476 |
+
raise FileNotFoundError(img_path)
|
| 477 |
+
img = iio.imread(img_path)
|
| 478 |
+
if img.ndim == 2:
|
| 479 |
+
img = np.stack([img, img, img], axis=-1)
|
| 480 |
+
if img.shape[-1] == 4:
|
| 481 |
+
img = img[..., :3]
|
| 482 |
+
fx, fy, cx, cy, w, h = _intr_for_frame(fr)
|
| 483 |
+
if w is None: w = img.shape[1]
|
| 484 |
+
if h is None: h = img.shape[0]
|
| 485 |
+
if cx is None: cx = w / 2.0
|
| 486 |
+
if cy is None: cy = h / 2.0
|
| 487 |
+
img_resized, intr_new = _resize_and_crop(
|
| 488 |
+
img, dict(fx=fx, fy=fy, cx=cx, cy=cy), target_h, target_w,
|
| 489 |
+
)
|
| 490 |
+
img_arr[i] = img_resized
|
| 491 |
+
intr_arr[i] = (
|
| 492 |
+
intr_new["fx"] / target_w, intr_new["fy"] / target_h,
|
| 493 |
+
intr_new["cx"] / target_w, intr_new["cy"] / target_h,
|
| 494 |
+
)
|
| 495 |
+
|
| 496 |
+
c2w = np.array(fr["transform_matrix"], dtype=np.float32)
|
| 497 |
+
c2w_r = c2w[:3, :3]
|
| 498 |
+
c2w_t = c2w[:3, 3]
|
| 499 |
+
if convention == "opengl":
|
| 500 |
+
c2w_r = c2w_r @ _GL_TO_CV # flip camera-local Y, Z
|
| 501 |
+
elif convention != "opencv":
|
| 502 |
+
raise ValueError(f"unknown convention: {convention}")
|
| 503 |
+
c2w_r_arr[i] = c2w_r
|
| 504 |
+
c2w_t_arr[i] = c2w_t
|
| 505 |
+
|
| 506 |
+
image_t = torch.from_numpy(img_arr).permute(0, 3, 1, 2).float() / 255.0
|
| 507 |
+
return dict(
|
| 508 |
+
image=image_t,
|
| 509 |
+
intr=torch.from_numpy(intr_arr),
|
| 510 |
+
c2w_r=torch.from_numpy(c2w_r_arr),
|
| 511 |
+
c2w_t=torch.from_numpy(c2w_t_arr),
|
| 512 |
+
H=target_h, W=target_w,
|
| 513 |
+
scene_name=json_path.parent.name,
|
| 514 |
+
source="transforms_json",
|
| 515 |
+
n_input_views=V,
|
| 516 |
+
)
|
| 517 |
+
|
| 518 |
+
|
| 519 |
+
# ---------------------------------------------------------------------------
|
| 520 |
+
# Dispatcher
|
| 521 |
+
# ---------------------------------------------------------------------------
|
| 522 |
+
|
| 523 |
+
def load_external_scene(
|
| 524 |
+
path: Path,
|
| 525 |
+
target_h: int,
|
| 526 |
+
target_w: int,
|
| 527 |
+
max_views: Optional[int] = None,
|
| 528 |
+
stride: int = 1,
|
| 529 |
+
convention: str = "opengl",
|
| 530 |
+
images_subdir: Optional[str] = None,
|
| 531 |
+
) -> dict:
|
| 532 |
+
"""Auto-detect format and load.
|
| 533 |
+
|
| 534 |
+
Acceptable ``path``:
|
| 535 |
+
- a directory with ``transforms.json`` inside β nerfstudio
|
| 536 |
+
- a directory with ``sparse/.../cameras.{bin,txt}`` inside β COLMAP
|
| 537 |
+
- a ``.json`` file β nerfstudio transforms.json
|
| 538 |
+
"""
|
| 539 |
+
path = Path(path)
|
| 540 |
+
if path.is_file() and path.suffix == ".json":
|
| 541 |
+
return load_transforms_json(
|
| 542 |
+
path, target_h, target_w, max_views=max_views, stride=stride,
|
| 543 |
+
convention=convention,
|
| 544 |
+
)
|
| 545 |
+
if path.is_dir():
|
| 546 |
+
if (path / "transforms.json").exists():
|
| 547 |
+
return load_transforms_json(
|
| 548 |
+
path / "transforms.json", target_h, target_w,
|
| 549 |
+
max_views=max_views, stride=stride, convention=convention,
|
| 550 |
+
)
|
| 551 |
+
# try COLMAP
|
| 552 |
+
try:
|
| 553 |
+
return load_colmap_scene(
|
| 554 |
+
path, target_h, target_w,
|
| 555 |
+
max_views=max_views, stride=stride,
|
| 556 |
+
images_subdir=images_subdir,
|
| 557 |
+
)
|
| 558 |
+
except FileNotFoundError as e:
|
| 559 |
+
raise FileNotFoundError(
|
| 560 |
+
f"{path} has neither transforms.json nor COLMAP sparse/*/cameras.{{bin,txt}}: {e}"
|
| 561 |
+
)
|
| 562 |
+
raise FileNotFoundError(path)
|
| 563 |
+
|
| 564 |
+
|
| 565 |
+
# ---------------------------------------------------------------------------
|
| 566 |
+
|
| 567 |
+
def _natural_key(s: str):
|
| 568 |
+
"""Sort by embedded integers naturally (e.g. img2 < img10)."""
|
| 569 |
+
return [int(t) if t.isdigit() else t for t in re.split(r"(\d+)", s)]
|
| 570 |
+
|
| 571 |
+
|
| 572 |
+
# ---------------------------------------------------------------------------
|
| 573 |
+
# Input file kinds + video frame extraction (shared by recon_scene.py / gradio)
|
| 574 |
+
# ---------------------------------------------------------------------------
|
| 575 |
+
VIDEO_EXTS = {".mp4", ".mov", ".mkv", ".webm", ".avi", ".m4v", ".flv", ".wmv"}
|
| 576 |
+
IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp"}
|
| 577 |
+
|
| 578 |
+
|
| 579 |
+
def extract_video_frames(video_path: Path, out_dir: Path, n_frames: int) -> int:
|
| 580 |
+
"""Sample ``n_frames`` evenly-spaced frames from ``video_path`` and write
|
| 581 |
+
them as JPEGs into ``out_dir``. Returns the number of frames actually
|
| 582 |
+
written. Tries cv2 first, falls back to imageio.
|
| 583 |
+
|
| 584 |
+
Frames keep the video's original aspect; the recon pipeline will resize.
|
| 585 |
+
"""
|
| 586 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 587 |
+
n_written = 0
|
| 588 |
+
|
| 589 |
+
# Try OpenCV (much faster on long videos due to keyframe seeking).
|
| 590 |
+
try:
|
| 591 |
+
import cv2
|
| 592 |
+
cap = cv2.VideoCapture(str(video_path))
|
| 593 |
+
total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
| 594 |
+
if total <= 0:
|
| 595 |
+
raise RuntimeError("cv2 saw 0 frames")
|
| 596 |
+
n_take = min(n_frames, total)
|
| 597 |
+
idxs = np.linspace(0, total - 1, n_take).round().astype(int)
|
| 598 |
+
for k, idx in enumerate(idxs):
|
| 599 |
+
cap.set(cv2.CAP_PROP_POS_FRAMES, int(idx))
|
| 600 |
+
ok, frame = cap.read()
|
| 601 |
+
if not ok or frame is None:
|
| 602 |
+
continue
|
| 603 |
+
cv2.imwrite(str(out_dir / f"frame_{k:04d}.jpg"),
|
| 604 |
+
frame, [int(cv2.IMWRITE_JPEG_QUALITY), 95])
|
| 605 |
+
n_written += 1
|
| 606 |
+
cap.release()
|
| 607 |
+
return n_written
|
| 608 |
+
except Exception as e:
|
| 609 |
+
print(f"[video] cv2 path failed ({e}); falling back to imageio")
|
| 610 |
+
|
| 611 |
+
# imageio fallback (works for most containers via PyAV).
|
| 612 |
+
import imageio.v3 as iio
|
| 613 |
+
frames = list(iio.imiter(str(video_path), plugin="pyav"))
|
| 614 |
+
total = len(frames)
|
| 615 |
+
if total == 0:
|
| 616 |
+
raise RuntimeError(f"imageio saw 0 frames in {video_path}")
|
| 617 |
+
n_take = min(n_frames, total)
|
| 618 |
+
idxs = np.linspace(0, total - 1, n_take).round().astype(int)
|
| 619 |
+
for k, idx in enumerate(idxs):
|
| 620 |
+
iio.imwrite(out_dir / f"frame_{k:04d}.jpg", frames[idx])
|
| 621 |
+
n_written += 1
|
| 622 |
+
return n_written
|
viser_app/viewer.py
ADDED
|
@@ -0,0 +1,620 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Stage B: viser server serving real-time NVS from a pre-computed recon artifact.
|
| 2 |
+
|
| 3 |
+
Loads the model checkpoint plus the .recon.pt artifact produced by
|
| 4 |
+
recon_scene.py, reconstructs the NeuralRenderer in memory (no input images
|
| 5 |
+
needed), and exposes it over a viser web client. Each connected client gets a
|
| 6 |
+
dedicated render thread that produces frames in response to camera updates,
|
| 7 |
+
with dynamic downscale while the user is dragging the camera.
|
| 8 |
+
|
| 9 |
+
Usage:
|
| 10 |
+
python -m viser_app.viewer \
|
| 11 |
+
--ckpt_root outputs/dl3dv/960p_ttb_dvsm_minus \
|
| 12 |
+
--recon outputs/recon/960p_ttb_dvsm_minus_scene0.recon.pt \
|
| 13 |
+
--port 8080
|
| 14 |
+
|
| 15 |
+
Open the printed URL in a browser, or SSH-forward the port:
|
| 16 |
+
ssh -L 8080:<jobnode>:8080 <login_node>
|
| 17 |
+
"""
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import argparse
|
| 21 |
+
import sys
|
| 22 |
+
import threading
|
| 23 |
+
import time
|
| 24 |
+
from pathlib import Path
|
| 25 |
+
|
| 26 |
+
import asyncio
|
| 27 |
+
import ssl
|
| 28 |
+
|
| 29 |
+
import numpy as np
|
| 30 |
+
import torch
|
| 31 |
+
import viser
|
| 32 |
+
import viser._tunnel as _viser_tunnel
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
# Replace viser's share-URL tunnel relay coroutine with a version that also
|
| 36 |
+
# catches ssl.SSLError during connection teardown. Without this, an
|
| 37 |
+
# "APPLICATION_DATA_AFTER_CLOSE_NOTIFY" raised while closing a relayed SSL
|
| 38 |
+
# stream propagates out and kills the tunnel's multiprocessing.Process, which
|
| 39 |
+
# permanently breaks the share URL. _simple_proxy is referenced by module name
|
| 40 |
+
# from _make_tunnel, so this monkey-patch is picked up on every relay call.
|
| 41 |
+
# Patch must run BEFORE request_share_url() spawns the child process; under
|
| 42 |
+
# Linux fork start method the child inherits the patched module.
|
| 43 |
+
async def _patched_simple_proxy(local_host, local_port, remote_host, remote_port, close_event):
|
| 44 |
+
ssl_context = ssl.create_default_context()
|
| 45 |
+
|
| 46 |
+
async def close_writer(w):
|
| 47 |
+
try:
|
| 48 |
+
if not w.is_closing():
|
| 49 |
+
w.close()
|
| 50 |
+
await w.wait_closed()
|
| 51 |
+
except (ConnectionError, ssl.SSLError, OSError):
|
| 52 |
+
pass
|
| 53 |
+
|
| 54 |
+
async def relay(r, w):
|
| 55 |
+
try:
|
| 56 |
+
while True:
|
| 57 |
+
data = await r.read(4096)
|
| 58 |
+
if len(data) == 0:
|
| 59 |
+
break
|
| 60 |
+
w.write(data)
|
| 61 |
+
await w.drain()
|
| 62 |
+
except Exception:
|
| 63 |
+
pass
|
| 64 |
+
finally:
|
| 65 |
+
await close_writer(w)
|
| 66 |
+
|
| 67 |
+
while True:
|
| 68 |
+
local_w = None
|
| 69 |
+
remote_w = None
|
| 70 |
+
try:
|
| 71 |
+
local_r, local_w = await asyncio.open_connection(local_host, local_port)
|
| 72 |
+
remote_r, remote_w = await asyncio.open_connection(
|
| 73 |
+
remote_host, remote_port, ssl=ssl_context,
|
| 74 |
+
)
|
| 75 |
+
await asyncio.wait(
|
| 76 |
+
[
|
| 77 |
+
asyncio.gather(
|
| 78 |
+
asyncio.create_task(relay(local_r, remote_w)),
|
| 79 |
+
asyncio.create_task(relay(remote_r, local_w)),
|
| 80 |
+
),
|
| 81 |
+
asyncio.create_task(close_event.wait()),
|
| 82 |
+
],
|
| 83 |
+
return_when=asyncio.FIRST_COMPLETED,
|
| 84 |
+
)
|
| 85 |
+
except Exception:
|
| 86 |
+
pass
|
| 87 |
+
finally:
|
| 88 |
+
if local_w is not None:
|
| 89 |
+
await close_writer(local_w)
|
| 90 |
+
if remote_w is not None:
|
| 91 |
+
await close_writer(remote_w)
|
| 92 |
+
if close_event.is_set():
|
| 93 |
+
break
|
| 94 |
+
|
| 95 |
+
_viser_tunnel._simple_proxy = _patched_simple_proxy
|
| 96 |
+
|
| 97 |
+
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
| 98 |
+
sys.path.insert(0, str(PROJECT_ROOT))
|
| 99 |
+
|
| 100 |
+
from dvsm.models.DVSM import DecoderViewSynthesisModel, NeuralRenderer
|
| 101 |
+
from dvsm.models.pose_preprocessor import PosePreprocessor
|
| 102 |
+
from viser_app.cam_convention import dvsm_cam_to_viser, viser_cam_to_dvsm
|
| 103 |
+
from viser_app.render import (
|
| 104 |
+
WORLD_DVSM_TO_VISER as _WORLD_DVSM_TO_VISER,
|
| 105 |
+
apply_pp_single as _apply_pp_single,
|
| 106 |
+
build_renderer,
|
| 107 |
+
load_model,
|
| 108 |
+
)
|
| 109 |
+
from viser_app.viewer_optim import install_optimizations
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
# -----------------------------------------------------------------------------
|
| 113 |
+
# Per-client render state + loop
|
| 114 |
+
# -----------------------------------------------------------------------------
|
| 115 |
+
|
| 116 |
+
class RenderState:
|
| 117 |
+
"""Thread-safe state shared between camera callback and render thread."""
|
| 118 |
+
|
| 119 |
+
def __init__(self):
|
| 120 |
+
self.dirty = threading.Event()
|
| 121 |
+
self.dirty.set()
|
| 122 |
+
self.snapshot = None # (position, wxyz, fov_y, aspect)
|
| 123 |
+
self.last_update_t = time.monotonic()
|
| 124 |
+
self.lock = threading.Lock()
|
| 125 |
+
|
| 126 |
+
def update_from_camera(self, cam: viser.CameraHandle):
|
| 127 |
+
with self.lock:
|
| 128 |
+
self.snapshot = (
|
| 129 |
+
np.asarray(cam.position, dtype=np.float32).copy(),
|
| 130 |
+
np.asarray(cam.wxyz, dtype=np.float32).copy(),
|
| 131 |
+
float(cam.fov),
|
| 132 |
+
float(cam.aspect),
|
| 133 |
+
)
|
| 134 |
+
self.last_update_t = time.monotonic()
|
| 135 |
+
self.dirty.set()
|
| 136 |
+
|
| 137 |
+
def consume(self):
|
| 138 |
+
with self.lock:
|
| 139 |
+
snap = self.snapshot
|
| 140 |
+
since = time.monotonic() - self.last_update_t
|
| 141 |
+
self.dirty.clear()
|
| 142 |
+
return snap, since
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def render_loop(
|
| 146 |
+
client: viser.ClientHandle,
|
| 147 |
+
renderer: NeuralRenderer,
|
| 148 |
+
state: RenderState,
|
| 149 |
+
max_res_slider,
|
| 150 |
+
interactive_scale_slider,
|
| 151 |
+
fps_readout,
|
| 152 |
+
pp_scale: float,
|
| 153 |
+
bench_table: dict,
|
| 154 |
+
interactive_threshold_s: float = 0.15,
|
| 155 |
+
min_dim: int = 64,
|
| 156 |
+
async_push: bool = True,
|
| 157 |
+
):
|
| 158 |
+
patch = int(renderer.model.patch_size)
|
| 159 |
+
Rw_cuda = _WORLD_DVSM_TO_VISER.cuda()
|
| 160 |
+
def _snap(x):
|
| 161 |
+
return max(patch, (int(x) // patch) * patch)
|
| 162 |
+
last_render_t = 0.0
|
| 163 |
+
bench_widths = sorted(bench_table.keys())
|
| 164 |
+
|
| 165 |
+
def _bench_predict_s(W: int) -> float:
|
| 166 |
+
"""Pure-GPU predicted render time at this W (from startup benchmark)."""
|
| 167 |
+
if not bench_widths:
|
| 168 |
+
return 0.0
|
| 169 |
+
if W <= bench_widths[0]:
|
| 170 |
+
return bench_table[bench_widths[0]]
|
| 171 |
+
if W >= bench_widths[-1]:
|
| 172 |
+
return bench_table[bench_widths[-1]]
|
| 173 |
+
for i in range(len(bench_widths) - 1):
|
| 174 |
+
a, b = bench_widths[i], bench_widths[i + 1]
|
| 175 |
+
if a <= W <= b:
|
| 176 |
+
r = (W - a) / (b - a)
|
| 177 |
+
return bench_table[a] + r * (bench_table[b] - bench_table[a])
|
| 178 |
+
return bench_table[bench_widths[-1]]
|
| 179 |
+
|
| 180 |
+
# EMA over the real iteration time (GPU + JPEG encode + websocket push).
|
| 181 |
+
# This is the true cadence at which complete frames reach the client and
|
| 182 |
+
# is the right cap for rate-limiting during drag. We per-W bucket it so
|
| 183 |
+
# changing the slider doesn't pollute the average.
|
| 184 |
+
ema_iter_s_by_W: dict[int, float] = {}
|
| 185 |
+
EMA_ALPHA = 0.3
|
| 186 |
+
|
| 187 |
+
# Inertia detection: three.js OrbitControls keep producing camera updates
|
| 188 |
+
# for ~300-500 ms after the user releases the mouse (damping). Server-side
|
| 189 |
+
# we track recent (t, pos) and detect a decreasing-velocity tail; skip
|
| 190 |
+
# rendering mid-decay frames so the user perceives "mouse stop β image stop".
|
| 191 |
+
import collections
|
| 192 |
+
pos_history: collections.deque = collections.deque(maxlen=5)
|
| 193 |
+
|
| 194 |
+
# Async-push: overlap viser's JPEG encode + websocket send with the next
|
| 195 |
+
# render. Single-worker so frames stay ordered. Each iteration waits for
|
| 196 |
+
# the previous push to drain before queuing a new one.
|
| 197 |
+
push_executor = None
|
| 198 |
+
push_future = None
|
| 199 |
+
if async_push:
|
| 200 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 201 |
+
push_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="viser-push")
|
| 202 |
+
while True:
|
| 203 |
+
# Short timeout so we can transition out of interactive mode
|
| 204 |
+
# (full-quality follow-up render) shortly after the user stops moving.
|
| 205 |
+
if not state.dirty.wait(timeout=0.05):
|
| 206 |
+
continue
|
| 207 |
+
snap, since_update = state.consume()
|
| 208 |
+
if snap is None:
|
| 209 |
+
continue
|
| 210 |
+
position, wxyz, fov_y, aspect = snap
|
| 211 |
+
|
| 212 |
+
# Dynamic downscale: optionally drop resolution while moving for smoother
|
| 213 |
+
# drag. Slider == 1.0 disables the downscale entirely.
|
| 214 |
+
i_scale = float(interactive_scale_slider.value)
|
| 215 |
+
is_interactive = (since_update < interactive_threshold_s) and (i_scale < 1.0)
|
| 216 |
+
scale = i_scale if is_interactive else 1.0
|
| 217 |
+
# If this render is at interactive quality, keep dirty set so we
|
| 218 |
+
# re-render at full quality after the user stops moving.
|
| 219 |
+
if is_interactive:
|
| 220 |
+
state.dirty.set()
|
| 221 |
+
W = _snap(max(min_dim, int(round(max_res_slider.value * scale))))
|
| 222 |
+
H = _snap(max(min_dim, int(round(W / aspect))))
|
| 223 |
+
|
| 224 |
+
# Per-resolution rate limit during active drag. Use the EMA of real
|
| 225 |
+
# iteration time (incl. JPEG + websocket overhead) when we have one,
|
| 226 |
+
# otherwise fall back to the GPU-only benchmark prediction.
|
| 227 |
+
now_t = time.monotonic()
|
| 228 |
+
predicted = ema_iter_s_by_W.get(W, _bench_predict_s(W))
|
| 229 |
+
is_during_drag = since_update < 0.3
|
| 230 |
+
|
| 231 |
+
# Track recent position for inertia/deceleration detection.
|
| 232 |
+
pos_history.append((now_t, np.asarray(position, dtype=np.float64)))
|
| 233 |
+
is_decelerating = False
|
| 234 |
+
if len(pos_history) >= 3 and is_during_drag:
|
| 235 |
+
(t_a, p_a), (t_b, p_b), (t_c, p_c) = pos_history[-3], pos_history[-2], pos_history[-1]
|
| 236 |
+
v_ab = float(np.linalg.norm(p_b - p_a)) / max(t_b - t_a, 1e-3)
|
| 237 |
+
v_bc = float(np.linalg.norm(p_c - p_b)) / max(t_c - t_b, 1e-3)
|
| 238 |
+
# Decelerating: velocity dropped >40% AND is now small. Both
|
| 239 |
+
# thresholds scale with current scene scale (pp.s makes scene unit
|
| 240 |
+
# size O(1) so absolute 0.005 unit/s is a reasonable "tiny").
|
| 241 |
+
is_decelerating = (v_bc < v_ab * 0.6) and (v_bc < 0.005)
|
| 242 |
+
|
| 243 |
+
# Skip if: we're rate-limited OR we're in inertia tail (and not too long
|
| 244 |
+
# since last render β if it's been a while, render anyway so the final
|
| 245 |
+
# settled frame always goes through).
|
| 246 |
+
if is_during_drag and (now_t - last_render_t) < predicted * 0.9:
|
| 247 |
+
state.dirty.set()
|
| 248 |
+
time.sleep(0.005)
|
| 249 |
+
continue
|
| 250 |
+
if is_decelerating and (now_t - last_render_t) < 0.6:
|
| 251 |
+
state.dirty.set()
|
| 252 |
+
time.sleep(0.020)
|
| 253 |
+
continue
|
| 254 |
+
iter_t0 = time.monotonic()
|
| 255 |
+
|
| 256 |
+
intr, c2w_r, c2w_t = viser_cam_to_dvsm(
|
| 257 |
+
position, wxyz, fov_y, aspect, device="cuda", dtype=torch.float32
|
| 258 |
+
)
|
| 259 |
+
# viser_cam_to_dvsm returns poses in viser world frame; rotate the
|
| 260 |
+
# world back into DVSM's normalized frame before rendering. Rw is
|
| 261 |
+
# self-inverse so the same matrix works in both directions. Then apply
|
| 262 |
+
# pp.s scaling (which we deliberately omitted from the viser frame so
|
| 263 |
+
# the controls feel natural at the scene's original metric scale).
|
| 264 |
+
c2w_r = torch.einsum('ij,bvjk->bvik', Rw_cuda, c2w_r)
|
| 265 |
+
c2w_t = torch.einsum('ij,bvj->bvi', Rw_cuda, c2w_t) * pp_scale
|
| 266 |
+
|
| 267 |
+
t0 = time.perf_counter()
|
| 268 |
+
try:
|
| 269 |
+
with torch.no_grad():
|
| 270 |
+
img = renderer(intr, c2w_r, c2w_t, H, W)
|
| 271 |
+
torch.cuda.synchronize()
|
| 272 |
+
except RuntimeError as e:
|
| 273 |
+
print(f"[render] error: {e}", flush=True)
|
| 274 |
+
time.sleep(0.1)
|
| 275 |
+
continue
|
| 276 |
+
dt_ms = (time.perf_counter() - t0) * 1000.0
|
| 277 |
+
last_render_t = time.monotonic()
|
| 278 |
+
|
| 279 |
+
rgb = (
|
| 280 |
+
img[0, 0].clamp(0.0, 1.0).mul(255.0).to(torch.uint8)
|
| 281 |
+
.permute(1, 2, 0).contiguous().cpu().numpy()
|
| 282 |
+
)
|
| 283 |
+
if push_executor is not None:
|
| 284 |
+
# Drain the previous push (keeps frame order, also surfaces send
|
| 285 |
+
# errors). On disconnect we exit cleanly.
|
| 286 |
+
if push_future is not None:
|
| 287 |
+
try:
|
| 288 |
+
push_future.result()
|
| 289 |
+
except Exception as e:
|
| 290 |
+
print(f"[render] set_background_image failed ({e}); exiting loop", flush=True)
|
| 291 |
+
push_executor.shutdown(wait=False, cancel_futures=True)
|
| 292 |
+
return
|
| 293 |
+
push_future = push_executor.submit(
|
| 294 |
+
lambda r=rgb: client.scene.set_background_image(
|
| 295 |
+
r, format="jpeg", jpeg_quality=60),
|
| 296 |
+
)
|
| 297 |
+
else:
|
| 298 |
+
try:
|
| 299 |
+
client.scene.set_background_image(rgb, format="jpeg", jpeg_quality=60)
|
| 300 |
+
except Exception as e:
|
| 301 |
+
print(f"[render] set_background_image failed ({e}); exiting loop", flush=True)
|
| 302 |
+
return
|
| 303 |
+
|
| 304 |
+
# Update EMA with the complete iteration time (GPU + encode + push).
|
| 305 |
+
iter_s = time.monotonic() - iter_t0
|
| 306 |
+
prev_ema = ema_iter_s_by_W.get(W)
|
| 307 |
+
ema_iter_s_by_W[W] = (
|
| 308 |
+
iter_s if prev_ema is None
|
| 309 |
+
else EMA_ALPHA * iter_s + (1 - EMA_ALPHA) * prev_ema
|
| 310 |
+
)
|
| 311 |
+
eff_fps = 1.0 / max(ema_iter_s_by_W[W], 1e-6)
|
| 312 |
+
fps_readout.value = (
|
| 313 |
+
f"{eff_fps:.1f} fps (gpu {dt_ms:.0f}ms, iter {iter_s * 1000:.0f}ms, "
|
| 314 |
+
f"{W}x{H}{', interactive' if is_interactive else ''})"
|
| 315 |
+
)
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
# -----------------------------------------------------------------------------
|
| 319 |
+
# Viser server setup
|
| 320 |
+
# -----------------------------------------------------------------------------
|
| 321 |
+
|
| 322 |
+
def add_input_view_frustums(server: viser.ViserServer, scene_meta: dict,
|
| 323 |
+
pp: PosePreprocessor, scale: float = 0.08):
|
| 324 |
+
"""Add a camera frustum per input view, in the DVSM normalized world frame
|
| 325 |
+
(poses passed through `pp` before conversion to viser)."""
|
| 326 |
+
c2w_r = scene_meta["input_c2w_r"]
|
| 327 |
+
c2w_t = scene_meta["input_c2w_t"]
|
| 328 |
+
intr = scene_meta["input_intr"]
|
| 329 |
+
V = c2w_r.shape[0]
|
| 330 |
+
handles = []
|
| 331 |
+
for v in range(V):
|
| 332 |
+
r_n, t_n = _apply_pp_single(pp, c2w_r[v], c2w_t[v])
|
| 333 |
+
pos, wxyz, fov_y, aspect = dvsm_cam_to_viser(intr[v], r_n, t_n)
|
| 334 |
+
h = server.scene.add_camera_frustum(
|
| 335 |
+
name=f"/input_views/{v:03d}",
|
| 336 |
+
fov=float(fov_y),
|
| 337 |
+
aspect=float(aspect),
|
| 338 |
+
scale=scale,
|
| 339 |
+
color=(0, 200, 200),
|
| 340 |
+
wxyz=tuple(map(float, wxyz)),
|
| 341 |
+
position=tuple(map(float, pos)),
|
| 342 |
+
)
|
| 343 |
+
handles.append(h)
|
| 344 |
+
return handles
|
| 345 |
+
|
| 346 |
+
|
| 347 |
+
def install_viser_handlers(server, renderer, scene_meta, pp_cpu, pp_scale,
|
| 348 |
+
model, args):
|
| 349 |
+
"""Add input-view frustums, warm up + benchmark the renderer, and register
|
| 350 |
+
per-client connect/disconnect handlers (each connection gets a render_loop
|
| 351 |
+
thread). Shared by the standalone ``serve()`` CLI and the HF Spaces ZeroGPU
|
| 352 |
+
``live_session_gpu`` generator. Returns the benchmark table.
|
| 353 |
+
|
| 354 |
+
Caller is responsible for creating ``server`` (so it can pick the port) and
|
| 355 |
+
for keeping the process alive afterwards.
|
| 356 |
+
"""
|
| 357 |
+
frustum_handles = add_input_view_frustums(server, scene_meta, pp_cpu)
|
| 358 |
+
|
| 359 |
+
# Pre-compute viser camera state for every input view (for jump-to-view +
|
| 360 |
+
# warm-up). Poses are normalized via pp first so they live in viser's
|
| 361 |
+
# (= DVSM normalized) world frame.
|
| 362 |
+
V_inp = int(scene_meta["n_input_views"])
|
| 363 |
+
input_views_viser = []
|
| 364 |
+
for v in range(V_inp):
|
| 365 |
+
r_n, t_n = _apply_pp_single(
|
| 366 |
+
pp_cpu,
|
| 367 |
+
scene_meta["input_c2w_r"][v],
|
| 368 |
+
scene_meta["input_c2w_t"][v],
|
| 369 |
+
)
|
| 370 |
+
pos_v, wxyz_v, fov_v, _asp_v = dvsm_cam_to_viser(
|
| 371 |
+
scene_meta["input_intr"][v], r_n, t_n,
|
| 372 |
+
)
|
| 373 |
+
input_views_viser.append((
|
| 374 |
+
tuple(map(float, pos_v)),
|
| 375 |
+
tuple(map(float, wxyz_v)),
|
| 376 |
+
float(fov_v),
|
| 377 |
+
))
|
| 378 |
+
init_pos_t, init_wxyz_t, init_fov_f = input_views_viser[0]
|
| 379 |
+
init_pos = np.asarray(init_pos_t, dtype=np.float32)
|
| 380 |
+
init_wxyz = np.asarray(init_wxyz_t, dtype=np.float32)
|
| 381 |
+
init_fov = init_fov_f
|
| 382 |
+
_aspect0 = float(scene_meta["input_res"][1]) / float(scene_meta["input_res"][0])
|
| 383 |
+
|
| 384 |
+
# Warm up the renderer (first call may compile / autotune).
|
| 385 |
+
print(f"[viewer] warming up renderer...", flush=True)
|
| 386 |
+
patch = int(model.patch_size)
|
| 387 |
+
Ww = max(patch, (256 // patch) * patch)
|
| 388 |
+
Hw = max(patch, (int(round(Ww / float(_aspect0))) // patch) * patch)
|
| 389 |
+
Rw_cuda_warmup = _WORLD_DVSM_TO_VISER.cuda()
|
| 390 |
+
with torch.no_grad():
|
| 391 |
+
intr_w, r_w, t_w = viser_cam_to_dvsm(
|
| 392 |
+
init_pos, init_wxyz, init_fov, float(_aspect0),
|
| 393 |
+
device="cuda", dtype=torch.float32,
|
| 394 |
+
)
|
| 395 |
+
r_w = torch.einsum('ij,bvjk->bvik', Rw_cuda_warmup, r_w)
|
| 396 |
+
t_w = torch.einsum('ij,bvj->bvi', Rw_cuda_warmup, t_w) * pp_scale
|
| 397 |
+
_ = renderer(intr_w, r_w, t_w, Hw, Ww)
|
| 398 |
+
torch.cuda.synchronize()
|
| 399 |
+
|
| 400 |
+
# Benchmark render time at common W values so the render loop can rate-
|
| 401 |
+
# limit during drag (avoiding piled-up stale renders that look like
|
| 402 |
+
# "screen keeps moving after the mouse stopped").
|
| 403 |
+
print(f"[viewer] benchmarking render time...", flush=True)
|
| 404 |
+
bench_widths_to_test = args.bench_widths
|
| 405 |
+
bench_table = {}
|
| 406 |
+
with torch.no_grad():
|
| 407 |
+
for Wb in bench_widths_to_test:
|
| 408 |
+
Wt = max(patch, (Wb // patch) * patch)
|
| 409 |
+
Ht = max(patch, (int(round(Wt / float(_aspect0))) // patch) * patch)
|
| 410 |
+
ts = []
|
| 411 |
+
for _ in range(3):
|
| 412 |
+
torch.cuda.synchronize()
|
| 413 |
+
tb = time.perf_counter()
|
| 414 |
+
_ = renderer(intr_w, r_w, t_w, Ht, Wt)
|
| 415 |
+
torch.cuda.synchronize()
|
| 416 |
+
ts.append(time.perf_counter() - tb)
|
| 417 |
+
bench_table[Wt] = min(ts)
|
| 418 |
+
print(f"[bench] W={Wt:4d} H={Ht:4d} -> {min(ts) * 1000:5.0f} ms "
|
| 419 |
+
f"(~{1.0 / min(ts):.1f} fps)", flush=True)
|
| 420 |
+
|
| 421 |
+
@server.on_client_connect
|
| 422 |
+
def _on_connect(client: viser.ClientHandle) -> None:
|
| 423 |
+
print(f"[viewer] client {client.client_id} connected", flush=True)
|
| 424 |
+
client.camera.position = init_pos_t
|
| 425 |
+
client.camera.wxyz = init_wxyz_t
|
| 426 |
+
client.camera.fov = init_fov_f
|
| 427 |
+
# IMPORTANT: setting position alone shifts look_at by the same delta
|
| 428 |
+
# (viser doc). Pin look_at to viser-frame origin = pp.r/t centroid of
|
| 429 |
+
# input cameras β scene center, so orbit drags rotate around the scene
|
| 430 |
+
# instead of some skewed-off point.
|
| 431 |
+
try:
|
| 432 |
+
client.camera.look_at = (0.0, 0.0, 0.0)
|
| 433 |
+
except Exception:
|
| 434 |
+
pass
|
| 435 |
+
|
| 436 |
+
with client.gui.add_folder("Render"):
|
| 437 |
+
max_res = client.gui.add_slider(
|
| 438 |
+
"Max W", min=256, max=960, step=64, initial_value=min(args.max_res, 960)
|
| 439 |
+
)
|
| 440 |
+
interactive_scale_slider = client.gui.add_slider(
|
| 441 |
+
"Interactive scale", min=0.25, max=1.0, step=0.25, initial_value=1.0,
|
| 442 |
+
)
|
| 443 |
+
fps_txt = client.gui.add_text("FPS", "β", disabled=True)
|
| 444 |
+
with client.gui.add_folder("Camera"):
|
| 445 |
+
view_slider = client.gui.add_slider(
|
| 446 |
+
"Jump to input view", min=0, max=max(0, V_inp - 1), step=1, initial_value=0
|
| 447 |
+
)
|
| 448 |
+
reset_btn = client.gui.add_button("Reset View")
|
| 449 |
+
with client.gui.add_folder("Scene"):
|
| 450 |
+
show_frustums = client.gui.add_checkbox("Show input view frustums", initial_value=True)
|
| 451 |
+
|
| 452 |
+
# Frustums default to visible (mirroring checkbox initial state).
|
| 453 |
+
for h in frustum_handles:
|
| 454 |
+
h.visible = True
|
| 455 |
+
|
| 456 |
+
def _jump_to(v: int, recenter: bool = False):
|
| 457 |
+
"""Jump camera to input view ``v``.
|
| 458 |
+
|
| 459 |
+
recenter=False (default): keep the input view's own orientation β
|
| 460 |
+
the camera-to-look_at vector stays as captured.
|
| 461 |
+
recenter=True (reset button only): after jumping, re-pin look_at
|
| 462 |
+
to viser-frame origin so orbit target is the scene center.
|
| 463 |
+
"""
|
| 464 |
+
v = max(0, min(V_inp - 1, int(v)))
|
| 465 |
+
p, q, f = input_views_viser[v]
|
| 466 |
+
client.camera.position = p
|
| 467 |
+
client.camera.wxyz = q
|
| 468 |
+
client.camera.fov = f
|
| 469 |
+
if recenter:
|
| 470 |
+
try:
|
| 471 |
+
client.camera.look_at = (0.0, 0.0, 0.0)
|
| 472 |
+
except Exception:
|
| 473 |
+
pass
|
| 474 |
+
|
| 475 |
+
@view_slider.on_update
|
| 476 |
+
def _(_):
|
| 477 |
+
_jump_to(view_slider.value, recenter=False)
|
| 478 |
+
|
| 479 |
+
@reset_btn.on_click
|
| 480 |
+
def _(_):
|
| 481 |
+
view_slider.value = 0
|
| 482 |
+
_jump_to(0, recenter=True)
|
| 483 |
+
|
| 484 |
+
@show_frustums.on_update
|
| 485 |
+
def _(_):
|
| 486 |
+
for h in frustum_handles:
|
| 487 |
+
h.visible = show_frustums.value
|
| 488 |
+
|
| 489 |
+
state = RenderState()
|
| 490 |
+
|
| 491 |
+
@client.camera.on_update
|
| 492 |
+
def _(_cam):
|
| 493 |
+
state.update_from_camera(client.camera)
|
| 494 |
+
|
| 495 |
+
# Re-render immediately when render-control sliders change, even if the
|
| 496 |
+
# camera hasn't moved (e.g. user only adjusted Max W or Interactive scale).
|
| 497 |
+
@max_res.on_update
|
| 498 |
+
def _(_):
|
| 499 |
+
state.update_from_camera(client.camera)
|
| 500 |
+
|
| 501 |
+
@interactive_scale_slider.on_update
|
| 502 |
+
def _(_):
|
| 503 |
+
state.update_from_camera(client.camera)
|
| 504 |
+
|
| 505 |
+
thread = threading.Thread(
|
| 506 |
+
target=render_loop,
|
| 507 |
+
args=(client, renderer, state, max_res, interactive_scale_slider,
|
| 508 |
+
fps_txt, pp_scale, bench_table),
|
| 509 |
+
kwargs={"async_push": not args.no_async_push},
|
| 510 |
+
daemon=True,
|
| 511 |
+
)
|
| 512 |
+
thread.start()
|
| 513 |
+
|
| 514 |
+
@server.on_client_disconnect
|
| 515 |
+
def _on_disconnect(client: viser.ClientHandle) -> None:
|
| 516 |
+
print(f"[viewer] client {client.client_id} disconnected", flush=True)
|
| 517 |
+
|
| 518 |
+
return bench_table
|
| 519 |
+
|
| 520 |
+
|
| 521 |
+
def serve(args):
|
| 522 |
+
if args.hf_variant:
|
| 523 |
+
print(f"[viewer] loading model from HuggingFace: {args.hf_variant}", flush=True)
|
| 524 |
+
model = load_model(hf_variant=args.hf_variant)
|
| 525 |
+
else:
|
| 526 |
+
print(f"[viewer] loading model from {args.ckpt_root}", flush=True)
|
| 527 |
+
model = load_model(args.ckpt_root, args.ckpt_name, [])
|
| 528 |
+
print(f"[viewer] loading recon artifact {args.recon}", flush=True)
|
| 529 |
+
artifact = torch.load(args.recon, weights_only=False, map_location="cpu")
|
| 530 |
+
renderer = build_renderer(model, artifact)
|
| 531 |
+
scene_meta = artifact["scene_meta"]
|
| 532 |
+
print(f"[viewer] V={scene_meta['n_input_views']} input res={scene_meta['input_res']}", flush=True)
|
| 533 |
+
|
| 534 |
+
# Align viser's world frame to DVSM's normalized frame (full SE(3) + scale
|
| 535 |
+
# to unit sphere via the saved PosePreprocessor). All viser-side positions
|
| 536 |
+
# are expressed in this normalized frame; the renderer's own preprocessor
|
| 537 |
+
# is bypassed (set to None below) so the same normalization doesn't get
|
| 538 |
+
# applied twice.
|
| 539 |
+
pp_cpu = PosePreprocessor.__new__(PosePreprocessor)
|
| 540 |
+
pp_cpu.r = artifact["pose_preproc"]["r"].cpu()
|
| 541 |
+
pp_cpu.t = artifact["pose_preproc"]["t"].cpu()
|
| 542 |
+
pp_cpu.s = artifact["pose_preproc"]["s"].cpu()
|
| 543 |
+
renderer.pose_preprocessor = None
|
| 544 |
+
pp_scale = float(pp_cpu.s.reshape(-1)[0])
|
| 545 |
+
print(f"[viewer] viser frame = pp.r + pp.t + Rw (no pp.s); renderer-side pp.s = {pp_scale:.4f}", flush=True)
|
| 546 |
+
|
| 547 |
+
# Viewer-only optimizations (FastPlucker cache + CUDA Graph + cuDNN warmup).
|
| 548 |
+
# No-op for training: nothing in dvsm/ is modified; train.py never imports
|
| 549 |
+
# viser_app/*. Optimization wraps run only in this serve() process.
|
| 550 |
+
if not args.no_optim:
|
| 551 |
+
renderer = install_optimizations(
|
| 552 |
+
renderer,
|
| 553 |
+
use_fast_plucker=not args.no_fast_plucker,
|
| 554 |
+
use_cuda_graph=not args.no_cuda_graph,
|
| 555 |
+
n_warmup=args.graph_warmup,
|
| 556 |
+
max_graphs=args.max_graphs,
|
| 557 |
+
)
|
| 558 |
+
print(f"[viewer] optim installed: fast_plucker={not args.no_fast_plucker} "
|
| 559 |
+
f"cuda_graph={not args.no_cuda_graph} warmup={args.graph_warmup} "
|
| 560 |
+
f"max_graphs={args.max_graphs}", flush=True)
|
| 561 |
+
|
| 562 |
+
server = viser.ViserServer(host="0.0.0.0", port=args.port)
|
| 563 |
+
# World is rotated into viser's native Y-up convention (see _WORLD_DVSM_TO_VISER),
|
| 564 |
+
# so scene up = +y (three.js default).
|
| 565 |
+
server.scene.set_up_direction("+y")
|
| 566 |
+
if args.share:
|
| 567 |
+
try:
|
| 568 |
+
share_url = server.request_share_url()
|
| 569 |
+
print(f"[viewer] share URL: {share_url}", flush=True)
|
| 570 |
+
except Exception as e:
|
| 571 |
+
print(f"[viewer] share URL request failed ({e}); fall back to SSH forward", flush=True)
|
| 572 |
+
install_viser_handlers(server, renderer, scene_meta, pp_cpu, pp_scale,
|
| 573 |
+
model, args)
|
| 574 |
+
print(f"[viewer] ready β open http://localhost:{args.port}", flush=True)
|
| 575 |
+
|
| 576 |
+
if args.smoke_test:
|
| 577 |
+
print("[viewer] smoke test: warm-up + server started OK, exiting", flush=True)
|
| 578 |
+
return
|
| 579 |
+
|
| 580 |
+
while True:
|
| 581 |
+
time.sleep(1.0)
|
| 582 |
+
|
| 583 |
+
|
| 584 |
+
def main():
|
| 585 |
+
parser = argparse.ArgumentParser()
|
| 586 |
+
parser.add_argument("--ckpt_root", type=Path, default=None)
|
| 587 |
+
parser.add_argument("--hf", dest="hf_variant", default=None,
|
| 588 |
+
help="HuggingFace variant, e.g. DVSM-TTB-p8-DL3DV-960p")
|
| 589 |
+
parser.add_argument("--ckpt_name", default="latest.pt")
|
| 590 |
+
parser.add_argument("--recon", type=Path, required=True,
|
| 591 |
+
help="Path to .recon.pt produced by recon_scene.py")
|
| 592 |
+
parser.add_argument("--port", type=int, default=8080)
|
| 593 |
+
parser.add_argument("--max_res", type=int, default=768,
|
| 594 |
+
help="Initial Max W slider value; must align with step=64 used by the slider.")
|
| 595 |
+
parser.add_argument("--smoke_test", action="store_true",
|
| 596 |
+
help="Exit after warm-up render and server start (CI / sbatch validation).")
|
| 597 |
+
parser.add_argument("--share", action="store_true",
|
| 598 |
+
help="Request a viser studio share URL so the viewer is reachable without SSH forward.")
|
| 599 |
+
parser.add_argument("--no_optim", action="store_true",
|
| 600 |
+
help="Disable all viewer-side speedups (FastPlucker + CUDAGraph + async push). Debug only.")
|
| 601 |
+
parser.add_argument("--no_fast_plucker", action="store_true",
|
| 602 |
+
help="Disable FastPluckerEncoding (cache + matmul shortcut).")
|
| 603 |
+
parser.add_argument("--no_cuda_graph", action="store_true",
|
| 604 |
+
help="Disable CUDA Graph capture/replay around the renderer.")
|
| 605 |
+
parser.add_argument("--no_async_push", action="store_true",
|
| 606 |
+
help="Push frames synchronously instead of overlapping with the next render.")
|
| 607 |
+
parser.add_argument("--graph_warmup", type=int, default=5,
|
| 608 |
+
help="cuDNN warmup iterations before each CUDA Graph capture.")
|
| 609 |
+
parser.add_argument("--max_graphs", type=int, default=6,
|
| 610 |
+
help="Max number of cached (H, W, V) CUDA Graphs (LRU).")
|
| 611 |
+
parser.add_argument("--bench_widths", type=int, nargs="+",
|
| 612 |
+
default=[256, 384, 512, 640, 768, 960],
|
| 613 |
+
help="Widths to benchmark at startup for drag rate-limiting. "
|
| 614 |
+
"Reduce (e.g. 512 768) to shorten warmup on quota-limited GPUs.")
|
| 615 |
+
args = parser.parse_args()
|
| 616 |
+
serve(args)
|
| 617 |
+
|
| 618 |
+
|
| 619 |
+
if __name__ == "__main__":
|
| 620 |
+
main()
|
viser_app/viewer_optim.py
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Viewer-side rendering optimizations. All wrappers; never touches dvsm/*
|
| 2 |
+
weights or training paths. Only the viewer process loads this module.
|
| 3 |
+
|
| 4 |
+
Optimizations (mirrored from facebookresearch/lagernvs):
|
| 5 |
+
1. FastPluckerEncoding β cache camera-frame ray dirs per (H, W); per-frame
|
| 6 |
+
cost is one batched matmul + cross product.
|
| 7 |
+
2. GraphRenderer β per-(H, W, V) CUDA Graph capture + replay around
|
| 8 |
+
a NeuralRenderer. First call at a new size warms
|
| 9 |
+
cuDNN then captures; later calls just copy inputs
|
| 10 |
+
and replay.
|
| 11 |
+
3. cuDNN warmup β n_warmup forward passes on a side stream before
|
| 12 |
+
capture (built into GraphRenderer).
|
| 13 |
+
4. Async JPEG push β see render_loop; ThreadPoolExecutor overlaps
|
| 14 |
+
viser set_background_image with the next render.
|
| 15 |
+
"""
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import threading
|
| 19 |
+
from typing import Any
|
| 20 |
+
|
| 21 |
+
import torch
|
| 22 |
+
import torch.nn as nn
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class FastPluckerEncoding(nn.Module):
|
| 26 |
+
"""Drop-in replacement for `dvsm.models.plucker.PluckerEncoding` that
|
| 27 |
+
caches the camera-frame ray directions for a given (H, W) and the current
|
| 28 |
+
intrinsics. Per-frame cost is then:
|
| 29 |
+
ray_d = normalize(c2w_r @ dirs_local)
|
| 30 |
+
plucker = pose_enc(c2w_t, ray_d)
|
| 31 |
+
|
| 32 |
+
The cache key is (H, W). Intrinsics are assumed to stay constant within a
|
| 33 |
+
single viewer session (true in viser_app/viewer.py β fov is set once from
|
| 34 |
+
the first input view and the only thing changing per frame is c2w).
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
def __init__(self, original: nn.Module):
|
| 38 |
+
super().__init__()
|
| 39 |
+
# Keep the original module around so we still delegate the plucker
|
| 40 |
+
# concat (compute_plucker / compute_plucker_aug / ...) to whatever the
|
| 41 |
+
# checkpoint trained against.
|
| 42 |
+
self._original = original
|
| 43 |
+
self.dim = original.dim
|
| 44 |
+
self.pose_enc = original.pose_enc
|
| 45 |
+
|
| 46 |
+
self._cache_lock = threading.Lock()
|
| 47 |
+
# Per-(H, W) cache. Entries are kept alive for the full session so any
|
| 48 |
+
# CUDA Graph that captured a kernel reading from an entry's GPU memory
|
| 49 |
+
# keeps a valid pointer (the captured kernel runs at replay without
|
| 50 |
+
# going through Python, so we can't invalidate per-graph references).
|
| 51 |
+
self._cache: dict[tuple[int, int], torch.Tensor] = {}
|
| 52 |
+
|
| 53 |
+
@staticmethod
|
| 54 |
+
def _compute_dirs_local(intr: torch.Tensor, h: int, w: int) -> torch.Tensor:
|
| 55 |
+
"""Camera-frame ray directions. Depends only on (intr, h, w).
|
| 56 |
+
Mirrors the first half of `dvsm.models.plucker.compute_rays`."""
|
| 57 |
+
b, v = intr.shape[:2]
|
| 58 |
+
fx, fy, cx, cy = intr.flatten(0, 1).T.unsqueeze(-1).contiguous()
|
| 59 |
+
fx = fx * w
|
| 60 |
+
fy = fy * h
|
| 61 |
+
cx = cx * w
|
| 62 |
+
cy = cy * h
|
| 63 |
+
idx_x = torch.arange(w, device=intr.device) + 0.5
|
| 64 |
+
idx_y = torch.arange(h, device=intr.device) + 0.5
|
| 65 |
+
idx_x = idx_x[None, :].expand(h, -1)
|
| 66 |
+
idx_y = idx_y[:, None].expand(-1, w)
|
| 67 |
+
idx_x = idx_x.flatten().expand(b * v, -1)
|
| 68 |
+
idx_y = idx_y.flatten().expand(b * v, -1)
|
| 69 |
+
x = (idx_x - cx) / fx
|
| 70 |
+
y = (idx_y - cy) / fy
|
| 71 |
+
z = torch.ones_like(x)
|
| 72 |
+
dirs = torch.stack([x, y, z], dim=1) # [B*V, 3, H*W]
|
| 73 |
+
return dirs.reshape(b, v, 3, h, w).contiguous()
|
| 74 |
+
|
| 75 |
+
def forward(self, intr, c2w_r, c2w_t, h, w):
|
| 76 |
+
# Cache check is python-level (no GPU sync). Inside a CUDA Graph
|
| 77 |
+
# capture, this if-branch evaluates once at capture time; the cache
|
| 78 |
+
# is pre-warmed before capture so we take the cache-hit path and only
|
| 79 |
+
# those kernels (bmm + normalize + plucker concat) get recorded.
|
| 80 |
+
with self._cache_lock:
|
| 81 |
+
dirs_local = self._cache.get((h, w))
|
| 82 |
+
if dirs_local is None:
|
| 83 |
+
dirs_local = self._compute_dirs_local(intr, h, w)
|
| 84 |
+
self._cache[(h, w)] = dirs_local
|
| 85 |
+
|
| 86 |
+
b, v = intr.shape[:2]
|
| 87 |
+
c2w_r_bv = c2w_r.reshape(b * v, 3, 3)
|
| 88 |
+
ray_d = torch.bmm(c2w_r_bv, dirs_local.reshape(b * v, 3, h * w))
|
| 89 |
+
ray_d = torch.nn.functional.normalize(ray_d, dim=1)
|
| 90 |
+
ray_d = ray_d.reshape(b, v, 3, h, w)
|
| 91 |
+
ray_o = c2w_t.view(b, v, 3, 1, 1).expand(b, v, 3, h, w)
|
| 92 |
+
return self.pose_enc(ray_o, ray_d)
|
| 93 |
+
|
| 94 |
+
def warm_cache_for(self, intr: torch.Tensor, h: int, w: int) -> None:
|
| 95 |
+
"""Pre-populate the cache for (H, W). Call before CUDA Graph capture
|
| 96 |
+
so the cache-hit path is what gets recorded."""
|
| 97 |
+
with self._cache_lock:
|
| 98 |
+
if (h, w) not in self._cache:
|
| 99 |
+
self._cache[(h, w)] = self._compute_dirs_local(intr, h, w)
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
class GraphRenderer:
|
| 103 |
+
"""CUDA Graph wrapper around a NeuralRenderer.
|
| 104 |
+
|
| 105 |
+
Each unique (H, W, V) gets its own captured graph + static IO buffers.
|
| 106 |
+
First call at a new size:
|
| 107 |
+
warmup -> capture -> replay (slow, ~tens of ms).
|
| 108 |
+
Subsequent calls at the same size:
|
| 109 |
+
copy live inputs into static buffers -> replay (fast).
|
| 110 |
+
|
| 111 |
+
Returns a `.clone()` of the static output so the caller can freely mutate
|
| 112 |
+
or send it to viser without aliasing the graph's internal buffer.
|
| 113 |
+
"""
|
| 114 |
+
|
| 115 |
+
def __init__(self, renderer, n_warmup: int = 5, max_graphs: int = 3):
|
| 116 |
+
self.renderer = renderer
|
| 117 |
+
self.n_warmup = n_warmup
|
| 118 |
+
self.max_graphs = max_graphs
|
| 119 |
+
self._graphs: dict[tuple, dict[str, Any]] = {}
|
| 120 |
+
self._pool = torch.cuda.graph_pool_handle()
|
| 121 |
+
self._lock = threading.Lock()
|
| 122 |
+
# Identify the underlying FastPluckerEncoding (if installed) so we can
|
| 123 |
+
# pre-warm its cache before capture. Otherwise the captured graph would
|
| 124 |
+
# record the cache-miss branch the first time.
|
| 125 |
+
self._fast_plucker = renderer.model.plucker_enc \
|
| 126 |
+
if isinstance(renderer.model.plucker_enc, FastPluckerEncoding) else None
|
| 127 |
+
|
| 128 |
+
# Make the wrapper look like a NeuralRenderer to downstream consumers
|
| 129 |
+
# (render_loop reads `renderer.model.patch_size`, etc.).
|
| 130 |
+
@property
|
| 131 |
+
def model(self):
|
| 132 |
+
return self.renderer.model
|
| 133 |
+
|
| 134 |
+
def _capture(self, intr, c2w_r, c2w_t, H, W):
|
| 135 |
+
static_intr = intr.clone().contiguous()
|
| 136 |
+
static_r = c2w_r.clone().contiguous()
|
| 137 |
+
static_t = c2w_t.clone().contiguous()
|
| 138 |
+
|
| 139 |
+
# Pre-warm the plucker cache against the static intr β otherwise the
|
| 140 |
+
# cache-miss branch (which allocates a new dirs_local on the default
|
| 141 |
+
# allocator) gets captured, and the next capture at a different (H,W)
|
| 142 |
+
# would re-trigger that branch with stale memory.
|
| 143 |
+
if self._fast_plucker is not None:
|
| 144 |
+
self._fast_plucker.warm_cache_for(static_intr, H, W)
|
| 145 |
+
|
| 146 |
+
# cuDNN warmup on a side stream so the graph captures the optimal
|
| 147 |
+
# kernel selection, not the first-time-autotune kernels.
|
| 148 |
+
with torch.inference_mode():
|
| 149 |
+
s = torch.cuda.Stream()
|
| 150 |
+
s.wait_stream(torch.cuda.current_stream())
|
| 151 |
+
with torch.cuda.stream(s):
|
| 152 |
+
for _ in range(self.n_warmup):
|
| 153 |
+
_ = self.renderer(static_intr, static_r, static_t, H, W)
|
| 154 |
+
torch.cuda.current_stream().wait_stream(s)
|
| 155 |
+
torch.cuda.synchronize()
|
| 156 |
+
|
| 157 |
+
graph = torch.cuda.CUDAGraph()
|
| 158 |
+
with torch.cuda.graph(graph, pool=self._pool):
|
| 159 |
+
static_out = self.renderer(static_intr, static_r, static_t, H, W)
|
| 160 |
+
|
| 161 |
+
return {
|
| 162 |
+
"intr": static_intr,
|
| 163 |
+
"r": static_r,
|
| 164 |
+
"t": static_t,
|
| 165 |
+
"out": static_out,
|
| 166 |
+
"graph": graph,
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
def __call__(self, intr, c2w_r, c2w_t, H, W, ts=None):
|
| 170 |
+
# ts is ignored β the baked render path doesn't use it.
|
| 171 |
+
V = int(intr.shape[1])
|
| 172 |
+
key = (int(H), int(W), V)
|
| 173 |
+
with self._lock:
|
| 174 |
+
entry = self._graphs.get(key)
|
| 175 |
+
if entry is None:
|
| 176 |
+
if len(self._graphs) >= self.max_graphs:
|
| 177 |
+
# LRU: dicts preserve insertion order, pop the oldest.
|
| 178 |
+
oldest = next(iter(self._graphs))
|
| 179 |
+
self._graphs.pop(oldest)
|
| 180 |
+
entry = self._capture(intr, c2w_r, c2w_t, H, W)
|
| 181 |
+
self._graphs[key] = entry
|
| 182 |
+
|
| 183 |
+
entry["intr"].copy_(intr)
|
| 184 |
+
entry["r"].copy_(c2w_r)
|
| 185 |
+
entry["t"].copy_(c2w_t)
|
| 186 |
+
entry["graph"].replay()
|
| 187 |
+
return entry["out"].clone()
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def install_optimizations(renderer, use_fast_plucker: bool = True,
|
| 191 |
+
use_cuda_graph: bool = True,
|
| 192 |
+
n_warmup: int = 5,
|
| 193 |
+
max_graphs: int = 3):
|
| 194 |
+
"""Apply both optimizations to a NeuralRenderer (in place + wrap). Returns
|
| 195 |
+
the (possibly wrapped) renderer that the viewer should call from now on.
|
| 196 |
+
Safe to call once at viewer startup; never invoked from training."""
|
| 197 |
+
if use_fast_plucker:
|
| 198 |
+
renderer.model.plucker_enc = FastPluckerEncoding(renderer.model.plucker_enc)
|
| 199 |
+
if use_cuda_graph:
|
| 200 |
+
renderer = GraphRenderer(renderer, n_warmup=n_warmup, max_graphs=max_graphs)
|
| 201 |
+
return renderer
|