Download viser_app/viewer_optim.py from nvidia/DVSM: direct link, hf CLI and curl.
- Browser
- Download file 8.97 kB
-
https://huggingface.co/spaces/nvidia/DVSM/resolve/main/viser_app/viewer_optim.py
- Command line
-
hf download hf://spaces/nvidia/DVSM/viser_app/viewer_optim.py
-
curl -L -o viewer_optim.py https://huggingface.co/spaces/nvidia/DVSM/resolve/main/viser_app/viewer_optim.py
8.97 kB
| """Viewer-side rendering optimizations. All wrappers; never touches dvsm/* | |
| weights or training paths. Only the viewer process loads this module. | |
| Optimizations (mirrored from facebookresearch/lagernvs): | |
| 1. FastPluckerEncoding β cache camera-frame ray dirs per (H, W); per-frame | |
| cost is one batched matmul + cross product. | |
| 2. GraphRenderer β per-(H, W, V) CUDA Graph capture + replay around | |
| a NeuralRenderer. First call at a new size warms | |
| cuDNN then captures; later calls just copy inputs | |
| and replay. | |
| 3. cuDNN warmup β n_warmup forward passes on a side stream before | |
| capture (built into GraphRenderer). | |
| 4. Async JPEG push β see render_loop; ThreadPoolExecutor overlaps | |
| viser set_background_image with the next render. | |
| """ | |
| from __future__ import annotations | |
| import threading | |
| from typing import Any | |
| import torch | |
| import torch.nn as nn | |
| class FastPluckerEncoding(nn.Module): | |
| """Drop-in replacement for `dvsm.models.plucker.PluckerEncoding` that | |
| caches the camera-frame ray directions for a given (H, W) and the current | |
| intrinsics. Per-frame cost is then: | |
| ray_d = normalize(c2w_r @ dirs_local) | |
| plucker = pose_enc(c2w_t, ray_d) | |
| The cache key is (H, W). Intrinsics are assumed to stay constant within a | |
| single viewer session (true in viser_app/viewer.py β fov is set once from | |
| the first input view and the only thing changing per frame is c2w). | |
| """ | |
| def __init__(self, original: nn.Module): | |
| super().__init__() | |
| # Keep the original module around so we still delegate the plucker | |
| # concat (compute_plucker / compute_plucker_aug / ...) to whatever the | |
| # checkpoint trained against. | |
| self._original = original | |
| self.dim = original.dim | |
| self.pose_enc = original.pose_enc | |
| self._cache_lock = threading.Lock() | |
| # Per-(H, W) cache. Entries are kept alive for the full session so any | |
| # CUDA Graph that captured a kernel reading from an entry's GPU memory | |
| # keeps a valid pointer (the captured kernel runs at replay without | |
| # going through Python, so we can't invalidate per-graph references). | |
| self._cache: dict[tuple[int, int], torch.Tensor] = {} | |
| def _compute_dirs_local(intr: torch.Tensor, h: int, w: int) -> torch.Tensor: | |
| """Camera-frame ray directions. Depends only on (intr, h, w). | |
| Mirrors the first half of `dvsm.models.plucker.compute_rays`.""" | |
| b, v = intr.shape[:2] | |
| fx, fy, cx, cy = intr.flatten(0, 1).T.unsqueeze(-1).contiguous() | |
| fx = fx * w | |
| fy = fy * h | |
| cx = cx * w | |
| cy = cy * h | |
| idx_x = torch.arange(w, device=intr.device) + 0.5 | |
| idx_y = torch.arange(h, device=intr.device) + 0.5 | |
| idx_x = idx_x[None, :].expand(h, -1) | |
| idx_y = idx_y[:, None].expand(-1, w) | |
| idx_x = idx_x.flatten().expand(b * v, -1) | |
| idx_y = idx_y.flatten().expand(b * v, -1) | |
| x = (idx_x - cx) / fx | |
| y = (idx_y - cy) / fy | |
| z = torch.ones_like(x) | |
| dirs = torch.stack([x, y, z], dim=1) # [B*V, 3, H*W] | |
| return dirs.reshape(b, v, 3, h, w).contiguous() | |
| def forward(self, intr, c2w_r, c2w_t, h, w): | |
| # Cache check is python-level (no GPU sync). Inside a CUDA Graph | |
| # capture, this if-branch evaluates once at capture time; the cache | |
| # is pre-warmed before capture so we take the cache-hit path and only | |
| # those kernels (bmm + normalize + plucker concat) get recorded. | |
| with self._cache_lock: | |
| dirs_local = self._cache.get((h, w)) | |
| if dirs_local is None: | |
| dirs_local = self._compute_dirs_local(intr, h, w) | |
| self._cache[(h, w)] = dirs_local | |
| b, v = intr.shape[:2] | |
| c2w_r_bv = c2w_r.reshape(b * v, 3, 3) | |
| ray_d = torch.bmm(c2w_r_bv, dirs_local.reshape(b * v, 3, h * w)) | |
| ray_d = torch.nn.functional.normalize(ray_d, dim=1) | |
| ray_d = ray_d.reshape(b, v, 3, h, w) | |
| ray_o = c2w_t.view(b, v, 3, 1, 1).expand(b, v, 3, h, w) | |
| return self.pose_enc(ray_o, ray_d) | |
| def warm_cache_for(self, intr: torch.Tensor, h: int, w: int) -> None: | |
| """Pre-populate the cache for (H, W). Call before CUDA Graph capture | |
| so the cache-hit path is what gets recorded.""" | |
| with self._cache_lock: | |
| if (h, w) not in self._cache: | |
| self._cache[(h, w)] = self._compute_dirs_local(intr, h, w) | |
| class GraphRenderer: | |
| """CUDA Graph wrapper around a NeuralRenderer. | |
| Each unique (H, W, V) gets its own captured graph + static IO buffers. | |
| First call at a new size: | |
| warmup -> capture -> replay (slow, ~tens of ms). | |
| Subsequent calls at the same size: | |
| copy live inputs into static buffers -> replay (fast). | |
| Returns a `.clone()` of the static output so the caller can freely mutate | |
| or send it to viser without aliasing the graph's internal buffer. | |
| """ | |
| def __init__(self, renderer, n_warmup: int = 5, max_graphs: int = 3): | |
| self.renderer = renderer | |
| self.n_warmup = n_warmup | |
| self.max_graphs = max_graphs | |
| self._graphs: dict[tuple, dict[str, Any]] = {} | |
| self._pool = torch.cuda.graph_pool_handle() | |
| self._lock = threading.Lock() | |
| # Identify the underlying FastPluckerEncoding (if installed) so we can | |
| # pre-warm its cache before capture. Otherwise the captured graph would | |
| # record the cache-miss branch the first time. | |
| self._fast_plucker = renderer.model.plucker_enc \ | |
| if isinstance(renderer.model.plucker_enc, FastPluckerEncoding) else None | |
| # Make the wrapper look like a NeuralRenderer to downstream consumers | |
| # (render_loop reads `renderer.model.patch_size`, etc.). | |
| def model(self): | |
| return self.renderer.model | |
| def _capture(self, intr, c2w_r, c2w_t, H, W): | |
| static_intr = intr.clone().contiguous() | |
| static_r = c2w_r.clone().contiguous() | |
| static_t = c2w_t.clone().contiguous() | |
| # Pre-warm the plucker cache against the static intr β otherwise the | |
| # cache-miss branch (which allocates a new dirs_local on the default | |
| # allocator) gets captured, and the next capture at a different (H,W) | |
| # would re-trigger that branch with stale memory. | |
| if self._fast_plucker is not None: | |
| self._fast_plucker.warm_cache_for(static_intr, H, W) | |
| # cuDNN warmup on a side stream so the graph captures the optimal | |
| # kernel selection, not the first-time-autotune kernels. | |
| with torch.inference_mode(): | |
| s = torch.cuda.Stream() | |
| s.wait_stream(torch.cuda.current_stream()) | |
| with torch.cuda.stream(s): | |
| for _ in range(self.n_warmup): | |
| _ = self.renderer(static_intr, static_r, static_t, H, W) | |
| torch.cuda.current_stream().wait_stream(s) | |
| torch.cuda.synchronize() | |
| graph = torch.cuda.CUDAGraph() | |
| with torch.cuda.graph(graph, pool=self._pool): | |
| static_out = self.renderer(static_intr, static_r, static_t, H, W) | |
| return { | |
| "intr": static_intr, | |
| "r": static_r, | |
| "t": static_t, | |
| "out": static_out, | |
| "graph": graph, | |
| } | |
| def __call__(self, intr, c2w_r, c2w_t, H, W, ts=None): | |
| # ts is ignored β the baked render path doesn't use it. | |
| V = int(intr.shape[1]) | |
| key = (int(H), int(W), V) | |
| with self._lock: | |
| entry = self._graphs.get(key) | |
| if entry is None: | |
| if len(self._graphs) >= self.max_graphs: | |
| # LRU: dicts preserve insertion order, pop the oldest. | |
| oldest = next(iter(self._graphs)) | |
| self._graphs.pop(oldest) | |
| entry = self._capture(intr, c2w_r, c2w_t, H, W) | |
| self._graphs[key] = entry | |
| entry["intr"].copy_(intr) | |
| entry["r"].copy_(c2w_r) | |
| entry["t"].copy_(c2w_t) | |
| entry["graph"].replay() | |
| return entry["out"].clone() | |
| def install_optimizations(renderer, use_fast_plucker: bool = True, | |
| use_cuda_graph: bool = True, | |
| n_warmup: int = 5, | |
| max_graphs: int = 3): | |
| """Apply both optimizations to a NeuralRenderer (in place + wrap). Returns | |
| the (possibly wrapped) renderer that the viewer should call from now on. | |
| Safe to call once at viewer startup; never invoked from training.""" | |
| if use_fast_plucker: | |
| renderer.model.plucker_enc = FastPluckerEncoding(renderer.model.plucker_enc) | |
| if use_cuda_graph: | |
| renderer = GraphRenderer(renderer, n_warmup=n_warmup, max_graphs=max_graphs) | |
| return renderer | |