DVSM / viser_app /viewer_optim.py
linjohnss's picture
Initial commit
cebb76b
Raw History Blame Contribute Delete
8.97 kB
"""Viewer-side rendering optimizations. All wrappers; never touches dvsm/*
weights or training paths. Only the viewer process loads this module.
Optimizations (mirrored from facebookresearch/lagernvs):
1. FastPluckerEncoding β€” cache camera-frame ray dirs per (H, W); per-frame
cost is one batched matmul + cross product.
2. GraphRenderer β€” per-(H, W, V) CUDA Graph capture + replay around
a NeuralRenderer. First call at a new size warms
cuDNN then captures; later calls just copy inputs
and replay.
3. cuDNN warmup β€” n_warmup forward passes on a side stream before
capture (built into GraphRenderer).
4. Async JPEG push β€” see render_loop; ThreadPoolExecutor overlaps
viser set_background_image with the next render.
"""
from __future__ import annotations
import threading
from typing import Any
import torch
import torch.nn as nn
class FastPluckerEncoding(nn.Module):
"""Drop-in replacement for `dvsm.models.plucker.PluckerEncoding` that
caches the camera-frame ray directions for a given (H, W) and the current
intrinsics. Per-frame cost is then:
ray_d = normalize(c2w_r @ dirs_local)
plucker = pose_enc(c2w_t, ray_d)
The cache key is (H, W). Intrinsics are assumed to stay constant within a
single viewer session (true in viser_app/viewer.py β€” fov is set once from
the first input view and the only thing changing per frame is c2w).
"""
def __init__(self, original: nn.Module):
super().__init__()
# Keep the original module around so we still delegate the plucker
# concat (compute_plucker / compute_plucker_aug / ...) to whatever the
# checkpoint trained against.
self._original = original
self.dim = original.dim
self.pose_enc = original.pose_enc
self._cache_lock = threading.Lock()
# Per-(H, W) cache. Entries are kept alive for the full session so any
# CUDA Graph that captured a kernel reading from an entry's GPU memory
# keeps a valid pointer (the captured kernel runs at replay without
# going through Python, so we can't invalidate per-graph references).
self._cache: dict[tuple[int, int], torch.Tensor] = {}
@staticmethod
def _compute_dirs_local(intr: torch.Tensor, h: int, w: int) -> torch.Tensor:
"""Camera-frame ray directions. Depends only on (intr, h, w).
Mirrors the first half of `dvsm.models.plucker.compute_rays`."""
b, v = intr.shape[:2]
fx, fy, cx, cy = intr.flatten(0, 1).T.unsqueeze(-1).contiguous()
fx = fx * w
fy = fy * h
cx = cx * w
cy = cy * h
idx_x = torch.arange(w, device=intr.device) + 0.5
idx_y = torch.arange(h, device=intr.device) + 0.5
idx_x = idx_x[None, :].expand(h, -1)
idx_y = idx_y[:, None].expand(-1, w)
idx_x = idx_x.flatten().expand(b * v, -1)
idx_y = idx_y.flatten().expand(b * v, -1)
x = (idx_x - cx) / fx
y = (idx_y - cy) / fy
z = torch.ones_like(x)
dirs = torch.stack([x, y, z], dim=1) # [B*V, 3, H*W]
return dirs.reshape(b, v, 3, h, w).contiguous()
def forward(self, intr, c2w_r, c2w_t, h, w):
# Cache check is python-level (no GPU sync). Inside a CUDA Graph
# capture, this if-branch evaluates once at capture time; the cache
# is pre-warmed before capture so we take the cache-hit path and only
# those kernels (bmm + normalize + plucker concat) get recorded.
with self._cache_lock:
dirs_local = self._cache.get((h, w))
if dirs_local is None:
dirs_local = self._compute_dirs_local(intr, h, w)
self._cache[(h, w)] = dirs_local
b, v = intr.shape[:2]
c2w_r_bv = c2w_r.reshape(b * v, 3, 3)
ray_d = torch.bmm(c2w_r_bv, dirs_local.reshape(b * v, 3, h * w))
ray_d = torch.nn.functional.normalize(ray_d, dim=1)
ray_d = ray_d.reshape(b, v, 3, h, w)
ray_o = c2w_t.view(b, v, 3, 1, 1).expand(b, v, 3, h, w)
return self.pose_enc(ray_o, ray_d)
def warm_cache_for(self, intr: torch.Tensor, h: int, w: int) -> None:
"""Pre-populate the cache for (H, W). Call before CUDA Graph capture
so the cache-hit path is what gets recorded."""
with self._cache_lock:
if (h, w) not in self._cache:
self._cache[(h, w)] = self._compute_dirs_local(intr, h, w)
class GraphRenderer:
"""CUDA Graph wrapper around a NeuralRenderer.
Each unique (H, W, V) gets its own captured graph + static IO buffers.
First call at a new size:
warmup -> capture -> replay (slow, ~tens of ms).
Subsequent calls at the same size:
copy live inputs into static buffers -> replay (fast).
Returns a `.clone()` of the static output so the caller can freely mutate
or send it to viser without aliasing the graph's internal buffer.
"""
def __init__(self, renderer, n_warmup: int = 5, max_graphs: int = 3):
self.renderer = renderer
self.n_warmup = n_warmup
self.max_graphs = max_graphs
self._graphs: dict[tuple, dict[str, Any]] = {}
self._pool = torch.cuda.graph_pool_handle()
self._lock = threading.Lock()
# Identify the underlying FastPluckerEncoding (if installed) so we can
# pre-warm its cache before capture. Otherwise the captured graph would
# record the cache-miss branch the first time.
self._fast_plucker = renderer.model.plucker_enc \
if isinstance(renderer.model.plucker_enc, FastPluckerEncoding) else None
# Make the wrapper look like a NeuralRenderer to downstream consumers
# (render_loop reads `renderer.model.patch_size`, etc.).
@property
def model(self):
return self.renderer.model
def _capture(self, intr, c2w_r, c2w_t, H, W):
static_intr = intr.clone().contiguous()
static_r = c2w_r.clone().contiguous()
static_t = c2w_t.clone().contiguous()
# Pre-warm the plucker cache against the static intr β€” otherwise the
# cache-miss branch (which allocates a new dirs_local on the default
# allocator) gets captured, and the next capture at a different (H,W)
# would re-trigger that branch with stale memory.
if self._fast_plucker is not None:
self._fast_plucker.warm_cache_for(static_intr, H, W)
# cuDNN warmup on a side stream so the graph captures the optimal
# kernel selection, not the first-time-autotune kernels.
with torch.inference_mode():
s = torch.cuda.Stream()
s.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(s):
for _ in range(self.n_warmup):
_ = self.renderer(static_intr, static_r, static_t, H, W)
torch.cuda.current_stream().wait_stream(s)
torch.cuda.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, pool=self._pool):
static_out = self.renderer(static_intr, static_r, static_t, H, W)
return {
"intr": static_intr,
"r": static_r,
"t": static_t,
"out": static_out,
"graph": graph,
}
def __call__(self, intr, c2w_r, c2w_t, H, W, ts=None):
# ts is ignored β€” the baked render path doesn't use it.
V = int(intr.shape[1])
key = (int(H), int(W), V)
with self._lock:
entry = self._graphs.get(key)
if entry is None:
if len(self._graphs) >= self.max_graphs:
# LRU: dicts preserve insertion order, pop the oldest.
oldest = next(iter(self._graphs))
self._graphs.pop(oldest)
entry = self._capture(intr, c2w_r, c2w_t, H, W)
self._graphs[key] = entry
entry["intr"].copy_(intr)
entry["r"].copy_(c2w_r)
entry["t"].copy_(c2w_t)
entry["graph"].replay()
return entry["out"].clone()
def install_optimizations(renderer, use_fast_plucker: bool = True,
use_cuda_graph: bool = True,
n_warmup: int = 5,
max_graphs: int = 3):
"""Apply both optimizations to a NeuralRenderer (in place + wrap). Returns
the (possibly wrapped) renderer that the viewer should call from now on.
Safe to call once at viewer startup; never invoked from training."""
if use_fast_plucker:
renderer.model.plucker_enc = FastPluckerEncoding(renderer.model.plucker_enc)
if use_cuda_graph:
renderer = GraphRenderer(renderer, n_warmup=n_warmup, max_graphs=max_graphs)
return renderer