"""The demo's solves. Every mode keeps the solver state (a Session), so it can be continued or extended with new photos (solved with the scene, or against the scene held fixed), and records snapshots of the state for the viewer to play the solve back.""" from __future__ import annotations import math import os import re import shutil import subprocess import tempfile import time import zipfile from dataclasses import dataclass, field from types import SimpleNamespace import numpy as np import torch from solv3r import PRESETS, Solv3R, StreamingSolver, cameras_from_pose_enc, load_images from solv3r.export import write_ply from solv3r.images import _ffmpeg, model_size from solv3r.model import Context, autocast from solv3r.network.block import _attn_kv, _attn_with_cache from solv3r.viewer.pack import frame_of, pack_solve, pack_stream, solve_export, stream_export DEV = "cuda" if torch.cuda.is_available() or os.environ.get("SPACES_ZERO_GPU") else "cpu" SNAPS = (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024) OUT = os.environ.get("SOLV3R_DEMO_TMP", os.path.join(tempfile.gettempdir(), "solv3r_demo")) os.makedirs(OUT, exist_ok=True) _model = None CACHE = os.environ.get("SOLV3R_DEMO_CACHE", os.path.join(os.path.expanduser("~"), ".cache", "solv3r")) def sky_of(images, grow=6): """(V, H, W) bool, True on sky (the sky segmentation of solv3r.export, model cached in CACHE), grown by `grow` pixels so that the band along the horizon goes too.""" import cv2 import solv3r.network.skyseg as S from solv3r.export import sky_mask os.makedirs(CACHE, exist_ok=True) S.SKYSEG_ONNX_PATH = os.environ.get("SOLV3R_SKYSEG", os.path.join(CACHE, "skyseg.onnx")) images = images.float() / 255 if images.dtype == torch.uint8 else images.float() m = np.concatenate([sky_mask(images[a:a + 16]) for a in range(0, len(images), 16)]) if grow: k = np.ones((2 * grow + 1, 2 * grow + 1), np.uint8) m = np.stack([cv2.dilate(x.astype(np.uint8), k) > 0 for x in m]) return m def model(): global _model if _model is None: _model = Solv3R.from_pretrained(os.environ.get("SOLV3R_CHECKPOINT"), device=DEV) return _model def sync(): if DEV == "cuda": torch.cuda.synchronize() def new_dir(tag): d = tempfile.mkdtemp(prefix=f"{tag}_", dir=OUT) return d # ---------------------------------------------------------------------- inputs def video_duration(path): r = subprocess.run([_ffmpeg(), "-i", path], capture_output=True, text=True) m = re.search(r"Duration: (\d+):(\d+):(\d+\.?\d*)", r.stderr) return int(m[1]) * 3600 + int(m[2]) * 60 + float(m[3]) if m else None def video_info(path): r = subprocess.run([_ffmpeg(), "-i", path], capture_output=True, text=True) dur = video_duration(path) fps = re.search(r"(\d+(?:\.\d+)?) fps", r.stderr) size = re.search(r"Video: [^\n]*?(\d{2,5})x(\d{2,5})", r.stderr) return dict(duration=dur, fps=float(fps[1]) if fps else None, size=(int(size[1]), int(size[2])) if size else None) def extract_frames(video, n, long_side=1008, max_secs=None): """n frames spread evenly over the video (or its first max_secs), as .jpg files (long side at most long_side).""" d = new_dir("frames") dur = video_duration(video) # unknown for browser recordings (webm): take every frame if dur and max_secs: dur = min(dur, max_secs) scale = f"scale='if(gt(iw,ih),min({long_side},iw),-2)':'if(gt(iw,ih),-2,min({long_side},ih))'" vf = f"fps={max(n / max(dur, 1e-3), 0.05):.5f},{scale}" if dur else scale cut = ["-t", str(max_secs)] if max_secs else [] subprocess.run([_ffmpeg(), "-loglevel", "error", "-i", video, *cut, "-vf", vf, "-q:v", "2", os.path.join(d, "%05d.jpg")], check=True) paths = sorted(os.path.join(d, f) for f in os.listdir(d) if f.endswith(".jpg")) if len(paths) > n: paths = [paths[i] for i in np.linspace(0, len(paths) - 1, n).round().astype(int)] return paths def depth_edges(d, thr=0.04): """(S, H, W) depth -> bool mask of depth discontinuities (relative central difference above thr): their pixels land between the surfaces ("flying pixels").""" g = torch.zeros_like(d) gx = (d[:, :, 2:] - d[:, :, :-2]).abs() / 2 gy = (d[:, 2:, :] - d[:, :-2, :]).abs() / 2 g[:, :, 1:-1] = gx g[:, 1:-1, :] = torch.maximum(g[:, 1:-1, :], gy) return g / d.clamp_min(1e-6) > thr def to_u8(images): return images if images.dtype == torch.uint8 else (images.clamp(0, 1) * 255).round().byte() def thumbs_of(images, h=96): """(S, 3, H, W) in [0, 1] -> (S, h, w, 3) uint8.""" S, _, H, W = images.shape w = max(1, round(h * W / H)) images = images.float() / 255 if images.dtype == torch.uint8 else images.float() t = torch.nn.functional.interpolate(images, size=(h, w), mode="area") return (t.permute(0, 2, 3, 1).clamp(0, 1) * 255).byte().numpy() # ---------------------------------------------------------------------- the kept solve @dataclass class Session: """Everything a solve leaves behind, on the CPU. Views are in arrival order (row i of the state).""" images: torch.Tensor # (V, 3, H, W) uint8 labels: list paths: list x0: torch.Tensor | None = None # (V, P, C) encodings pos: torch.Tensor | None = None z: torch.Tensor | None = None # (V, P, C) state first_call: list = field(default_factory=list) # call at which each view joined (0 = from the start) calls: int = 0 # calls made so far residual: list = field(default_factory=list) # (call, relative change) of every call ms_per_call: list = field(default_factory=list) kind: str = "solve" # "solve" | "stream" final: dict | None = None # last decode: c2w, K, depth, conf, world points (numpy) stream: dict | None = None # streaming result frame: tuple | None = None # viewer frame (centre, radius, rotation, aerial), kept across updates log: list = field(default_factory=list) peak_gb: float = 0.0 gpu: str = "" # the device that ran it (ZeroGPU assigns one per run) mode: str = "" sky: bool = False # leave the sky out of the points (outdoor scenes) source: str = "" # what was solved (an example's name) skymask: np.ndarray | None = None # (V, H, W) True on sky, computed once view_keep: float = 1.0 # share of each view's points the viewer shows (its point budget): exports keep it hist: object = None # (Recorder, first_snap) of the last solve shown: a continuation plays after it @property def V(self): return len(self.labels) @property def imf(self): return self.images.float() / 255 @property def hw(self): return tuple(self.images.shape[-2:]) def ctx(self): return Context(self.x0.to(DEV), self.pos.to(DEV), *self.hw) # ---------------------------------------------------------------------- recording snapshots class Recorder: """Snapshots of the state for the viewer: dense points of a pixel subset of every view at some calls, cameras at every call (for the camera motion colour).""" def __init__(self, images, budget=2_000_000): V, _, H, W = images.shape self.H, self.W = H, W s = 1 while V * (H // s) * (W // s) > budget and s < 8: s += 1 self.stride = s ys, xs = np.mgrid[s // 2:H:s, s // 2:W:s] self.ys, self.xs = ys.ravel(), xs.ravel() self.images = images self.snaps = [] # dict(call, P (V, n, 3) with NaN for absent views, conf, depth) self.cams = {} # call -> (V_present, 4, 4) def cameras(self, m, z, ctx, call): with torch.no_grad(): c2w, K = cameras_from_pose_enc(m.decode_cameras(z, ctx), (ctx.H, ctx.W)) self.cams[call] = c2w.numpy() self.K = K.numpy() return self.cams[call] def dense(self, m, z, ctx, call, chunk=16): V = z.shape[0] n = len(self.ys) P = np.full((V, n, 3), np.nan, np.float32) conf = np.zeros((V, n), np.float32) depth = np.zeros((V, n), np.float32) edge = np.zeros((V, n), bool) yi, xi = torch.as_tensor(self.ys), torch.as_tensor(self.xs) with torch.no_grad(): for a in range(0, V, chunk): idx = list(range(a, min(V, a + chunk))) pred = m.decode(z[idx], ctx.subset(idx)) wp = pred["world_points"].float()[:, yi, xi].cpu().numpy() P[a:a + len(idx)] = wp conf[a:a + len(idx)] = pred["depth_conf"].float()[:, yi, xi].cpu().numpy() d = pred["depth"][..., 0].float() depth[a:a + len(idx)] = d[:, yi, xi].cpu().numpy() edge[a:a + len(idx)] = depth_edges(d)[:, yi, xi].cpu().numpy() c2w = self.cameras(m, z, ctx, call) self.snaps.append(dict(call=call, P=P, conf=conf, depth=depth, c2w=c2w, K=self.K.copy(), edge=edge)) def motion(self, call, V): """Rotation change (deg) of every view's camera over the last call.""" a, b = self.cams.get(call - 1), self.cams.get(call) out = np.full(V, np.nan, np.float32) if a is None or b is None: return out n = min(len(a), len(b)) Ra, Rb = a[:n, :3, :3], b[:n, :3, :3] Ra, Rb = Ra[0].T[None] @ Ra, Rb[0].T[None] @ Rb c = (np.einsum("nij,nij->n", Ra, Rb) - 1) / 2 out[:n] = np.degrees(np.arccos(np.clip(c, -1, 1))) return out def pack(self, path, labels, first_snap, title="", frame=None, n_pts=None, sky=False, exposure=1.0, horizon=None, final=None, mv=None): V = len(labels) if n_pts is None: # about 40 MB of positions in all, at most a million points n_pts = int(np.clip(40e6 / (6 * len(self.snaps)), 200_000, 1_000_000)) S = len(self.snaps) n = len(self.ys) P = np.full((S, V, n, 3), np.nan, np.float32) c2w = np.tile(np.eye(4, dtype=np.float32), (S, V, 1, 1)) motion = np.full((S, V), np.nan, np.float32) for k, s in enumerate(self.snaps): v = s["P"].shape[0] P[k, :v] = s["P"] c2w[k, :v] = s["c2w"] for j in range(v, V): # views not yet present: their first camera (hidden by the viewer) c2w[k, j] = np.nan motion[k] = self.motion(s["call"], V) for j in range(V): # absent views take their first known camera known = [k for k in range(S) if np.isfinite(c2w[k, j]).all()] for k in range(S): if not np.isfinite(c2w[k, j]).all(): c2w[k, j] = c2w[known[0], j] if known else np.eye(4) last = self.snaps[-1] K = last["K"] vid = np.repeat(np.arange(V), n) img = to_u8(self.images).permute(0, 2, 3, 1).numpy() rgb = lift(img[:, self.ys, self.xs].reshape(-1, 3), exposure) conf = last["conf"].reshape(-1).astype(np.float32) # raw: ranked over the whole scene in pack_solve foot = (last["depth"] / K[:, 0, 0, None] * self.stride * 1.1).reshape(-1) calls = [s["call"] for s in self.snaps] P = P.reshape(S, V * n, 3) keep = ~last.get("edge", np.zeros((V, n), bool)).reshape(-1) # depth edges (flying pixels) leave if sky: # and the sky keep &= ~sky_of(self.images)[:, self.ys, self.xs].reshape(-1) if horizon is not None: # aerial examples: nothing seen near or above the horizon keep &= below_horizon(P[-1], c2w[-1][vid, :3, 3], horizon) if mv and final is not None: # floaters leave: a point stays when enough other views see its depth keep &= agreeing_views(P[-1], vid, final) >= mv P, vid, rgb, foot, conf = P[:, keep], vid[keep], rgb[keep], foot[keep], conf[keep] meta = pack_solve(path, P, vid, rgb, foot, conf, c2w, calls, first_snap, K, (self.H, self.W), thumbs_of(self.images), labels, motion=motion, n_pts=n_pts, title=title, frame=frame) return meta def snap_calls(total, start=0): """Calls after `start` at which to keep dense snapshots, up to `total` (always the last).""" out = [start + c for c in SNAPS if c < total - start] if total not in out: out.append(total) return out def finish(sess, m, z, ctx, keep_final=True): """Final decode kept for the exports and the views panel.""" sess.z = z.float().cpu() if not keep_final: return with torch.no_grad(): c2w, K = cameras_from_pose_enc(m.decode_cameras(z, ctx), (ctx.H, ctx.W)) depth, conf, wp = [], [], [] for a in range(0, z.shape[0], 16): idx = list(range(a, min(z.shape[0], a + 16))) pred = m.decode(z[idx], ctx.subset(idx)) depth.append(pred["depth"][..., 0].float().cpu().numpy().astype(np.float16)) conf.append(pred["depth_conf"].float().cpu().numpy().astype(np.float16)) wp.append(pred["world_points"].float().cpu().numpy().astype(np.float16)) sess.final = dict(c2w=c2w.numpy(), K=K.numpy(), depth=np.concatenate(depth), conf=np.concatenate(conf), points=np.concatenate(wp)) def _peak(sess): if DEV == "cuda": sess.peak_gb = max(sess.peak_gb, torch.cuda.max_memory_allocated() / 2 ** 30) sess.gpu = torch.cuda.get_device_name(0).replace("NVIDIA ", "") # ---------------------------------------------------------------------- calls with bookkeeping def run_calls(sess, m, z, ctx, n, rec=None, dense_at=(), adaptive=None, min_calls=4, progress=None, offset=0): """n calls from z (or fewer with adaptive), recording cameras at every call and dense snapshots at dense_at.""" done = 0 for i in range(n): sync() t0 = time.perf_counter() with torch.no_grad(): zn = m.solve(ctx, 1, z) sync() sess.ms_per_call.append((time.perf_counter() - t0) * 1000) change = float((zn.float() - z.float()).norm() / z.float().norm().clamp_min(1e-9)) z = zn done += 1 sess.calls += 1 sess.residual.append((sess.calls, change)) if rec is not None: if sess.calls in dense_at: rec.dense(m, z, ctx, sess.calls) else: rec.cameras(m, z, ctx, sess.calls) if progress: progress((offset + done) / max(offset + n, 1), f"call {sess.calls}") if adaptive and done >= min_calls and change < adaptive: break if rec is not None and (not rec.snaps or rec.snaps[-1]["call"] != sess.calls): rec.dense(m, z, ctx, sess.calls) return z # ---------------------------------------------------------------------- modes def solve_offline(images, labels, paths, calls=16, adaptive=None, max_calls=64, progress=None): m = model() sess = Session(images=to_u8(images), labels=labels, paths=paths, mode="offline") if DEV == "cuda": torch.cuda.reset_peak_memory_stats() t0 = time.perf_counter() ctx = m.encode(images) sess.x0, sess.pos = ctx.x0.cpu(), ctx.pos.cpu() sess.first_call = [0] * len(labels) z = m.start(ctx) rec = Recorder(images) rec.cameras(m, z, ctx, 0) total = max_calls if adaptive else calls # adaptive stopping decides when to stop, up to max_calls z = run_calls(sess, m, z, ctx, total, rec, set(snap_calls(total)), adaptive, progress=progress) finish(sess, m, z, ctx) _peak(sess) sess.log.append(f"Offline: {sess.V} views, {sess.calls} calls, {(time.perf_counter() - t0):.1f} s") return sess, rec def stages_of(n, initial, group, order="file", seed=0): """Stages of view indices: the first `initial`, then groups; order 'file' or 'random' (any order).""" idx = np.arange(n) if order == "random": rng = np.random.default_rng(seed) first = np.linspace(0, n - 1, min(initial, n)).round().astype(int) # spread over the capture rest = rng.permutation(np.setdiff1d(idx, first)) idx = np.concatenate([first, rest]) stages = [idx[:initial].tolist()] for a in range(initial, n, group): stages.append(idx[a:a + group].tolist()) return stages def solve_stages(images, labels, paths, stages, calls=8, progress=None, mode="incremental"): """Views join in stages; stage i gets calls[i] calls (or `calls` each), which also update the views solved before.""" cs = list(calls) if isinstance(calls, (list, tuple)) else [calls] * len(stages) m = model() order = [i for s in stages for i in s] images, labels, paths = images[order], [labels[i] for i in order], [paths[i] for i in order] sess = Session(images=to_u8(images), labels=labels, paths=paths, mode=mode) if DEV == "cuda": torch.cuda.reset_peak_memory_stats() t0 = time.perf_counter() rec = Recorder(images) first_snap, n_done = [], 0 total = sum(cs) for i, st in enumerate(stages): rows = list(range(n_done, n_done + len(st))) if i == 0: ctx = m.encode(images[rows]) z = m.start(ctx) rec.cameras(m, z, ctx, 0) else: z, ctx = m.add_views(z, ctx, images[rows]) sess.first_call += [sess.calls] * len(st) first_snap += [len(rec.snaps)] * len(st) n_done += len(st) dense = {sess.calls + 1, sess.calls + cs[i]} z = run_calls(sess, m, z, ctx, cs[i], rec, dense, progress=progress, offset=sum(cs[:i])) if progress: progress(sum(cs[:i + 1]) / total, f"{n_done} of {len(order)} views") sess.x0, sess.pos = ctx.x0.cpu(), ctx.pos.cpu() finish(sess, m, z, ctx) _peak(sess) sess.log.append(f"{mode.capitalize()}: {sess.V} views in {len(stages)} stages, {sess.calls} calls, " f"{(time.perf_counter() - t0):.1f} s") return sess, rec, first_snap HIST_BYTES = 600e6 # snapshots kept for a continuation to play after, at most HIST_SNAPS = 2 # of which a continuation's timeline shows about this many (each costs points) def with_history(hist, rec, first): """A continuation's snapshots after those of the solve it continues (hist: its Recorder and first_snap), so its timeline shows the whole solve (views that join appear at their call). rec's first snapshot is the state the kept solve ended in: it is dropped.""" prev, pfirst = hist size = sum(v.nbytes for s in prev.snaps + rec.snaps for v in s.values() if isinstance(v, np.ndarray)) if size > HIST_BYTES or (prev.H, prev.W) != (rec.H, rec.W): return rec, first grid = {(y, x): i for i, (y, x) in enumerate(zip(prev.ys.tolist(), prev.xs.tolist()))} idx = [grid.get(p) for p in zip(rec.ys.tolist(), rec.xs.tolist())] # rec's pixels on prev's grid (a coarser if any(i is None for i in idx): # stride: a subset of a finer one) return rec, first idx = np.array(idx) S = len(prev.snaps) # a few of them (the snapshots cost the browser points): ends and joins kept keep = sorted({0, S - 1, *pfirst, *np.linspace(0, S - 1, HIST_SNAPS).round().astype(int).tolist()}) at = {j: i for i, j in enumerate(keep)} k, n_old = len(keep), len(pfirst) pick = lambda d: {k: (v[:, idx] if isinstance(v, np.ndarray) and v.ndim >= 2 and k != "c2w" and k != "K" else v) for k, v in d.items()} rec.snaps = [pick(prev.snaps[j]) for j in keep] + rec.snaps[1:] rec.cams = {**prev.cams, **rec.cams} return rec, [at[f] for f in pfirst] + [k - 1 + f for f in first[n_old:]] def continue_solve(sess, calls, progress=None): """More calls on the kept state (all views).""" m = model() ctx = sess.ctx() z = sess.z.to(DEV) rec = Recorder(sess.imf) start = sess.calls rec.cameras(m, z, ctx, start) rec.dense(m, z, ctx, start) dense = set(snap_calls(start + calls, start)) z = run_calls(sess, m, z, ctx, calls, rec, dense, progress=progress) finish(sess, m, z, ctx) _peak(sess) sess.log.append(f"Continued: +{calls} calls (now {sess.calls})") return sess, rec def add_photos(sess, images, labels, paths, calls=8, progress=None): """New views join the kept solve; all views are updated.""" m = model() images = load_resized(images, sess.hw) ctx = sess.ctx() z = sess.z.to(DEV) rec = Recorder(torch.cat([sess.imf, images])) start = sess.calls rec.cameras(m, z, ctx, start) rec.dense(m, z, ctx, start) z, ctx = m.add_views(z, ctx, images) n_old = sess.V sess.images = torch.cat([sess.images, to_u8(images)]) sess.labels += labels sess.paths += paths sess.first_call += [start] * len(labels) sess.x0, sess.pos = ctx.x0.cpu(), ctx.pos.cpu() dense = set(snap_calls(start + calls, start)) z = run_calls(sess, m, z, ctx, calls, rec, dense, progress=progress) finish(sess, m, z, ctx) _peak(sess) first_snap = [0] * n_old + [1] * len(labels) sess.log.append(f"Added {len(labels)} photos at call {start}, +{calls} calls") return sess, rec, first_snap def load_resized(images, hw): if tuple(images.shape[-2:]) == tuple(hw): return images from solv3r.images import resize return resize(images, hw) def add_fixed(sess, images, labels, paths, calls=16, progress=None): """New views join with the scene held fixed: the kept views enter every call through one key / value cache and only the new views are solved, together. They then belong to the scene like any other view.""" m = model() images = load_resized(images, sess.hw) ctx = sess.ctx() z = sess.z.to(DEV) rec = Recorder(torch.cat([sess.imf, images])) start, n_old = sess.calls, sess.V rec.cameras(m, z, ctx, start) rec.dense(m, z, ctx, start) sync() t0 = time.perf_counter() with torch.no_grad(), autocast(): cache = m.cache(z, ctx) sync() t_cache = time.perf_counter() - t0 new = m.encode(images, reference_first=False) zq = m.start(new) ctx_all = Context.cat([ctx, new]) dense = set(snap_calls(start + calls, start)) ms = [] for i in range(calls): sync() t1 = time.perf_counter() with torch.no_grad(), autocast(): zn = m.step_cached(zq, new, cache) sync() ms.append((time.perf_counter() - t1) * 1000) change = float((zn.float() - zq.float()).norm() / zq.float().norm().clamp_min(1e-9)) zq = zn sess.calls += 1 sess.residual.append((sess.calls, change)) z_all = torch.cat([z, zq.to(z.dtype)]) if sess.calls in dense: rec.dense(m, z_all, ctx_all, sess.calls) else: rec.cameras(m, z_all, ctx_all, sess.calls) if progress: progress((i + 1) / calls, f"call {sess.calls}") z_all = torch.cat([z, zq.to(z.dtype)]) if not rec.snaps or rec.snaps[-1]["call"] != sess.calls: rec.dense(m, z_all, ctx_all, sess.calls) sess.images = torch.cat([sess.images, to_u8(images)]) sess.labels += labels sess.paths += paths sess.first_call += [start] * len(labels) sess.x0, sess.pos = ctx_all.x0.cpu(), ctx_all.pos.cpu() finish(sess, m, z_all, ctx_all) _peak(sess) first_snap = [0] * n_old + [1] * len(labels) sess.log.append(f"Added {len(labels)} photos with the scene frozen: one cache of {n_old} views ({t_cache * 1000:.0f} ms), " f"{calls} calls on the new views ({np.median(ms):.1f} ms per call)") return sess, rec, first_snap def solve_stream(paths, labels, config="L8", progress=None, keep=0.5, per_frame=2500): m = model() size = load_images(paths[:1]).shape[-2:] cfg = PRESETS[config] import dataclasses cfg = dataclasses.replace(cfg, memory_stride=min(cfg.memory_stride, max(1, -(-len(paths) // cfg.memory)))) solver = StreamingSolver(m, cfg, depth=True) if DEV == "cuda": torch.cuda.reset_peak_memory_stats() frames, live = [], [] rng = np.random.default_rng(0) for i, p in enumerate(paths): im = load_images([p], size)[0] frames.append(to_u8(im)) for f in solver.push(im): # each frame answered now: its camera and a few of its points live.append(live_frame(solver, f, frames[f], rng)) if progress and (i % 2 == 0 or i == len(paths) - 1): progress((i + 1) / len(paths), f"frame {i + 1} of {len(paths)} answered", live) live = [] res = solver.result() images = torch.stack(frames) sess = Session(images=images, labels=labels, paths=paths, kind="stream", mode="streaming") sess.calls = len(paths) sess.stream = dict(c2w=res.c2w, c2w_exit=res.c2w_exit, K=res.intrinsics, depth=res.depth, conf=res.depth_conf, ms=res.ms_per_frame, refreshed=list(res.refreshed), config=config) sess.final = dict(c2w=res.c2w_exit, K=res.intrinsics, depth=res.depth.astype(np.float16), conf=res.depth_conf.astype(np.float16), points=None) _peak(sess) sess.log.append(f"Streaming ({config}): {len(paths)} frames, {res.ms_per_frame:.1f} ms per frame (solver), " f"memory refreshed at {sum(res.refreshed)} frames") return sess def live_frame(solver, f, image, rng, n=1500, stride=3): """What the live view draws for a frame on arrival: camera centre, forward and up (world), intrinsics (fractions of the image), n of its confident points (float16 xyz + uint8 rgb + float16 footprint, base64) and a small JPEG. The points and their footprints are chosen as in pack_stream_session, so the live cloud looks like the final one.""" import base64 import cv2 R, C, K = solver.R[f].float().numpy(), solver.C[f].float().numpy(), solver.K[f].float().numpy() img = image.permute(1, 2, 0).numpy() H, W = img.shape[:2] out = [int(f)] + np.round(np.concatenate([C, R[:, 2], -R[:, 1]]), 5).tolist() out += [round(float(K[0, 0] / W), 5), round(float(K[1, 1] / H), 5), round(float(K[0, 2] / W), 5), round(float(K[1, 2] / H), 5)] pts, rgb, foot = b"", b"", b"" if f in solver.depths: d, c = solver.depths[f], solver.confs[f] h, w = d.shape ys, xs = np.mgrid[stride // 2:h:stride, stride // 2:w:stride] ys, xs = ys.ravel(), xs.ravel() z, cf = d[ys, xs], c[ys, xs] ok = np.isfinite(z) & (z > 0) & ~depth_edges(torch.as_tensor(d[None]).float())[0].numpy()[ys, xs] if ok.any(): ok &= cf >= np.quantile(cf[ok], 0.5) idx = np.nonzero(ok)[0] if len(idx) > n: idx = rng.choice(idx, n, replace=False) if len(idx): Kd = K * np.array([[w / W], [h / H], [1]]) # intrinsics at the depth resolution zz = z[idx] X = np.stack([(xs[idx] + 0.5 - Kd[0, 2]) / Kd[0, 0] * zz, (ys[idx] + 0.5 - Kd[1, 2]) / Kd[1, 1] * zz, zz], 1) pts = (X @ R.T + C).astype(np.float16).tobytes() rgb = img[(ys[idx] * H) // h, (xs[idx] * W) // w].astype(np.uint8).tobytes() dens = min(1.4, np.sqrt(np.count_nonzero(ok) / len(idx))) foot = (zz / Kd[0, 0] * stride * 1.1 * dens).astype(np.float16).tobytes() th = cv2.resize(img, (96, max(1, round(96 * H / W))), interpolation=cv2.INTER_AREA) ok_, jpg = cv2.imencode(".jpg", cv2.cvtColor(th, cv2.COLOR_RGB2BGR), [cv2.IMWRITE_JPEG_QUALITY, 70]) b64 = lambda b: base64.b64encode(b).decode() return out + [b64(pts), b64(rgb), b64(foot), b64(jpg.tobytes())] def consistent(P, f, S, tol=0.04): """Points of frame f that agree with the depth of frames f +- 2 and f +- 4 (exit states) where those frames see them.""" F_, H, W = S["depth"].shape ok, seen = np.zeros(len(P), int), np.zeros(len(P), int) for g in (f - 4, f - 2, f + 2, f + 4): if g < 0 or g >= F_: continue w2c = np.linalg.inv(S["c2w_exit"][g]); Q = P @ w2c[:3, :3].T + w2c[:3, 3]; zq = Q[:, 2] K = S["K"][g] u = K[0, 0] * Q[:, 0] / np.maximum(zq, 1e-6) + K[0, 2]; v = K[1, 1] * Q[:, 1] / np.maximum(zq, 1e-6) + K[1, 2] i = np.nonzero((zq > 1e-4) & (u >= 0) & (u < W - 1) & (v >= 0) & (v < H - 1))[0] dz = np.abs(S["depth"][g][v[i].astype(int), u[i].astype(int)] - zq[i]) / zq[i] seen[i] += 1; ok[i] += dz < tol return (seen == 0) | (ok * 2 >= seen) def agreeing_views(P, vid, final, tol=0.03): """For each point (N, 3) of view vid, the number of other views whose final depth map agrees with it (relative tol).""" D, K, W2C = final["depth"], final["K"], np.linalg.inv(final["c2w"].astype(np.float64)) V, H, W = D.shape cnt = np.zeros(len(P), np.int16) fin = np.isfinite(P).all(1) for i in range(V): X = P @ W2C[i, :3, :3].T.astype(np.float32) + W2C[i, :3, 3].astype(np.float32) z = X[:, 2] with np.errstate(all="ignore"): u = K[i, 0, 0] * X[:, 0] / z + K[i, 0, 2] v = K[i, 1, 1] * X[:, 1] / z + K[i, 1, 2] idx = np.nonzero(fin & (vid != i) & (z > 0) & (u >= 0) & (u < W) & (v >= 0) & (v < H))[0] d = D[i, v[idx].astype(int), u[idx].astype(int)].astype(np.float32) cnt[idx] += np.abs(d - z[idx]) < tol * z[idx] return cnt def below_horizon(P, C, deg=3.0): """Aerial scenes: points seen at least `deg` below the horizon from their camera (P points, C their cameras' centres). Distant hills, city lights and the sky that the sky mask misses come back with a compressed depth and float above the map; they are seen near or above the horizon. Up is the ground normal (smallest principal axis of the points, toward the cameras), not the cameras' -y, which the drone's pitch tilts (as the project page's demo videos).""" ok = np.isfinite(P).all(1) Q = P[ok][np.random.default_rng(0).choice(int(ok.sum()), min(int(ok.sum()), 200000), replace=False)].astype(np.float64) c = np.median(Q, 0) lo, hi = np.quantile(Q, 0.05, 0), np.quantile(Q, 0.95, 0) Q = Q[((Q >= lo) & (Q <= hi)).all(1)] - c up = np.linalg.eigh(Q.T @ Q / len(Q))[1][:, 0] up = up if (np.nanmean(C, 0) - c) @ up > 0 else -up ray = P - C el = np.degrees(np.arcsin(np.clip((ray @ up) / np.maximum(np.linalg.norm(ray, axis=1), 1e-9), -1, 1))) return ok & (el < -deg) def lift(rgb, exposure): """Display brightness of point colours (night scenes): a gamma lift, dark tones raised most, nothing clipped.""" if exposure == 1: return rgb return np.round(255 * (np.asarray(rgb, np.float32) / 255) ** (1 / exposure)).astype(np.uint8) def facing(depth, K, ys, xs): """|cos| of the angle between each pixel's viewing ray and the surface it sees (normal from the depth map's neighbours): small where the frame sees the surface at a grazing angle, where its depth is least precise.""" H, W = depth.shape v, u = np.mgrid[0:H, 0:W] X = np.stack([(u + 0.5 - K[0, 2]) / K[0, 0] * depth, (v + 0.5 - K[1, 2]) / K[1, 1] * depth, depth], -1) du, dv = np.zeros_like(X), np.zeros_like(X) du[:, 1:-1] = X[:, 2:] - X[:, :-2] dv[1:-1] = X[2:] - X[:-2] n = np.cross(du, dv)[ys, xs] r = X[ys, xs] return np.abs((n * r).sum(1)) / np.maximum(np.linalg.norm(n, axis=1) * np.linalg.norm(r, axis=1), 1e-12) def first_seen(P, fid, vox=0.006, slack=2): """Points in space that no earlier frame (more than `slack` frames before) already covered: a later, less precise view of the same surface does not blur what an earlier frame saw sharply; later frames only extend the map (voxels of `vox` x the scene radius).""" c = np.median(P, 0) r = float(np.quantile(np.linalg.norm(P - c, axis=1), 0.9)) key = np.floor((P - c) / (vox * r)).astype(np.int64) key = (key[:, 0] * 73856093) ^ (key[:, 1] * 19349663) ^ (key[:, 2] * 83492791) order = np.argsort(fid, kind="stable") _, first_idx = np.unique(key[order], return_index=True) first = dict(zip(key[order][first_idx].tolist(), fid[order][first_idx].tolist())) return fid <= np.array([first[k] for k in key.tolist()]) + slack def pack_stream_session(sess, path, keep=0.9, per_frame=8000, stride=3, sky=False, exposure=1.0, horizon=None, graze=None, firstseen=False): # the viewer's confidence slider does the rest S = sess.stream skym = sky_of(sess.images) if sky else None F_, H, W = S["depth"].shape ys, xs = np.mgrid[stride // 2:H:stride, stride // 2:W:stride] ys, xs = ys.ravel(), xs.ravel() img = sess.images.permute(0, 2, 3, 1).numpy() rng = np.random.default_rng(0) Pn, Pl, col, fid, foot, cf = [], [], [], [], [], [] cand = 0 for f in range(F_): d, c = S["depth"][f][ys, xs], S["conf"][f][ys, xs] ok = np.isfinite(d) & (d > 0) & ~depth_edges(torch.as_tensor(S["depth"][f][None]).float())[0].numpy()[ys, xs] if skym is not None: ok &= ~skym[f][ys, xs] thr = np.quantile(c[ok], 1 - keep) if ok.any() else 0 K = S["K"][f] if graze is not None: # nothing seen at a grazing angle: it would blur what frontal frames saw sharply ok &= facing(S["depth"][f], K, ys, xs) > graze idx = np.nonzero(ok & (c >= thr))[0] z = d[idx] X = np.stack([(xs[idx] + 0.5 - K[0, 2]) / K[0, 0] * z, (ys[idx] + 0.5 - K[1, 2]) / K[1, 1] * z, z], 1) good = consistent(X @ S["c2w_exit"][f][:3, :3].T + S["c2w_exit"][f][:3, 3], f, S) # as the project page's streams idx, z, X = idx[good], z[good], X[good] cand += len(idx) if len(idx) > per_frame: sub = np.sort(rng.choice(len(idx), per_frame, replace=False)) idx, z, X = idx[sub], z[sub], X[sub] for out, C in ((Pn, S["c2w"][f]), (Pl, S["c2w_exit"][f])): out.append(X @ C[:3, :3].T + C[:3, 3]) col.append(img[f, ys[idx], xs[idx]]) fid.append(np.full(len(idx), f)) foot.append(z / K[0, 0] * stride * 1.1) cf.append(c[idx]) cat = np.concatenate Pn, Pl, fid, col, foot, cf = cat(Pn), cat(Pl), cat(fid), cat(col), cat(foot), cat(cf) if firstseen: # later frames only extend the map k = first_seen(Pl, fid) Pn, Pl, fid, col, foot, cf = Pn[k], Pl[k], fid[k], col[k], foot[k], cf[k] if horizon is not None: # aerial examples: nothing seen near or above the horizon k = below_horizon(Pl, S["c2w_exit"][fid][:, :3, 3], horizon) Pn, Pl, fid, col, foot, cf = Pn[k], Pl[k], fid[k], col[k], foot[k], cf[k] return pack_stream(path, Pn, Pl, fid, lift(col, exposure), foot, S["c2w"], S["c2w_exit"], S["K"], sess.hw, thumbs_of(sess.images, 64), labels=sess.labels, dens=min(2.0, np.sqrt(cand / max(len(fid), 1))), conf=cf) # ---------------------------------------------------------------------- large scenes: a random subset of views per call class Stochastic: """Each call updates b random views; the others join as constraints through their cached keys / values. Per view we keep the global-attention keys / values of its current state and the energy of its latest update (for the state scale), so a call costs b views of frame attention plus their attention over the cache. Peak memory grows with the number of views only through the cache (about 2.7 MB per view).""" def __init__(self, m, ctx, chunk=16): self.m, self.ctx, self.blk = m, ctx, m.net.block self.N, self.P, self.C = ctx.x0.shape attn = self.blk.global_attn.attn self.Hh = attn.num_heads self.z = m.start(ctx).float() self.K = torch.empty(self.N, self.Hh, self.P, self.C // self.Hh, dtype=torch.bfloat16, device=DEV) self.Vv = torch.empty_like(self.K) tgt = float(self.blk.target_rms[3]) self.e = torch.full((self.N,), tgt ** 2 * self.P * self.C, device=DEV) self.updates = torch.zeros(self.N, dtype=torch.long) for a in range(0, self.N, chunk): self._kv(list(range(a, min(self.N, a + chunk)))) @torch.no_grad() def _kv(self, S): b = len(S) with autocast(): y = self.blk._frame(self.z[S], self.ctx.x0[S], self.ctx.pos[S]).reshape(1, b * self.P, self.C) k, v = _attn_kv(self.blk.global_attn.attn, self.blk.global_attn.norm1(y)) self.K[S] = k[0].reshape(self.Hh, b, self.P, -1).permute(1, 0, 2, 3).to(torch.bfloat16) self.Vv[S] = v[0].reshape(self.Hh, b, self.P, -1).permute(1, 0, 2, 3).to(torch.bfloat16) @torch.no_grad() def update(self, S): b = len(S) held = torch.ones(self.N, dtype=torch.bool, device=DEV) held[S] = False nh = int(held.sum()) Kh = self.K[held].permute(1, 0, 2, 3).reshape(1, self.Hh, nh * self.P, -1) Vh = self.Vv[held].permute(1, 0, 2, 3).reshape(1, self.Hh, nh * self.P, -1) attn = self.blk.global_attn.attn with autocast(): y = self.blk._frame(self.z[S], self.ctx.x0[S], self.ctx.pos[S]).reshape(1, b * self.P, self.C) y = self.blk.global_attn(y, self.blk.g_global, attn_fn=lambda x: _attn_with_cache(attn, x, Kh, Vh)) y = y.reshape(b, self.P, self.C) eq = y.float().pow(2).sum((1, 2)) rms = ((self.e[held].sum() + eq.sum()) / (self.N * self.P * self.C)).clamp_min(1e-12).sqrt() zq = self.blk._rescale(y, rms).float() self.e[S] = eq self.z[S] = zq self.updates[S] += 1 self._kv(S) def solve_stochastic(images, labels, paths, per_call=8, updates=32, seed=0, progress=None): m = model() sess = Session(images=to_u8(images), labels=labels, paths=paths, mode="large") if DEV == "cuda": torch.cuda.reset_peak_memory_stats() t0 = time.perf_counter() ctx = m.encode(images) sess.x0, sess.pos = ctx.x0.cpu(), ctx.pos.cpu() sess.first_call = [0] * len(labels) N = len(labels) st = Stochastic(m, ctx) rec = Recorder(images) rec.cameras(m, st.z, ctx, 0) g = torch.Generator().manual_seed(seed) total = math.ceil(updates * N / per_call) per_epoch = max(1, N // per_call) dense_epochs = {1, 2, 4, 8, 16, 32, 64, updates} z_prev = st.z.clone() for c in range(1, total + 1): if (c - 1) % per_epoch == 0: torch.randperm(N, generator=g) S = torch.randperm(N, generator=g)[:per_call].tolist() sync() t1 = time.perf_counter() st.update(S) sync() sess.ms_per_call.append((time.perf_counter() - t1) * 1000) sess.calls = c if c % per_epoch == 0 or c == total: ep = c // per_epoch sess.residual.append((c, float((st.z - z_prev).norm() / z_prev.norm().clamp_min(1e-9)))) # per pass z_prev = st.z.clone() if ep in dense_epochs or c == total: rec.dense(m, st.z, ctx, c) else: rec.cameras(m, st.z, ctx, c) if progress: progress(c / total, f"{c} calls, {ep} updates per view") finish(sess, m, st.z, ctx) _peak(sess) sess.log.append(f"Large scene: {N} views, {per_call} per call, {total} calls (~{updates} updates per view), " f"{(time.perf_counter() - t0):.1f} s, peak {sess.peak_gb:.2f} GB") return sess, rec # ---------------------------------------------------------------------- exports def confident(sess, keep=1.0): """The points the viewer shows, at every pixel (solv3r.viewer.pack), and their colours: an export is what one sees. keep: the viewer's confidence filtering (1: all).""" f = sess.final if f.get("points") is None: # stream: the depth on arrival placed with the exit cameras S = sess.stream res = SimpleNamespace(c2w=S["c2w"], c2w_exit=S["c2w_exit"], intrinsics=S["K"], depth=S["depth"], depth_conf=S["conf"]) return stream_export(sess.images, res, keep, sess.sky) return solve_export(f["points"], f["conf"], f["depth"], sess.images, sess.view_keep, keep, sess.sky) def export_ply(sess, keep=1.0, max_points=8_000_000): xyz, rgb = confident(sess, keep) if len(xyz) > max_points: sel = np.random.default_rng(0).choice(len(xyz), max_points, replace=False) xyz, rgb = xyz[sel], rgb[sel] path = os.path.join(new_dir("export"), "solv3r_points.ply") write_ply(path, xyz, rgb) return path def export_npz(sess): f = sess.final path = os.path.join(new_dir("export"), "solv3r_cameras_depth.npz") np.savez_compressed(path, c2w=f["c2w"], K=f["K"], depth=f["depth"], depth_conf=f["conf"], names=np.array(sess.labels), image_size=np.array(sess.hw)) return path def _qvec(R): """Rotation matrix -> COLMAP quaternion (w, x, y, z).""" from scipy.spatial.transform import Rotation x, y, z, w = Rotation.from_matrix(R).as_quat() return w, x, y, z def export_colmap(sess, keep=1.0, max_points=300_000, with_images=True): """COLMAP text model (sparse/0: cameras, images, points3D) at the original image sizes, plus the images; for Gaussian splatting and other tools that start from a COLMAP model.""" from PIL import Image f = sess.final d = new_dir("colmap") sp = os.path.join(d, "sparse", "0") os.makedirs(sp) H, W = sess.hw cams, ims = [], [] names = [] for i in range(sess.V): src = sess.paths[i] try: w0, h0 = Image.open(src).size if src and os.path.isfile(src) else (W, H) except OSError: w0, h0 = W, H K = f["K"][i] sx, sy = w0 / W, h0 / H name = f"{i:05d}_{os.path.basename(src) if src else sess.labels[i] + '.jpg'}" names.append(name) cams.append(f"{i + 1} PINHOLE {w0} {h0} {K[0, 0] * sx:.6f} {K[1, 1] * sy:.6f} {K[0, 2] * sx:.6f} {K[1, 2] * sy:.6f}") w2c = np.linalg.inv(f["c2w"][i].astype(np.float64)) q = _qvec(w2c[:3, :3]) t = w2c[:3, 3] ims.append(f"{i + 1} {q[0]:.9f} {q[1]:.9f} {q[2]:.9f} {q[3]:.9f} {t[0]:.9f} {t[1]:.9f} {t[2]:.9f} {i + 1} {name}\n") open(os.path.join(sp, "cameras.txt"), "w").write("# Camera list: CAMERA_ID, MODEL, WIDTH, HEIGHT, PARAMS[]\n" + "\n".join(cams) + "\n") open(os.path.join(sp, "images.txt"), "w").write("# IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME\n" + "\n".join(ims)) xyz, rgb = confident(sess, keep) if len(xyz) > max_points: sel = np.random.default_rng(0).choice(len(xyz), max_points, replace=False) xyz, rgb = xyz[sel], rgb[sel] with open(os.path.join(sp, "points3D.txt"), "w") as fh: fh.write("# POINT3D_ID, X, Y, Z, R, G, B, ERROR, TRACK[]\n") for j, (p, c) in enumerate(zip(xyz, rgb)): fh.write(f"{j + 1} {p[0]:.6f} {p[1]:.6f} {p[2]:.6f} {c[0]} {c[1]} {c[2]} 0\n") if with_images: os.makedirs(os.path.join(d, "images")) for i, name in enumerate(names): src = sess.paths[i] if src and os.path.isfile(src): shutil.copy(src, os.path.join(d, "images", name)) else: Image.fromarray(sess.images[i].permute(1, 2, 0).numpy()).save(os.path.join(d, "images", name)) zp = os.path.join(new_dir("export"), "solv3r_colmap.zip") with zipfile.ZipFile(zp, "w", zipfile.ZIP_DEFLATED) as z: for root, _, files in os.walk(d): for fn in files: full = os.path.join(root, fn) z.write(full, os.path.relpath(full, d)) shutil.rmtree(d, ignore_errors=True) return zp