solv3r / engine.py
jgkwak's picture
Continuation history on its own grid
fe066dd verified
Raw History Blame Contribute Delete
44.2 kB
"""The demo's solves. Every mode keeps the solver state (a Session), so it can be continued or extended with new photos
(solved with the scene, or against the scene held fixed), and records snapshots of the state for the viewer to play the
solve back."""
from __future__ import annotations
import math
import os
import re
import shutil
import subprocess
import tempfile
import time
import zipfile
from dataclasses import dataclass, field
from types import SimpleNamespace
import numpy as np
import torch
from solv3r import PRESETS, Solv3R, StreamingSolver, cameras_from_pose_enc, load_images
from solv3r.export import write_ply
from solv3r.images import _ffmpeg, model_size
from solv3r.model import Context, autocast
from solv3r.network.block import _attn_kv, _attn_with_cache
from solv3r.viewer.pack import frame_of, pack_solve, pack_stream, solve_export, stream_export
DEV = "cuda" if torch.cuda.is_available() or os.environ.get("SPACES_ZERO_GPU") else "cpu"
SNAPS = (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024)
OUT = os.environ.get("SOLV3R_DEMO_TMP", os.path.join(tempfile.gettempdir(), "solv3r_demo"))
os.makedirs(OUT, exist_ok=True)
_model = None
CACHE = os.environ.get("SOLV3R_DEMO_CACHE", os.path.join(os.path.expanduser("~"), ".cache", "solv3r"))
def sky_of(images, grow=6):
"""(V, H, W) bool, True on sky (the sky segmentation of solv3r.export, model cached in CACHE), grown by `grow`
pixels so that the band along the horizon goes too."""
import cv2
import solv3r.network.skyseg as S
from solv3r.export import sky_mask
os.makedirs(CACHE, exist_ok=True)
S.SKYSEG_ONNX_PATH = os.environ.get("SOLV3R_SKYSEG", os.path.join(CACHE, "skyseg.onnx"))
images = images.float() / 255 if images.dtype == torch.uint8 else images.float()
m = np.concatenate([sky_mask(images[a:a + 16]) for a in range(0, len(images), 16)])
if grow:
k = np.ones((2 * grow + 1, 2 * grow + 1), np.uint8)
m = np.stack([cv2.dilate(x.astype(np.uint8), k) > 0 for x in m])
return m
def model():
global _model
if _model is None:
_model = Solv3R.from_pretrained(os.environ.get("SOLV3R_CHECKPOINT"), device=DEV)
return _model
def sync():
if DEV == "cuda":
torch.cuda.synchronize()
def new_dir(tag):
d = tempfile.mkdtemp(prefix=f"{tag}_", dir=OUT)
return d
# ---------------------------------------------------------------------- inputs
def video_duration(path):
r = subprocess.run([_ffmpeg(), "-i", path], capture_output=True, text=True)
m = re.search(r"Duration: (\d+):(\d+):(\d+\.?\d*)", r.stderr)
return int(m[1]) * 3600 + int(m[2]) * 60 + float(m[3]) if m else None
def video_info(path):
r = subprocess.run([_ffmpeg(), "-i", path], capture_output=True, text=True)
dur = video_duration(path)
fps = re.search(r"(\d+(?:\.\d+)?) fps", r.stderr)
size = re.search(r"Video: [^\n]*?(\d{2,5})x(\d{2,5})", r.stderr)
return dict(duration=dur, fps=float(fps[1]) if fps else None, size=(int(size[1]), int(size[2])) if size else None)
def extract_frames(video, n, long_side=1008, max_secs=None):
"""n frames spread evenly over the video (or its first max_secs), as .jpg files (long side at most long_side)."""
d = new_dir("frames")
dur = video_duration(video) # unknown for browser recordings (webm): take every frame
if dur and max_secs:
dur = min(dur, max_secs)
scale = f"scale='if(gt(iw,ih),min({long_side},iw),-2)':'if(gt(iw,ih),-2,min({long_side},ih))'"
vf = f"fps={max(n / max(dur, 1e-3), 0.05):.5f},{scale}" if dur else scale
cut = ["-t", str(max_secs)] if max_secs else []
subprocess.run([_ffmpeg(), "-loglevel", "error", "-i", video, *cut, "-vf", vf, "-q:v", "2",
os.path.join(d, "%05d.jpg")], check=True)
paths = sorted(os.path.join(d, f) for f in os.listdir(d) if f.endswith(".jpg"))
if len(paths) > n:
paths = [paths[i] for i in np.linspace(0, len(paths) - 1, n).round().astype(int)]
return paths
def depth_edges(d, thr=0.04):
"""(S, H, W) depth -> bool mask of depth discontinuities (relative central difference above thr): their pixels
land between the surfaces ("flying pixels")."""
g = torch.zeros_like(d)
gx = (d[:, :, 2:] - d[:, :, :-2]).abs() / 2
gy = (d[:, 2:, :] - d[:, :-2, :]).abs() / 2
g[:, :, 1:-1] = gx
g[:, 1:-1, :] = torch.maximum(g[:, 1:-1, :], gy)
return g / d.clamp_min(1e-6) > thr
def to_u8(images):
return images if images.dtype == torch.uint8 else (images.clamp(0, 1) * 255).round().byte()
def thumbs_of(images, h=96):
"""(S, 3, H, W) in [0, 1] -> (S, h, w, 3) uint8."""
S, _, H, W = images.shape
w = max(1, round(h * W / H))
images = images.float() / 255 if images.dtype == torch.uint8 else images.float()
t = torch.nn.functional.interpolate(images, size=(h, w), mode="area")
return (t.permute(0, 2, 3, 1).clamp(0, 1) * 255).byte().numpy()
# ---------------------------------------------------------------------- the kept solve
@dataclass
class Session:
"""Everything a solve leaves behind, on the CPU. Views are in arrival order (row i of the state)."""
images: torch.Tensor # (V, 3, H, W) uint8
labels: list
paths: list
x0: torch.Tensor | None = None # (V, P, C) encodings
pos: torch.Tensor | None = None
z: torch.Tensor | None = None # (V, P, C) state
first_call: list = field(default_factory=list) # call at which each view joined (0 = from the start)
calls: int = 0 # calls made so far
residual: list = field(default_factory=list) # (call, relative change) of every call
ms_per_call: list = field(default_factory=list)
kind: str = "solve" # "solve" | "stream"
final: dict | None = None # last decode: c2w, K, depth, conf, world points (numpy)
stream: dict | None = None # streaming result
frame: tuple | None = None # viewer frame (centre, radius, rotation, aerial), kept across updates
log: list = field(default_factory=list)
peak_gb: float = 0.0
gpu: str = "" # the device that ran it (ZeroGPU assigns one per run)
mode: str = ""
sky: bool = False # leave the sky out of the points (outdoor scenes)
source: str = "" # what was solved (an example's name)
skymask: np.ndarray | None = None # (V, H, W) True on sky, computed once
view_keep: float = 1.0 # share of each view's points the viewer shows (its point budget): exports keep it
hist: object = None # (Recorder, first_snap) of the last solve shown: a continuation plays after it
@property
def V(self):
return len(self.labels)
@property
def imf(self):
return self.images.float() / 255
@property
def hw(self):
return tuple(self.images.shape[-2:])
def ctx(self):
return Context(self.x0.to(DEV), self.pos.to(DEV), *self.hw)
# ---------------------------------------------------------------------- recording snapshots
class Recorder:
"""Snapshots of the state for the viewer: dense points of a pixel subset of every view at some calls, cameras at
every call (for the camera motion colour)."""
def __init__(self, images, budget=2_000_000):
V, _, H, W = images.shape
self.H, self.W = H, W
s = 1
while V * (H // s) * (W // s) > budget and s < 8:
s += 1
self.stride = s
ys, xs = np.mgrid[s // 2:H:s, s // 2:W:s]
self.ys, self.xs = ys.ravel(), xs.ravel()
self.images = images
self.snaps = [] # dict(call, P (V, n, 3) with NaN for absent views, conf, depth)
self.cams = {} # call -> (V_present, 4, 4)
def cameras(self, m, z, ctx, call):
with torch.no_grad():
c2w, K = cameras_from_pose_enc(m.decode_cameras(z, ctx), (ctx.H, ctx.W))
self.cams[call] = c2w.numpy()
self.K = K.numpy()
return self.cams[call]
def dense(self, m, z, ctx, call, chunk=16):
V = z.shape[0]
n = len(self.ys)
P = np.full((V, n, 3), np.nan, np.float32)
conf = np.zeros((V, n), np.float32)
depth = np.zeros((V, n), np.float32)
edge = np.zeros((V, n), bool)
yi, xi = torch.as_tensor(self.ys), torch.as_tensor(self.xs)
with torch.no_grad():
for a in range(0, V, chunk):
idx = list(range(a, min(V, a + chunk)))
pred = m.decode(z[idx], ctx.subset(idx))
wp = pred["world_points"].float()[:, yi, xi].cpu().numpy()
P[a:a + len(idx)] = wp
conf[a:a + len(idx)] = pred["depth_conf"].float()[:, yi, xi].cpu().numpy()
d = pred["depth"][..., 0].float()
depth[a:a + len(idx)] = d[:, yi, xi].cpu().numpy()
edge[a:a + len(idx)] = depth_edges(d)[:, yi, xi].cpu().numpy()
c2w = self.cameras(m, z, ctx, call)
self.snaps.append(dict(call=call, P=P, conf=conf, depth=depth, c2w=c2w, K=self.K.copy(), edge=edge))
def motion(self, call, V):
"""Rotation change (deg) of every view's camera over the last call."""
a, b = self.cams.get(call - 1), self.cams.get(call)
out = np.full(V, np.nan, np.float32)
if a is None or b is None:
return out
n = min(len(a), len(b))
Ra, Rb = a[:n, :3, :3], b[:n, :3, :3]
Ra, Rb = Ra[0].T[None] @ Ra, Rb[0].T[None] @ Rb
c = (np.einsum("nij,nij->n", Ra, Rb) - 1) / 2
out[:n] = np.degrees(np.arccos(np.clip(c, -1, 1)))
return out
def pack(self, path, labels, first_snap, title="", frame=None, n_pts=None, sky=False, exposure=1.0, horizon=None,
final=None, mv=None):
V = len(labels)
if n_pts is None: # about 40 MB of positions in all, at most a million points
n_pts = int(np.clip(40e6 / (6 * len(self.snaps)), 200_000, 1_000_000))
S = len(self.snaps)
n = len(self.ys)
P = np.full((S, V, n, 3), np.nan, np.float32)
c2w = np.tile(np.eye(4, dtype=np.float32), (S, V, 1, 1))
motion = np.full((S, V), np.nan, np.float32)
for k, s in enumerate(self.snaps):
v = s["P"].shape[0]
P[k, :v] = s["P"]
c2w[k, :v] = s["c2w"]
for j in range(v, V): # views not yet present: their first camera (hidden by the viewer)
c2w[k, j] = np.nan
motion[k] = self.motion(s["call"], V)
for j in range(V): # absent views take their first known camera
known = [k for k in range(S) if np.isfinite(c2w[k, j]).all()]
for k in range(S):
if not np.isfinite(c2w[k, j]).all():
c2w[k, j] = c2w[known[0], j] if known else np.eye(4)
last = self.snaps[-1]
K = last["K"]
vid = np.repeat(np.arange(V), n)
img = to_u8(self.images).permute(0, 2, 3, 1).numpy()
rgb = lift(img[:, self.ys, self.xs].reshape(-1, 3), exposure)
conf = last["conf"].reshape(-1).astype(np.float32) # raw: ranked over the whole scene in pack_solve
foot = (last["depth"] / K[:, 0, 0, None] * self.stride * 1.1).reshape(-1)
calls = [s["call"] for s in self.snaps]
P = P.reshape(S, V * n, 3)
keep = ~last.get("edge", np.zeros((V, n), bool)).reshape(-1) # depth edges (flying pixels) leave
if sky: # and the sky
keep &= ~sky_of(self.images)[:, self.ys, self.xs].reshape(-1)
if horizon is not None: # aerial examples: nothing seen near or above the horizon
keep &= below_horizon(P[-1], c2w[-1][vid, :3, 3], horizon)
if mv and final is not None: # floaters leave: a point stays when enough other views see its depth
keep &= agreeing_views(P[-1], vid, final) >= mv
P, vid, rgb, foot, conf = P[:, keep], vid[keep], rgb[keep], foot[keep], conf[keep]
meta = pack_solve(path, P, vid, rgb, foot, conf, c2w, calls, first_snap, K,
(self.H, self.W), thumbs_of(self.images), labels, motion=motion, n_pts=n_pts, title=title,
frame=frame)
return meta
def snap_calls(total, start=0):
"""Calls after `start` at which to keep dense snapshots, up to `total` (always the last)."""
out = [start + c for c in SNAPS if c < total - start]
if total not in out:
out.append(total)
return out
def finish(sess, m, z, ctx, keep_final=True):
"""Final decode kept for the exports and the views panel."""
sess.z = z.float().cpu()
if not keep_final:
return
with torch.no_grad():
c2w, K = cameras_from_pose_enc(m.decode_cameras(z, ctx), (ctx.H, ctx.W))
depth, conf, wp = [], [], []
for a in range(0, z.shape[0], 16):
idx = list(range(a, min(z.shape[0], a + 16)))
pred = m.decode(z[idx], ctx.subset(idx))
depth.append(pred["depth"][..., 0].float().cpu().numpy().astype(np.float16))
conf.append(pred["depth_conf"].float().cpu().numpy().astype(np.float16))
wp.append(pred["world_points"].float().cpu().numpy().astype(np.float16))
sess.final = dict(c2w=c2w.numpy(), K=K.numpy(), depth=np.concatenate(depth), conf=np.concatenate(conf),
points=np.concatenate(wp))
def _peak(sess):
if DEV == "cuda":
sess.peak_gb = max(sess.peak_gb, torch.cuda.max_memory_allocated() / 2 ** 30)
sess.gpu = torch.cuda.get_device_name(0).replace("NVIDIA ", "")
# ---------------------------------------------------------------------- calls with bookkeeping
def run_calls(sess, m, z, ctx, n, rec=None, dense_at=(), adaptive=None, min_calls=4, progress=None, offset=0):
"""n calls from z (or fewer with adaptive), recording cameras at every call and dense snapshots at dense_at."""
done = 0
for i in range(n):
sync()
t0 = time.perf_counter()
with torch.no_grad():
zn = m.solve(ctx, 1, z)
sync()
sess.ms_per_call.append((time.perf_counter() - t0) * 1000)
change = float((zn.float() - z.float()).norm() / z.float().norm().clamp_min(1e-9))
z = zn
done += 1
sess.calls += 1
sess.residual.append((sess.calls, change))
if rec is not None:
if sess.calls in dense_at:
rec.dense(m, z, ctx, sess.calls)
else:
rec.cameras(m, z, ctx, sess.calls)
if progress:
progress((offset + done) / max(offset + n, 1), f"call {sess.calls}")
if adaptive and done >= min_calls and change < adaptive:
break
if rec is not None and (not rec.snaps or rec.snaps[-1]["call"] != sess.calls):
rec.dense(m, z, ctx, sess.calls)
return z
# ---------------------------------------------------------------------- modes
def solve_offline(images, labels, paths, calls=16, adaptive=None, max_calls=64, progress=None):
m = model()
sess = Session(images=to_u8(images), labels=labels, paths=paths, mode="offline")
if DEV == "cuda":
torch.cuda.reset_peak_memory_stats()
t0 = time.perf_counter()
ctx = m.encode(images)
sess.x0, sess.pos = ctx.x0.cpu(), ctx.pos.cpu()
sess.first_call = [0] * len(labels)
z = m.start(ctx)
rec = Recorder(images)
rec.cameras(m, z, ctx, 0)
total = max_calls if adaptive else calls # adaptive stopping decides when to stop, up to max_calls
z = run_calls(sess, m, z, ctx, total, rec, set(snap_calls(total)), adaptive, progress=progress)
finish(sess, m, z, ctx)
_peak(sess)
sess.log.append(f"Offline: {sess.V} views, {sess.calls} calls, {(time.perf_counter() - t0):.1f} s")
return sess, rec
def stages_of(n, initial, group, order="file", seed=0):
"""Stages of view indices: the first `initial`, then groups; order 'file' or 'random' (any order)."""
idx = np.arange(n)
if order == "random":
rng = np.random.default_rng(seed)
first = np.linspace(0, n - 1, min(initial, n)).round().astype(int) # spread over the capture
rest = rng.permutation(np.setdiff1d(idx, first))
idx = np.concatenate([first, rest])
stages = [idx[:initial].tolist()]
for a in range(initial, n, group):
stages.append(idx[a:a + group].tolist())
return stages
def solve_stages(images, labels, paths, stages, calls=8, progress=None, mode="incremental"):
"""Views join in stages; stage i gets calls[i] calls (or `calls` each), which also update the views solved before."""
cs = list(calls) if isinstance(calls, (list, tuple)) else [calls] * len(stages)
m = model()
order = [i for s in stages for i in s]
images, labels, paths = images[order], [labels[i] for i in order], [paths[i] for i in order]
sess = Session(images=to_u8(images), labels=labels, paths=paths, mode=mode)
if DEV == "cuda":
torch.cuda.reset_peak_memory_stats()
t0 = time.perf_counter()
rec = Recorder(images)
first_snap, n_done = [], 0
total = sum(cs)
for i, st in enumerate(stages):
rows = list(range(n_done, n_done + len(st)))
if i == 0:
ctx = m.encode(images[rows])
z = m.start(ctx)
rec.cameras(m, z, ctx, 0)
else:
z, ctx = m.add_views(z, ctx, images[rows])
sess.first_call += [sess.calls] * len(st)
first_snap += [len(rec.snaps)] * len(st)
n_done += len(st)
dense = {sess.calls + 1, sess.calls + cs[i]}
z = run_calls(sess, m, z, ctx, cs[i], rec, dense, progress=progress, offset=sum(cs[:i]))
if progress:
progress(sum(cs[:i + 1]) / total, f"{n_done} of {len(order)} views")
sess.x0, sess.pos = ctx.x0.cpu(), ctx.pos.cpu()
finish(sess, m, z, ctx)
_peak(sess)
sess.log.append(f"{mode.capitalize()}: {sess.V} views in {len(stages)} stages, {sess.calls} calls, "
f"{(time.perf_counter() - t0):.1f} s")
return sess, rec, first_snap
HIST_BYTES = 600e6 # snapshots kept for a continuation to play after, at most
HIST_SNAPS = 2 # of which a continuation's timeline shows about this many (each costs points)
def with_history(hist, rec, first):
"""A continuation's snapshots after those of the solve it continues (hist: its Recorder and first_snap), so its
timeline shows the whole solve (views that join appear at their call). rec's first snapshot is the state the kept
solve ended in: it is dropped."""
prev, pfirst = hist
size = sum(v.nbytes for s in prev.snaps + rec.snaps for v in s.values() if isinstance(v, np.ndarray))
if size > HIST_BYTES or (prev.H, prev.W) != (rec.H, rec.W):
return rec, first
grid = {(y, x): i for i, (y, x) in enumerate(zip(prev.ys.tolist(), prev.xs.tolist()))}
idx = [grid.get(p) for p in zip(rec.ys.tolist(), rec.xs.tolist())] # rec's pixels on prev's grid (a coarser
if any(i is None for i in idx): # stride: a subset of a finer one)
return rec, first
idx = np.array(idx)
S = len(prev.snaps) # a few of them (the snapshots cost the browser points): ends and joins kept
keep = sorted({0, S - 1, *pfirst, *np.linspace(0, S - 1, HIST_SNAPS).round().astype(int).tolist()})
at = {j: i for i, j in enumerate(keep)}
k, n_old = len(keep), len(pfirst)
pick = lambda d: {k: (v[:, idx] if isinstance(v, np.ndarray) and v.ndim >= 2 and k != "c2w" and k != "K" else v)
for k, v in d.items()}
rec.snaps = [pick(prev.snaps[j]) for j in keep] + rec.snaps[1:]
rec.cams = {**prev.cams, **rec.cams}
return rec, [at[f] for f in pfirst] + [k - 1 + f for f in first[n_old:]]
def continue_solve(sess, calls, progress=None):
"""More calls on the kept state (all views)."""
m = model()
ctx = sess.ctx()
z = sess.z.to(DEV)
rec = Recorder(sess.imf)
start = sess.calls
rec.cameras(m, z, ctx, start)
rec.dense(m, z, ctx, start)
dense = set(snap_calls(start + calls, start))
z = run_calls(sess, m, z, ctx, calls, rec, dense, progress=progress)
finish(sess, m, z, ctx)
_peak(sess)
sess.log.append(f"Continued: +{calls} calls (now {sess.calls})")
return sess, rec
def add_photos(sess, images, labels, paths, calls=8, progress=None):
"""New views join the kept solve; all views are updated."""
m = model()
images = load_resized(images, sess.hw)
ctx = sess.ctx()
z = sess.z.to(DEV)
rec = Recorder(torch.cat([sess.imf, images]))
start = sess.calls
rec.cameras(m, z, ctx, start)
rec.dense(m, z, ctx, start)
z, ctx = m.add_views(z, ctx, images)
n_old = sess.V
sess.images = torch.cat([sess.images, to_u8(images)])
sess.labels += labels
sess.paths += paths
sess.first_call += [start] * len(labels)
sess.x0, sess.pos = ctx.x0.cpu(), ctx.pos.cpu()
dense = set(snap_calls(start + calls, start))
z = run_calls(sess, m, z, ctx, calls, rec, dense, progress=progress)
finish(sess, m, z, ctx)
_peak(sess)
first_snap = [0] * n_old + [1] * len(labels)
sess.log.append(f"Added {len(labels)} photos at call {start}, +{calls} calls")
return sess, rec, first_snap
def load_resized(images, hw):
if tuple(images.shape[-2:]) == tuple(hw):
return images
from solv3r.images import resize
return resize(images, hw)
def add_fixed(sess, images, labels, paths, calls=16, progress=None):
"""New views join with the scene held fixed: the kept views enter every call through one key / value cache and only
the new views are solved, together. They then belong to the scene like any other view."""
m = model()
images = load_resized(images, sess.hw)
ctx = sess.ctx()
z = sess.z.to(DEV)
rec = Recorder(torch.cat([sess.imf, images]))
start, n_old = sess.calls, sess.V
rec.cameras(m, z, ctx, start)
rec.dense(m, z, ctx, start)
sync()
t0 = time.perf_counter()
with torch.no_grad(), autocast():
cache = m.cache(z, ctx)
sync()
t_cache = time.perf_counter() - t0
new = m.encode(images, reference_first=False)
zq = m.start(new)
ctx_all = Context.cat([ctx, new])
dense = set(snap_calls(start + calls, start))
ms = []
for i in range(calls):
sync()
t1 = time.perf_counter()
with torch.no_grad(), autocast():
zn = m.step_cached(zq, new, cache)
sync()
ms.append((time.perf_counter() - t1) * 1000)
change = float((zn.float() - zq.float()).norm() / zq.float().norm().clamp_min(1e-9))
zq = zn
sess.calls += 1
sess.residual.append((sess.calls, change))
z_all = torch.cat([z, zq.to(z.dtype)])
if sess.calls in dense:
rec.dense(m, z_all, ctx_all, sess.calls)
else:
rec.cameras(m, z_all, ctx_all, sess.calls)
if progress:
progress((i + 1) / calls, f"call {sess.calls}")
z_all = torch.cat([z, zq.to(z.dtype)])
if not rec.snaps or rec.snaps[-1]["call"] != sess.calls:
rec.dense(m, z_all, ctx_all, sess.calls)
sess.images = torch.cat([sess.images, to_u8(images)])
sess.labels += labels
sess.paths += paths
sess.first_call += [start] * len(labels)
sess.x0, sess.pos = ctx_all.x0.cpu(), ctx_all.pos.cpu()
finish(sess, m, z_all, ctx_all)
_peak(sess)
first_snap = [0] * n_old + [1] * len(labels)
sess.log.append(f"Added {len(labels)} photos with the scene frozen: one cache of {n_old} views ({t_cache * 1000:.0f} ms), "
f"{calls} calls on the new views ({np.median(ms):.1f} ms per call)")
return sess, rec, first_snap
def solve_stream(paths, labels, config="L8", progress=None, keep=0.5, per_frame=2500):
m = model()
size = load_images(paths[:1]).shape[-2:]
cfg = PRESETS[config]
import dataclasses
cfg = dataclasses.replace(cfg, memory_stride=min(cfg.memory_stride, max(1, -(-len(paths) // cfg.memory))))
solver = StreamingSolver(m, cfg, depth=True)
if DEV == "cuda":
torch.cuda.reset_peak_memory_stats()
frames, live = [], []
rng = np.random.default_rng(0)
for i, p in enumerate(paths):
im = load_images([p], size)[0]
frames.append(to_u8(im))
for f in solver.push(im): # each frame answered now: its camera and a few of its points
live.append(live_frame(solver, f, frames[f], rng))
if progress and (i % 2 == 0 or i == len(paths) - 1):
progress((i + 1) / len(paths), f"frame {i + 1} of {len(paths)} answered", live)
live = []
res = solver.result()
images = torch.stack(frames)
sess = Session(images=images, labels=labels, paths=paths, kind="stream", mode="streaming")
sess.calls = len(paths)
sess.stream = dict(c2w=res.c2w, c2w_exit=res.c2w_exit, K=res.intrinsics, depth=res.depth, conf=res.depth_conf,
ms=res.ms_per_frame, refreshed=list(res.refreshed), config=config)
sess.final = dict(c2w=res.c2w_exit, K=res.intrinsics, depth=res.depth.astype(np.float16),
conf=res.depth_conf.astype(np.float16), points=None)
_peak(sess)
sess.log.append(f"Streaming ({config}): {len(paths)} frames, {res.ms_per_frame:.1f} ms per frame (solver), "
f"memory refreshed at {sum(res.refreshed)} frames")
return sess
def live_frame(solver, f, image, rng, n=1500, stride=3):
"""What the live view draws for a frame on arrival: camera centre, forward and up (world), intrinsics (fractions of
the image), n of its confident points (float16 xyz + uint8 rgb + float16 footprint, base64) and a small JPEG. The
points and their footprints are chosen as in pack_stream_session, so the live cloud looks like the final one."""
import base64
import cv2
R, C, K = solver.R[f].float().numpy(), solver.C[f].float().numpy(), solver.K[f].float().numpy()
img = image.permute(1, 2, 0).numpy()
H, W = img.shape[:2]
out = [int(f)] + np.round(np.concatenate([C, R[:, 2], -R[:, 1]]), 5).tolist()
out += [round(float(K[0, 0] / W), 5), round(float(K[1, 1] / H), 5), round(float(K[0, 2] / W), 5), round(float(K[1, 2] / H), 5)]
pts, rgb, foot = b"", b"", b""
if f in solver.depths:
d, c = solver.depths[f], solver.confs[f]
h, w = d.shape
ys, xs = np.mgrid[stride // 2:h:stride, stride // 2:w:stride]
ys, xs = ys.ravel(), xs.ravel()
z, cf = d[ys, xs], c[ys, xs]
ok = np.isfinite(z) & (z > 0) & ~depth_edges(torch.as_tensor(d[None]).float())[0].numpy()[ys, xs]
if ok.any():
ok &= cf >= np.quantile(cf[ok], 0.5)
idx = np.nonzero(ok)[0]
if len(idx) > n:
idx = rng.choice(idx, n, replace=False)
if len(idx):
Kd = K * np.array([[w / W], [h / H], [1]]) # intrinsics at the depth resolution
zz = z[idx]
X = np.stack([(xs[idx] + 0.5 - Kd[0, 2]) / Kd[0, 0] * zz, (ys[idx] + 0.5 - Kd[1, 2]) / Kd[1, 1] * zz, zz], 1)
pts = (X @ R.T + C).astype(np.float16).tobytes()
rgb = img[(ys[idx] * H) // h, (xs[idx] * W) // w].astype(np.uint8).tobytes()
dens = min(1.4, np.sqrt(np.count_nonzero(ok) / len(idx)))
foot = (zz / Kd[0, 0] * stride * 1.1 * dens).astype(np.float16).tobytes()
th = cv2.resize(img, (96, max(1, round(96 * H / W))), interpolation=cv2.INTER_AREA)
ok_, jpg = cv2.imencode(".jpg", cv2.cvtColor(th, cv2.COLOR_RGB2BGR), [cv2.IMWRITE_JPEG_QUALITY, 70])
b64 = lambda b: base64.b64encode(b).decode()
return out + [b64(pts), b64(rgb), b64(foot), b64(jpg.tobytes())]
def consistent(P, f, S, tol=0.04):
"""Points of frame f that agree with the depth of frames f +- 2 and f +- 4 (exit states) where those frames see them."""
F_, H, W = S["depth"].shape
ok, seen = np.zeros(len(P), int), np.zeros(len(P), int)
for g in (f - 4, f - 2, f + 2, f + 4):
if g < 0 or g >= F_:
continue
w2c = np.linalg.inv(S["c2w_exit"][g]); Q = P @ w2c[:3, :3].T + w2c[:3, 3]; zq = Q[:, 2]
K = S["K"][g]
u = K[0, 0] * Q[:, 0] / np.maximum(zq, 1e-6) + K[0, 2]; v = K[1, 1] * Q[:, 1] / np.maximum(zq, 1e-6) + K[1, 2]
i = np.nonzero((zq > 1e-4) & (u >= 0) & (u < W - 1) & (v >= 0) & (v < H - 1))[0]
dz = np.abs(S["depth"][g][v[i].astype(int), u[i].astype(int)] - zq[i]) / zq[i]
seen[i] += 1; ok[i] += dz < tol
return (seen == 0) | (ok * 2 >= seen)
def agreeing_views(P, vid, final, tol=0.03):
"""For each point (N, 3) of view vid, the number of other views whose final depth map agrees with it (relative tol)."""
D, K, W2C = final["depth"], final["K"], np.linalg.inv(final["c2w"].astype(np.float64))
V, H, W = D.shape
cnt = np.zeros(len(P), np.int16)
fin = np.isfinite(P).all(1)
for i in range(V):
X = P @ W2C[i, :3, :3].T.astype(np.float32) + W2C[i, :3, 3].astype(np.float32)
z = X[:, 2]
with np.errstate(all="ignore"):
u = K[i, 0, 0] * X[:, 0] / z + K[i, 0, 2]
v = K[i, 1, 1] * X[:, 1] / z + K[i, 1, 2]
idx = np.nonzero(fin & (vid != i) & (z > 0) & (u >= 0) & (u < W) & (v >= 0) & (v < H))[0]
d = D[i, v[idx].astype(int), u[idx].astype(int)].astype(np.float32)
cnt[idx] += np.abs(d - z[idx]) < tol * z[idx]
return cnt
def below_horizon(P, C, deg=3.0):
"""Aerial scenes: points seen at least `deg` below the horizon from their camera (P points, C their cameras' centres). Distant hills,
city lights and the sky that the sky mask misses come back with a compressed depth and float above the map; they are seen near or
above the horizon. Up is the ground normal (smallest principal axis of the points, toward the cameras), not the cameras' -y, which
the drone's pitch tilts (as the project page's demo videos)."""
ok = np.isfinite(P).all(1)
Q = P[ok][np.random.default_rng(0).choice(int(ok.sum()), min(int(ok.sum()), 200000), replace=False)].astype(np.float64)
c = np.median(Q, 0)
lo, hi = np.quantile(Q, 0.05, 0), np.quantile(Q, 0.95, 0)
Q = Q[((Q >= lo) & (Q <= hi)).all(1)] - c
up = np.linalg.eigh(Q.T @ Q / len(Q))[1][:, 0]
up = up if (np.nanmean(C, 0) - c) @ up > 0 else -up
ray = P - C
el = np.degrees(np.arcsin(np.clip((ray @ up) / np.maximum(np.linalg.norm(ray, axis=1), 1e-9), -1, 1)))
return ok & (el < -deg)
def lift(rgb, exposure):
"""Display brightness of point colours (night scenes): a gamma lift, dark tones raised most, nothing clipped."""
if exposure == 1:
return rgb
return np.round(255 * (np.asarray(rgb, np.float32) / 255) ** (1 / exposure)).astype(np.uint8)
def facing(depth, K, ys, xs):
"""|cos| of the angle between each pixel's viewing ray and the surface it sees (normal from the depth map's neighbours):
small where the frame sees the surface at a grazing angle, where its depth is least precise."""
H, W = depth.shape
v, u = np.mgrid[0:H, 0:W]
X = np.stack([(u + 0.5 - K[0, 2]) / K[0, 0] * depth, (v + 0.5 - K[1, 2]) / K[1, 1] * depth, depth], -1)
du, dv = np.zeros_like(X), np.zeros_like(X)
du[:, 1:-1] = X[:, 2:] - X[:, :-2]
dv[1:-1] = X[2:] - X[:-2]
n = np.cross(du, dv)[ys, xs]
r = X[ys, xs]
return np.abs((n * r).sum(1)) / np.maximum(np.linalg.norm(n, axis=1) * np.linalg.norm(r, axis=1), 1e-12)
def first_seen(P, fid, vox=0.006, slack=2):
"""Points in space that no earlier frame (more than `slack` frames before) already covered: a later, less precise view of
the same surface does not blur what an earlier frame saw sharply; later frames only extend the map (voxels of `vox` x the
scene radius)."""
c = np.median(P, 0)
r = float(np.quantile(np.linalg.norm(P - c, axis=1), 0.9))
key = np.floor((P - c) / (vox * r)).astype(np.int64)
key = (key[:, 0] * 73856093) ^ (key[:, 1] * 19349663) ^ (key[:, 2] * 83492791)
order = np.argsort(fid, kind="stable")
_, first_idx = np.unique(key[order], return_index=True)
first = dict(zip(key[order][first_idx].tolist(), fid[order][first_idx].tolist()))
return fid <= np.array([first[k] for k in key.tolist()]) + slack
def pack_stream_session(sess, path, keep=0.9, per_frame=8000, stride=3, sky=False, exposure=1.0, horizon=None, graze=None,
firstseen=False): # the viewer's confidence slider does the rest
S = sess.stream
skym = sky_of(sess.images) if sky else None
F_, H, W = S["depth"].shape
ys, xs = np.mgrid[stride // 2:H:stride, stride // 2:W:stride]
ys, xs = ys.ravel(), xs.ravel()
img = sess.images.permute(0, 2, 3, 1).numpy()
rng = np.random.default_rng(0)
Pn, Pl, col, fid, foot, cf = [], [], [], [], [], []
cand = 0
for f in range(F_):
d, c = S["depth"][f][ys, xs], S["conf"][f][ys, xs]
ok = np.isfinite(d) & (d > 0) & ~depth_edges(torch.as_tensor(S["depth"][f][None]).float())[0].numpy()[ys, xs]
if skym is not None:
ok &= ~skym[f][ys, xs]
thr = np.quantile(c[ok], 1 - keep) if ok.any() else 0
K = S["K"][f]
if graze is not None: # nothing seen at a grazing angle: it would blur what frontal frames saw sharply
ok &= facing(S["depth"][f], K, ys, xs) > graze
idx = np.nonzero(ok & (c >= thr))[0]
z = d[idx]
X = np.stack([(xs[idx] + 0.5 - K[0, 2]) / K[0, 0] * z, (ys[idx] + 0.5 - K[1, 2]) / K[1, 1] * z, z], 1)
good = consistent(X @ S["c2w_exit"][f][:3, :3].T + S["c2w_exit"][f][:3, 3], f, S) # as the project page's streams
idx, z, X = idx[good], z[good], X[good]
cand += len(idx)
if len(idx) > per_frame:
sub = np.sort(rng.choice(len(idx), per_frame, replace=False))
idx, z, X = idx[sub], z[sub], X[sub]
for out, C in ((Pn, S["c2w"][f]), (Pl, S["c2w_exit"][f])):
out.append(X @ C[:3, :3].T + C[:3, 3])
col.append(img[f, ys[idx], xs[idx]])
fid.append(np.full(len(idx), f))
foot.append(z / K[0, 0] * stride * 1.1)
cf.append(c[idx])
cat = np.concatenate
Pn, Pl, fid, col, foot, cf = cat(Pn), cat(Pl), cat(fid), cat(col), cat(foot), cat(cf)
if firstseen: # later frames only extend the map
k = first_seen(Pl, fid)
Pn, Pl, fid, col, foot, cf = Pn[k], Pl[k], fid[k], col[k], foot[k], cf[k]
if horizon is not None: # aerial examples: nothing seen near or above the horizon
k = below_horizon(Pl, S["c2w_exit"][fid][:, :3, 3], horizon)
Pn, Pl, fid, col, foot, cf = Pn[k], Pl[k], fid[k], col[k], foot[k], cf[k]
return pack_stream(path, Pn, Pl, fid, lift(col, exposure), foot, S["c2w"], S["c2w_exit"], S["K"],
sess.hw, thumbs_of(sess.images, 64), labels=sess.labels,
dens=min(2.0, np.sqrt(cand / max(len(fid), 1))), conf=cf)
# ---------------------------------------------------------------------- large scenes: a random subset of views per call
class Stochastic:
"""Each call updates b random views; the others join as constraints through their cached keys / values.
Per view we keep the global-attention keys / values of its current state and the energy of its latest update
(for the state scale), so a call costs b views of frame attention plus their attention over the cache. Peak
memory grows with the number of views only through the cache (about 2.7 MB per view)."""
def __init__(self, m, ctx, chunk=16):
self.m, self.ctx, self.blk = m, ctx, m.net.block
self.N, self.P, self.C = ctx.x0.shape
attn = self.blk.global_attn.attn
self.Hh = attn.num_heads
self.z = m.start(ctx).float()
self.K = torch.empty(self.N, self.Hh, self.P, self.C // self.Hh, dtype=torch.bfloat16, device=DEV)
self.Vv = torch.empty_like(self.K)
tgt = float(self.blk.target_rms[3])
self.e = torch.full((self.N,), tgt ** 2 * self.P * self.C, device=DEV)
self.updates = torch.zeros(self.N, dtype=torch.long)
for a in range(0, self.N, chunk):
self._kv(list(range(a, min(self.N, a + chunk))))
@torch.no_grad()
def _kv(self, S):
b = len(S)
with autocast():
y = self.blk._frame(self.z[S], self.ctx.x0[S], self.ctx.pos[S]).reshape(1, b * self.P, self.C)
k, v = _attn_kv(self.blk.global_attn.attn, self.blk.global_attn.norm1(y))
self.K[S] = k[0].reshape(self.Hh, b, self.P, -1).permute(1, 0, 2, 3).to(torch.bfloat16)
self.Vv[S] = v[0].reshape(self.Hh, b, self.P, -1).permute(1, 0, 2, 3).to(torch.bfloat16)
@torch.no_grad()
def update(self, S):
b = len(S)
held = torch.ones(self.N, dtype=torch.bool, device=DEV)
held[S] = False
nh = int(held.sum())
Kh = self.K[held].permute(1, 0, 2, 3).reshape(1, self.Hh, nh * self.P, -1)
Vh = self.Vv[held].permute(1, 0, 2, 3).reshape(1, self.Hh, nh * self.P, -1)
attn = self.blk.global_attn.attn
with autocast():
y = self.blk._frame(self.z[S], self.ctx.x0[S], self.ctx.pos[S]).reshape(1, b * self.P, self.C)
y = self.blk.global_attn(y, self.blk.g_global,
attn_fn=lambda x: _attn_with_cache(attn, x, Kh, Vh))
y = y.reshape(b, self.P, self.C)
eq = y.float().pow(2).sum((1, 2))
rms = ((self.e[held].sum() + eq.sum()) / (self.N * self.P * self.C)).clamp_min(1e-12).sqrt()
zq = self.blk._rescale(y, rms).float()
self.e[S] = eq
self.z[S] = zq
self.updates[S] += 1
self._kv(S)
def solve_stochastic(images, labels, paths, per_call=8, updates=32, seed=0, progress=None):
m = model()
sess = Session(images=to_u8(images), labels=labels, paths=paths, mode="large")
if DEV == "cuda":
torch.cuda.reset_peak_memory_stats()
t0 = time.perf_counter()
ctx = m.encode(images)
sess.x0, sess.pos = ctx.x0.cpu(), ctx.pos.cpu()
sess.first_call = [0] * len(labels)
N = len(labels)
st = Stochastic(m, ctx)
rec = Recorder(images)
rec.cameras(m, st.z, ctx, 0)
g = torch.Generator().manual_seed(seed)
total = math.ceil(updates * N / per_call)
per_epoch = max(1, N // per_call)
dense_epochs = {1, 2, 4, 8, 16, 32, 64, updates}
z_prev = st.z.clone()
for c in range(1, total + 1):
if (c - 1) % per_epoch == 0:
torch.randperm(N, generator=g)
S = torch.randperm(N, generator=g)[:per_call].tolist()
sync()
t1 = time.perf_counter()
st.update(S)
sync()
sess.ms_per_call.append((time.perf_counter() - t1) * 1000)
sess.calls = c
if c % per_epoch == 0 or c == total:
ep = c // per_epoch
sess.residual.append((c, float((st.z - z_prev).norm() / z_prev.norm().clamp_min(1e-9)))) # per pass
z_prev = st.z.clone()
if ep in dense_epochs or c == total:
rec.dense(m, st.z, ctx, c)
else:
rec.cameras(m, st.z, ctx, c)
if progress:
progress(c / total, f"{c} calls, {ep} updates per view")
finish(sess, m, st.z, ctx)
_peak(sess)
sess.log.append(f"Large scene: {N} views, {per_call} per call, {total} calls (~{updates} updates per view), "
f"{(time.perf_counter() - t0):.1f} s, peak {sess.peak_gb:.2f} GB")
return sess, rec
# ---------------------------------------------------------------------- exports
def confident(sess, keep=1.0):
"""The points the viewer shows, at every pixel (solv3r.viewer.pack), and their colours: an export is what one sees.
keep: the viewer's confidence filtering (1: all)."""
f = sess.final
if f.get("points") is None: # stream: the depth on arrival placed with the exit cameras
S = sess.stream
res = SimpleNamespace(c2w=S["c2w"], c2w_exit=S["c2w_exit"], intrinsics=S["K"], depth=S["depth"], depth_conf=S["conf"])
return stream_export(sess.images, res, keep, sess.sky)
return solve_export(f["points"], f["conf"], f["depth"], sess.images, sess.view_keep, keep, sess.sky)
def export_ply(sess, keep=1.0, max_points=8_000_000):
xyz, rgb = confident(sess, keep)
if len(xyz) > max_points:
sel = np.random.default_rng(0).choice(len(xyz), max_points, replace=False)
xyz, rgb = xyz[sel], rgb[sel]
path = os.path.join(new_dir("export"), "solv3r_points.ply")
write_ply(path, xyz, rgb)
return path
def export_npz(sess):
f = sess.final
path = os.path.join(new_dir("export"), "solv3r_cameras_depth.npz")
np.savez_compressed(path, c2w=f["c2w"], K=f["K"], depth=f["depth"], depth_conf=f["conf"],
names=np.array(sess.labels), image_size=np.array(sess.hw))
return path
def _qvec(R):
"""Rotation matrix -> COLMAP quaternion (w, x, y, z)."""
from scipy.spatial.transform import Rotation
x, y, z, w = Rotation.from_matrix(R).as_quat()
return w, x, y, z
def export_colmap(sess, keep=1.0, max_points=300_000, with_images=True):
"""COLMAP text model (sparse/0: cameras, images, points3D) at the original image sizes, plus the images; for
Gaussian splatting and other tools that start from a COLMAP model."""
from PIL import Image
f = sess.final
d = new_dir("colmap")
sp = os.path.join(d, "sparse", "0")
os.makedirs(sp)
H, W = sess.hw
cams, ims = [], []
names = []
for i in range(sess.V):
src = sess.paths[i]
try:
w0, h0 = Image.open(src).size if src and os.path.isfile(src) else (W, H)
except OSError:
w0, h0 = W, H
K = f["K"][i]
sx, sy = w0 / W, h0 / H
name = f"{i:05d}_{os.path.basename(src) if src else sess.labels[i] + '.jpg'}"
names.append(name)
cams.append(f"{i + 1} PINHOLE {w0} {h0} {K[0, 0] * sx:.6f} {K[1, 1] * sy:.6f} {K[0, 2] * sx:.6f} {K[1, 2] * sy:.6f}")
w2c = np.linalg.inv(f["c2w"][i].astype(np.float64))
q = _qvec(w2c[:3, :3])
t = w2c[:3, 3]
ims.append(f"{i + 1} {q[0]:.9f} {q[1]:.9f} {q[2]:.9f} {q[3]:.9f} {t[0]:.9f} {t[1]:.9f} {t[2]:.9f} {i + 1} {name}\n")
open(os.path.join(sp, "cameras.txt"), "w").write("# Camera list: CAMERA_ID, MODEL, WIDTH, HEIGHT, PARAMS[]\n" + "\n".join(cams) + "\n")
open(os.path.join(sp, "images.txt"), "w").write("# IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME\n" + "\n".join(ims))
xyz, rgb = confident(sess, keep)
if len(xyz) > max_points:
sel = np.random.default_rng(0).choice(len(xyz), max_points, replace=False)
xyz, rgb = xyz[sel], rgb[sel]
with open(os.path.join(sp, "points3D.txt"), "w") as fh:
fh.write("# POINT3D_ID, X, Y, Z, R, G, B, ERROR, TRACK[]\n")
for j, (p, c) in enumerate(zip(xyz, rgb)):
fh.write(f"{j + 1} {p[0]:.6f} {p[1]:.6f} {p[2]:.6f} {c[0]} {c[1]} {c[2]} 0\n")
if with_images:
os.makedirs(os.path.join(d, "images"))
for i, name in enumerate(names):
src = sess.paths[i]
if src and os.path.isfile(src):
shutil.copy(src, os.path.join(d, "images", name))
else:
Image.fromarray(sess.images[i].permute(1, 2, 0).numpy()).save(os.path.join(d, "images", name))
zp = os.path.join(new_dir("export"), "solv3r_colmap.zip")
with zipfile.ZipFile(zp, "w", zipfile.ZIP_DEFLATED) as z:
for root, _, files in os.walk(d):
for fn in files:
full = os.path.join(root, fn)
z.write(full, os.path.relpath(full, d))
shutil.rmtree(d, ignore_errors=True)
return zp