room-surface-studio / app /render.py
umang2191's picture
Upload 16 files
dbd4276 verified
Raw History Blame Contribute Delete
12.7 kB
"""Material rendering on cached detection results. Segmentation is never re-run from here."""
import json
import math
import numpy as np
from PIL import Image, ImageDraw
import torch
import torch.nn.functional as F
import geometry
import imgops
from textures import parse_color, sample_mipmapped
PREVIEW_SIDE = 1500
LUMA = imgops.LUMA
PALETTE = [(40, 44, 52), (66, 135, 245), (245, 166, 35), (120, 200, 120), (200, 90, 160), (90, 200, 200)]
# ------------------------------------------------------------------ session / cache
class Layer:
def __init__(self, arr, base, conf, label, raw_label, band):
self.arr, self.base, self.conf = arr, base, conf
self.label, self.raw_label, self.band = label, raw_label, band
self.H, self.W = arr.shape[:2]
def _down_layer(L: Layer, side: int) -> Layer:
s = side / max(L.H, L.W)
if s >= 1:
return L
size = (max(8, round(L.H * s)), max(8, round(L.W * s)))
nn = lambda a: F.interpolate(torch.from_numpy(a.astype(np.float32))[None, None], size=size, mode="nearest")[0, 0].numpy().astype(np.uint8)
return Layer(imgops.area_down(L.arr, size), {k: imgops.area_down(v, size) for k, v in L.base.items()},
imgops.area_down(L.conf, size), nn(L.label), nn(L.raw_label), nn(L.band))
class Session:
def __init__(self, det):
self.det = det
self.full = Layer(det.rgb, det.alpha, det.conf, det.labelmap, det.raw_label, det.band)
self.prev = _down_layer(self.full, PREVIEW_SIDE)
self._edit = {"full": self._blank(), "prev": self._blank()}
self._light, self._geo = {}, {}
@staticmethod
def _blank():
return {"keys": None, "alpha": None, "version": 0}
def layer(self, q):
return self.full if q == "full" else self.prev
# --- user edits (brush strokes), applied incrementally and cached
def alphas(self, q, strokes):
L, st = self.layer(q), self._edit[q]
keys = [json.dumps(s, sort_keys=True) for s in strokes]
if st["keys"] is not None and keys[:len(st["keys"])] == st["keys"] and st["alpha"] is not None:
new = keys[len(st["keys"]):]
todo = strokes[len(st["keys"]):]
else:
st["alpha"] = {k: v.copy() for k, v in L.base.items()}
new, todo = keys, strokes
for s in todo:
apply_stroke(st["alpha"], s, L.H, L.W)
if st["keys"] != keys:
st["version"] += 1
st["keys"] = keys
return st["alpha"], st["version"]
def geo(self, q, strokes):
al, ver = self.alphas(q, strokes)
# geometry only needs a small mask, so one cache entry per edit-state is enough
key = (self._edit[q]["keys"] and "|".join(self._edit[q]["keys"])) or ""
if key not in self._geo:
self._geo = {key: geometry.scene_geometry(al["wall"], al["floor"])}
return self._geo[key]
def lighting(self, q, surface, alpha, ver):
key = (q, ver, surface)
if key not in self._light:
if len(self._light) > 6:
self._light.clear()
L = self.layer(q)
lum = (L.arr @ LUMA).astype(np.float32)
r = max(3, min(L.H, L.W) // 85)
smooth = imgops.masked_blur(lum, alpha, r)
ref = float((smooth * alpha).sum() / max(alpha.sum(), 1.0))
ref = max(ref, 1e-3)
self._light[key] = (smooth / ref, (lum - smooth) / ref)
return self._light[key]
def apply_stroke(al, st, H, W):
"""Rasterise one brush stroke in its bounding box (fast) and edit the mattes."""
pts = [(x * W, y * H) for x, y in st["pts"]]
if not pts:
return
r = max(1.0, float(st["r"]) * W)
hard = float(st.get("hard", 0.7))
op = float(st.get("op", 1.0))
rf = int((1 - hard) * r * 0.6)
mg = int(r + 2 * rf + 3)
x0 = max(0, int(min(p[0] for p in pts)) - mg)
y0 = max(0, int(min(p[1] for p in pts)) - mg)
x1 = min(W, int(max(p[0] for p in pts)) + mg + 1)
y1 = min(H, int(max(p[1] for p in pts)) + mg + 1)
if x1 <= x0 or y1 <= y0:
return
im = Image.new("L", (x1 - x0, y1 - y0), 0)
d = ImageDraw.Draw(im)
loc = [(x - x0, y - y0) for x, y in pts]
if len(loc) > 1:
d.line(loc, fill=255, width=max(1, round(2 * r)), joint="curve")
for x, y in loc[:: max(1, len(loc) // 300)] + [loc[-1]]:
d.ellipse([x - r, y - r, x + r, y + r], fill=255)
m = np.asarray(im, np.float32) / 255.0
m = imgops.box_filter(m, max(1, rf)) if rf else imgops.box_filter(m, 1)
if rf > 1:
m = imgops.box_filter(m, max(1, rf // 2))
m = np.clip(m * op, 0, 1)
sl = (slice(y0, y1), slice(x0, x1))
aw, af = al["wall"][sl], al["floor"][sl]
mode = st["m"]
if mode == "floor":
al["floor"][sl] = af + (1 - af) * m
al["wall"][sl] = aw * (1 - m)
elif mode == "wall":
al["wall"][sl] = aw + (1 - aw) * m
al["floor"][sl] = af * (1 - m)
else:
al["wall"][sl] = aw * (1 - m)
al["floor"][sl] = af * (1 - m)
# ------------------------------------------------------------------ mapping
def map_floor(tex, xs, ys, H, W, p, geo, bottom):
scale = max(float(p.get("scale", 0.3)), 0.02)
aspect = tex.h / tex.w
if p.get("persp", True):
if p.get("persp_mode", "auto") == "manual":
yh, vpx = float(p.get("horizon", -0.2)) * H, W / 2
else:
yh, vpx = geo["floor"]["horizon"] * H, geo["floor"]["vp_x"] * W
d = np.maximum(ys - yh, 1.0)
db = max(bottom - yh, 1.0)
f = float(p.get("depth", 0.9)) * W
k = db / (W * scale)
u = (xs - vpx) * k / d
v = (k * f / d) / aspect
fp = np.sqrt((k / d * tex.w) * (k * f / d ** 2 * tex.h / aspect))
else:
u, v = xs / (W * scale), ys / (W * scale * aspect)
fp = np.full(u.shape, tex.w / (W * scale), np.float32)
return _finish(tex, u, v, fp, p)
def slider_vp(p):
"""Manual 'wall angle' slider -> vanishing-point x (image-width units from centre); None = front-facing."""
p = float(p)
return None if abs(p) < 0.03 else (0.5 / p)
def map_wall(tex, xs, ys, H, W, p, geo):
"""Planar homography per wall plane: u ~ 1/(x - x_vp), v ~ (y - horizon)/(x - x_vp), normalised so a
tile spans `scale` of the image width at the plane's visible centre. vp=None -> front-facing (affine)."""
scale = max(float(p.get("scale", 0.25)), 0.02)
aspect = tex.h / tex.w
manual = p.get("persp_mode", "auto") == "manual"
yh = (float(p["horizon"]) if manual and "horizon" in p else geo["floor"]["horizon"]) * H
segs = ([{"x0": 0.0, "x1": 1.0, "vp": slider_vp(p.get("wpersp", 0.0))}] if manual else geo["wall"]["segments"])
xn, yn = (xs - W / 2) / W, (ys - yh) / W
fx = xs / W
u, v, fp = np.empty_like(xn), np.empty_like(xn), np.empty_like(xn)
done = np.zeros(xn.shape, bool)
fn = 0.9 # focal length in image-width units (same assumption as the floor)
for i, s in enumerate(segs):
sel = (fx >= s["x0"]) & ((fx < s["x1"]) if i < len(segs) - 1 else (fx <= 1.0 + 1e-6)) & ~done
if not sel.any():
continue
done |= sel
vp = s["vp"]
if vp is None:
u[sel], v[sel] = xn[sel] / scale, yn[sel] / (scale * aspect)
fp[sel] = tex.w / (W * scale)
else:
xr = float(np.median(xn[sel]))
sg = 1.0 if xr - vp >= 0 else -1.0
dx = xn[sel] - vp
dx = np.where(sg * dx < 0.02, sg * 0.02, dx)
# exact plane homography: metric along-wall coord = |D| * t, height = y * t, t = K / (x_vp - x)
# with |D| = sqrt(vp^2 + f^2) (the wall's rotation relative to the camera)
D = float(np.hypot(vp, fn))
Q = (xr - vp) ** 2 / (D * scale)
u[sel] = Q * D / dx
v[sel] = Q * yn[sel] / dx / aspect
fp[sel] = np.sqrt((tex.w * Q * D / (dx ** 2 * W)) * (tex.w * Q / (np.abs(dx) * W)))
return _finish(tex, u, v, fp, p)
def _finish(tex, u, v, fp, p):
ang = math.radians(float(p.get("angle", 0)))
if ang:
c, s = math.cos(ang), math.sin(ang)
u, v = u * c - v * s, u * s + v * c
return sample_mipmapped(tex, u.astype(np.float32), v.astype(np.float32), fp.astype(np.float32))
# ------------------------------------------------------------------ views
def _img(a):
return Image.fromarray((np.clip(a, 0, 1) * 255 + 0.5).astype(np.uint8))
def _contour(mask):
inner = ~imgops.dilate_bool(~mask, 1)
return mask & ~inner
def _dim(arr, f=0.28):
return arr * f + 0.0
def view_detection(L, al, ids):
out = L.arr.copy()
for name, col in (("wall", (0.26, 0.53, 0.96)), ("floor", (0.96, 0.65, 0.14))):
a = al[name][..., None] * 0.30
out = out * (1 - a) + np.array(col, np.float32) * a
c = _contour(al[name] > 0.5)[..., None]
out = np.where(c, np.array(col, np.float32), out)
return _img(out)
def view_surface(L, al, name):
col = (0.96, 0.65, 0.14) if name == "floor" else (0.26, 0.53, 0.96)
a = al[name][..., None]
out = L.arr * a + _dim(L.arr) * (1 - a)
c = _contour(al[name] > 0.5)[..., None]
return _img(np.where(c, np.array(col, np.float32), out))
def view_confidence(L, al):
c = np.clip((L.conf - 0.5) / 0.5, 0, 1)
r, g = np.clip(2 * (1 - c), 0, 1), np.clip(2 * c, 0, 1)
heat = np.stack([r, g, np.full_like(c, 0.15)], -1)
return _img(_dim(L.arr, 0.45) * 0.45 + heat * 0.55)
def _palette(lbl, n=None):
pal = np.array([PALETTE[i % len(PALETTE)] for i in range(256)], np.float32) / 255.0
return pal[lbl]
def debug_view(sess, name, L, al, q):
det = sess.det
H, W = L.H, L.W
if name == "preprocessed":
v = det.preproc
s = min(W / v.width, H / v.height)
v = v.resize((max(1, round(v.width * s)), max(1, round(v.height * s))), Image.NEAREST)
bg = Image.new("RGB", (W, H), (24, 26, 30))
bg.paste(v, ((W - v.width) // 2, (H - v.height) // 2))
return bg
if name == "rawclass":
return _img(0.35 * L.arr + 0.65 * _palette(L.raw_label))
if name == "band":
b = L.band[..., None].astype(np.float32)
return _img(L.arr * (1 - 0.55 * b) + np.array([1.0, 0.2, 0.5], np.float32) * 0.55 * b)
if name == "refined":
return _img(0.35 * L.arr + 0.65 * _palette(L.label))
return view_detection(L, al, sess.det.ids)
# ------------------------------------------------------------------ main entry
def render(sess: Session, settings: dict, store, view="result", quality="prev", strokes=()):
q = "full" if quality == "full" else "prev"
L = sess.layer(q)
al, ver = sess.alphas(q, list(strokes))
arr = L.arr
H, W = L.H, L.W
if view == "original":
return _img(arr)
if view == "detection":
return view_detection(L, al, sess.det.ids)
if view in ("floor", "wall"):
return view_surface(L, al, view)
if view == "confidence":
return view_confidence(L, al)
if view.startswith("debug:"):
return debug_view(sess, view[6:], L, al, q)
out = arr.copy()
geo = None
for name in ("wall", "floor"):
p = settings.get(name)
if not p or not p.get("tex") or not store.valid(p["tex"]):
continue
a = al[name]
m = a > 0.004
if not m.any():
continue
if geo is None:
geo = sess.geo(q, list(strokes))
ys, xs = np.nonzero(m)
fx, fy = xs.astype(np.float32), ys.astype(np.float32)
color = parse_color(p["tex"])
if color is not None:
layer = np.empty((len(xs), 3), np.float32)
layer[:] = color
tmean = None
else:
tex = store.get(p["tex"], p.get("tiling", "auto"))
tmean = tex.mean
if name == "floor":
layer = map_floor(tex, fx, fy, H, W, p, geo, float(ys.max()))
else:
layer = map_wall(tex, fx, fy, H, W, p, geo)
con = float(p.get("contrast", 1.0))
if tmean is not None and con != 1.0:
layer = tmean + (layer - tmean) * con
ratio0, hp = sess.lighting(q, name, a, ver)
shade = float(p.get("shade", 0.2))
ratio = np.clip(ratio0[ys, xs], 0.3, 2.0) ** shade
refl = float(p.get("reflect", 0.2))
spec = 1.0 + refl * np.clip(hp[ys, xs], -0.5, 0.8)
layer = np.clip(layer * (ratio * spec * float(p.get("bright", 0.7)))[:, None], 0, 1)
w = (a[ys, xs] * float(p.get("blend", 1.0)))[:, None]
out[ys, xs] = out[ys, xs] * (1 - w) + layer * w
return _img(out)