Spaces:
Running
Running
File size: 3,328 Bytes
ab0a0fc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 | import sys, json, time, math
import numpy as np, onnxruntime as ort
from PIL import Image
from scipy.optimize import least_squares
from functools import partial
def uv_grid(W, H):
ar = W / H
sx = ar / math.sqrt(1 + ar**2); sy = 1 / math.sqrt(1 + ar**2)
u = np.linspace(-sx * (W - 1) / W, sx * (W - 1) / W, W)
v = np.linspace(-sy * (H - 1) / H, sy * (H - 1) / H, H)
u, v = np.meshgrid(u, v, indexing='xy')
return np.stack([u, v], -1)
def nearest_idx(n_in, n_out):
# torch F.interpolate mode='nearest': src = floor(dst * in/out)
return np.minimum(np.floor(np.arange(n_out) * (n_in / n_out)).astype(int), n_in - 1)
def solve_focal_shift(uv, xyz):
uv, xy, z = uv.reshape(-1, 2), xyz[..., :2].reshape(-1, 2), xyz[..., 2].reshape(-1)
def fn(shift):
xy_proj = xy / (z + shift)[:, None]
f = (xy_proj * uv).sum() / np.square(xy_proj).sum()
return (f * xy_proj - uv).ravel()
sol = least_squares(fn, x0=0, ftol=1e-3, method='lm')
s = float(sol['x'].squeeze())
xy_proj = xy / (z + s)[:, None]
f = (xy_proj * uv).sum() / np.square(xy_proj).sum()
return s, float(f)
def postprocess(points, mask_p, scale):
H, W = mask_p.shape
mask = mask_p > 0.5
uv = uv_grid(W, H)
ri, ci = nearest_idx(H, 64), nearest_idx(W, 64)
p_lr = points[ri][:, ci]; uv_lr = uv[ri][:, ci]; m_lr = mask[ri][:, ci]
shift, focal = solve_focal_shift(uv_lr[m_lr], p_lr[m_lr])
ar = W / H
fx = focal / 2 * math.sqrt(1 + ar**2) / ar; fy = focal / 2 * math.sqrt(1 + ar**2)
depth = points[..., 2] + shift
mask &= depth > 0
u = (np.arange(W) + 0.5) / W; v = (np.arange(H) + 0.5) / H
uu, vv = np.meshgrid(u, v)
X = (uu - 0.5) / fx * depth; Y = (vv - 0.5) / fy * depth
P = np.stack([X, Y, depth], -1) * scale
return dict(points=P, depth=depth * scale, mask=mask, fx=fx, fy=fy, shift=shift, focal=focal,
fov_x=math.degrees(2 * math.atan(0.5 / fx)), fov_y=math.degrees(2 * math.atan(0.5 / fy)))
def run(path, long_side=512, num_tokens=1800, sess=None):
im = Image.open(path).convert('RGB')
w, h = im.size; s = long_side / max(w, h)
W, H = round(w * s), round(h * s)
im = im.resize((W, H), Image.BICUBIC)
x = (np.asarray(im).astype(np.float32) / 255).transpose(2, 0, 1)[None]
t = time.time()
out = sess.run(None, {'image': x, 'num_tokens': np.array(num_tokens, dtype=np.int64)})
dt = time.time() - t
pts, nrm, msk, sc = out
r = postprocess(pts[0].astype(np.float64), msk[0], float(sc[0]))
r.update(ms=dt * 1000, W=W, H=H, scale=float(sc[0]), normal=nrm[0])
return r
if __name__ == '__main__':
so = ort.SessionOptions(); so.intra_op_num_threads = 4
sess = ort.InferenceSession('model.onnx', so, providers=['CPUExecutionProvider'])
res = {}
for f in sys.argv[1:]:
r = run(f, sess=sess)
d = r['depth'][r['mask']]
info = {k: r[k] for k in ['W', 'H', 'ms', 'scale', 'shift', 'focal', 'fov_x', 'fov_y']}
info.update(depth_min=float(d.min()), depth_med=float(np.median(d)), depth_max=float(d.max()), mask_pct=float(r['mask'].mean() * 100))
print(f, json.dumps(info))
import os; np.save('/workspace/geometria3d-lab/_work/' + os.path.basename(f) + '.points.npy', r['points'].astype(np.float32))
res[f] = info
|