Segizu's picture
Geometría 3D Lab v1: MoGe-2 ViT-S en el navegador (medidas 3D métricas)
ab0a0fc verified
Raw History Blame Contribute Delete
3.33 kB
import sys, json, time, math
import numpy as np, onnxruntime as ort
from PIL import Image
from scipy.optimize import least_squares
from functools import partial
def uv_grid(W, H):
ar = W / H
sx = ar / math.sqrt(1 + ar**2); sy = 1 / math.sqrt(1 + ar**2)
u = np.linspace(-sx * (W - 1) / W, sx * (W - 1) / W, W)
v = np.linspace(-sy * (H - 1) / H, sy * (H - 1) / H, H)
u, v = np.meshgrid(u, v, indexing='xy')
return np.stack([u, v], -1)
def nearest_idx(n_in, n_out):
# torch F.interpolate mode='nearest': src = floor(dst * in/out)
return np.minimum(np.floor(np.arange(n_out) * (n_in / n_out)).astype(int), n_in - 1)
def solve_focal_shift(uv, xyz):
uv, xy, z = uv.reshape(-1, 2), xyz[..., :2].reshape(-1, 2), xyz[..., 2].reshape(-1)
def fn(shift):
xy_proj = xy / (z + shift)[:, None]
f = (xy_proj * uv).sum() / np.square(xy_proj).sum()
return (f * xy_proj - uv).ravel()
sol = least_squares(fn, x0=0, ftol=1e-3, method='lm')
s = float(sol['x'].squeeze())
xy_proj = xy / (z + s)[:, None]
f = (xy_proj * uv).sum() / np.square(xy_proj).sum()
return s, float(f)
def postprocess(points, mask_p, scale):
H, W = mask_p.shape
mask = mask_p > 0.5
uv = uv_grid(W, H)
ri, ci = nearest_idx(H, 64), nearest_idx(W, 64)
p_lr = points[ri][:, ci]; uv_lr = uv[ri][:, ci]; m_lr = mask[ri][:, ci]
shift, focal = solve_focal_shift(uv_lr[m_lr], p_lr[m_lr])
ar = W / H
fx = focal / 2 * math.sqrt(1 + ar**2) / ar; fy = focal / 2 * math.sqrt(1 + ar**2)
depth = points[..., 2] + shift
mask &= depth > 0
u = (np.arange(W) + 0.5) / W; v = (np.arange(H) + 0.5) / H
uu, vv = np.meshgrid(u, v)
X = (uu - 0.5) / fx * depth; Y = (vv - 0.5) / fy * depth
P = np.stack([X, Y, depth], -1) * scale
return dict(points=P, depth=depth * scale, mask=mask, fx=fx, fy=fy, shift=shift, focal=focal,
fov_x=math.degrees(2 * math.atan(0.5 / fx)), fov_y=math.degrees(2 * math.atan(0.5 / fy)))
def run(path, long_side=512, num_tokens=1800, sess=None):
im = Image.open(path).convert('RGB')
w, h = im.size; s = long_side / max(w, h)
W, H = round(w * s), round(h * s)
im = im.resize((W, H), Image.BICUBIC)
x = (np.asarray(im).astype(np.float32) / 255).transpose(2, 0, 1)[None]
t = time.time()
out = sess.run(None, {'image': x, 'num_tokens': np.array(num_tokens, dtype=np.int64)})
dt = time.time() - t
pts, nrm, msk, sc = out
r = postprocess(pts[0].astype(np.float64), msk[0], float(sc[0]))
r.update(ms=dt * 1000, W=W, H=H, scale=float(sc[0]), normal=nrm[0])
return r
if __name__ == '__main__':
so = ort.SessionOptions(); so.intra_op_num_threads = 4
sess = ort.InferenceSession('model.onnx', so, providers=['CPUExecutionProvider'])
res = {}
for f in sys.argv[1:]:
r = run(f, sess=sess)
d = r['depth'][r['mask']]
info = {k: r[k] for k in ['W', 'H', 'ms', 'scale', 'shift', 'focal', 'fov_x', 'fov_y']}
info.update(depth_min=float(d.min()), depth_med=float(np.median(d)), depth_max=float(d.max()), mask_pct=float(r['mask'].mean() * 100))
print(f, json.dumps(info))
import os; np.save('/workspace/geometria3d-lab/_work/' + os.path.basename(f) + '.points.npy', r['points'].astype(np.float32))
res[f] = info