import math import torch from torch import nn import numpy as np def fov2focal(fov: float, pixels: float) -> float: return pixels / (2 * math.tan(fov / 2)) def focal2fov(focal: float, pixels: float) -> float: return 2 * math.atan(pixels / (2 * focal)) class Camera(nn.Module): def __init__( self, rot: np.ndarray = None, # rotation matrix, shape: [3, 3] pos: np.ndarray = None, # translation vector, shape: [3] width: int = 1024, height: int = 1024, znear: float = 0.001, zfar: float = 100.0 ) -> None: super(Camera, self).__init__() self.znear = znear self.zfar = zfar self.width = width self.height = height self._cx = None self._cy = None self.fov_x = None self.fov_y = None if rot is None: rot = np.eye(3) if pos is None: pos = np.zeros([3]) rot = rot.astype(np.float32) # caution the type from numpy pos = pos.astype(np.float32) self.register_buffer('_proj', torch.eye(4, dtype=torch.float32)) self.register_buffer('_w2v', torch.eye(4, dtype=torch.float32)) self.register_buffer('_v2w', torch.eye(4, dtype=torch.float32)) self.register_buffer('_rot', torch.from_numpy(rot)) self.register_buffer('_pos', torch.from_numpy(pos)) self._update_world2view_matrix() self._update_view2world_matrix() @property def cx(self): return (self.width - 1) / 2 if self._cx is None else self._cx # caution: default width @property def cy(self): return (self.height - 1) / 2 if self._cy is None else self._cy # caution: default height @property def fx(self): return fov2focal(self.fov_x, self.width) @property def fy(self): return fov2focal(self.fov_y, self.height) @property def intrisics(self): return torch.tensor([ [self.fx, 0.0, self.cx], [0.0, self.fy, self.cy], [0.0, 0.0, 1.0] ], dtype=self._proj.dtype, device=self._proj.device) @property def extrinsics(self): return self._w2v @property def get_pos(self) -> torch.Tensor: return self._pos @property def get_w2v(self) -> torch.Tensor: return self._w2v @property def get_v2w(self) -> torch.Tensor: return self._v2w @property def get_proj(self) -> torch.Tensor: return self._proj @property def get_full_proj(self) -> torch.Tensor: return torch.matmul(self._proj, self._w2v) def _update_projection_matrix(self) -> torch.Tensor: n, f = self.znear, self.zfar w, h = self.width, self.height P = torch.zeros_like(self._proj) P[0, 0] = 2 * self.fx / w P[1, 1] = 2 * self.fy / h # consider camera center cx, cy P[0, 2] = -1 + 2 * (self.cx / w) P[1, 2] = -1 + 2 * (self.cy / h) # z = zfar, depth = 1.0; z = znear, depth = 0.0 P[3, 2] = 1.0 P[2, 2] = f / (f - n) P[2, 3] = -(f * n) / (f - n) self._proj.copy_(P) def _update_view2world_matrix(self) -> torch.Tensor: self._v2w[:3, :3].copy_(self._rot) self._v2w[:3, 3].copy_(self._pos) def _update_world2view_matrix(self) -> torch.Tensor: R = self._rot.transpose(0, 1) T = -torch.matmul(R, self._pos) self._w2v[:3, :3].copy_(R) self._w2v[:3, 3].copy_(T) def set_pose(self, pose: torch.Tensor | np.ndarray) -> None: if isinstance(pose, np.ndarray): pose = torch.from_numpy(pose).to(self._rot.device, self._rot.dtype) self._rot.copy_(pose[:3, :3]) self._pos.copy_(pose[:3, 3]) self._update_view2world_matrix() self._update_world2view_matrix() def project_points(self, points: torch.Tensor, in_viewport: bool = False) -> tuple[torch.Tensor, torch.Tensor]: points_pad = torch.nn.functional.pad(points, [0, 1], value=1.0) proj_mat = self.get_full_proj.to(points_pad.device) points_proj = torch.matmul(points_pad, proj_mat.transpose(0, 1)) points_proj = points_proj[..., :3] / torch.clamp_min(points_proj[..., 3:4], 1e-8) filter_mask = torch.logical_and(points_proj[..., 2] > 0.0, points_proj[..., 2] < 1.0) if in_viewport: points_proj[..., :2] = points_proj[..., :2] * 0.5 + 0.5 points_proj[..., 0] = points_proj[..., 0] * self.width points_proj[..., 1] = points_proj[..., 1] * self.height return points_proj, filter_mask def clone(self): new_camera = Camera( rot=self._rot.cpu().numpy(), pos=self._pos.cpu().numpy(), width=self.width, height=self.height, znear=self.znear, zfar=self.zfar ) new_camera._cx = self._cx new_camera._cy = self._cy new_camera.fov_x = self.fov_x new_camera.fov_y = self.fov_y new_camera._proj.copy_(self._proj) new_camera._w2v.copy_(self._w2v) new_camera._v2w.copy_(self._v2w) return new_camera def crop(self, x_min: int, y_min: int, x_max: int, y_max: int): # assert x_min >= 0 and x_max <= self.width # NOTE: crop may go beyond image border # assert y_min >= 0 and y_max <= self.height new_width = x_max - x_min new_height = y_max - y_min new_fov_x = focal2fov(self.fx, new_width) new_fov_y = focal2fov(self.fy, new_height) adjust_position = False if adjust_position: # TODO: some bugs here new_cx = self.cx / self.width * new_width new_cy = self.cy / self.height * new_height u_c_new = new_cx + x_min v_c_new = new_cy + y_min shift_cam = torch.tensor([(u_c_new - self.cx) / self.fx, (v_c_new - self.cy) / self.fy, 0.0], device=self._pos.device, dtype=self._pos.dtype) shift_world = torch.matmul(self._rot, shift_cam) self._pos += shift_world self._update_world2view_matrix() self._update_view2world_matrix() else: new_cx = self.cx - x_min new_cy = self.cy - y_min self._cx = new_cx self._cy = new_cy self.fov_x = new_fov_x self.fov_y = new_fov_y self.width = new_width self.height = new_height self._update_projection_matrix() def resize(self, width: int, height: int): if self._cx is not None: self._cx = self._cx / self.width * width if self._cy is not None: self._cy = self._cy / self.height * height self.width = width self.height = height self._update_projection_matrix() def flip(self, axis: str = 'x'): # Caution: change winding order if axis == 'x': self._rot[:, 0] *= -1 self._cx = self.width - self.cx elif axis == 'y': self._rot[:, 1] *= -1 self._cy = self.height - self.cy else: raise ValueError("axis must be 'x' or 'y'") self._update_world2view_matrix() self._update_view2world_matrix() self._update_projection_matrix() def rotate90(self, clockwise: bool = True): device = self._rot.device dtype = self._rot.dtype if clockwise: trans_mat = torch.tensor([ [0.0, 1.0, 0.0], [-1.0, 0.0, 0.0], [0.0, 0.0, 1.0] ], device=device, dtype=dtype) else: trans_mat = torch.tensor([ [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0] ], device=device, dtype=dtype) new_rot = torch.matmul(self._rot, trans_mat) self._rot.copy_(new_rot) old_w, old_h = self.width, self.height old_cx, old_cy = self.cx, self.cy old_fov_x, old_fov_y = self.fov_x, self.fov_y self.width, self.height = old_h, old_w self.fov_x, self.fov_y = old_fov_y, old_fov_x if clockwise: # (x, y) -> (y, W-x) self._cx = old_h - old_cy self._cy = old_cx else: # (x, y) -> (H-y, x) self._cx = old_cy self._cy = old_w - old_cx self._update_world2view_matrix() self._update_view2world_matrix() self._update_projection_matrix() def print(self): print("Camera Parameters:") print("width:", self.width, "height:", self.height) print("focal x:", self.fx, "focal y:", self.fy) print("fov x:", self.fov_x, "fov y:", self.fov_y) print("principal point x:", self.cx, "principal point y:", self.cy) print("position:", self.get_pos) print("projection matrix:", self.get_proj) print("view2world matrix:", self.get_v2w) print("world2view matrix:", self.get_w2v) class PerspectiveCamera(Camera): def __init__( self, fov_y: float = np.pi / 3, rot: np.ndarray = None, # rotation matrix, shape: [3, 3] pos: np.ndarray = None, # translation vector, shape: [3] width: int = 1024, height: int = 1024, znear: float = 0.01, zfar: float = 100.0 ): super(PerspectiveCamera, self).__init__( rot=rot, pos=pos, width=width, height=height, znear=znear, zfar=zfar ) self.fov_y = fov_y focal = fov2focal(fov_y, height) self.fov_x = focal2fov(focal, width) # caution: compute fovx from fovy self._update_projection_matrix() def set_fov_y(self, fov_y: float) -> None: self.fov_y = float(fov_y) focal = fov2focal(self.fov_y, self.height) self.fov_x = focal2fov(focal, self.width) # caution: compute fovx from fovy self._update_projection_matrix() def resize(self, width: int, height: int): super().resize(width, height) # keep vertical fov fixed, recompute horizontal fov for the new aspect focal = fov2focal(self.fov_y, self.height) self.fov_x = focal2fov(focal, self.width) self._update_projection_matrix() class IntrinsicsCamera(Camera): def __init__( self, K: np.ndarray = None, # intrinsic parameters, shape: [3, 3] R: np.ndarray = None, # extrinsic rotation matrix, shape: [3, 3] T: np.ndarray = None, # extrinsic translation vector, shape: [3] D: np.ndarray = None, # distortion width: int = 1024, height: int = 1024, znear: float = 0.01, zfar: float = 100 ): # world2view -> view2world rot = np.transpose(R) pos = -np.dot(rot, T) super(IntrinsicsCamera, self).__init__( rot=rot, pos=pos, width=width, height=height, znear=znear, zfar=zfar ) fx, fy, self._cx, self._cy = K[0, 0].item(), K[1, 1].item(), K[0, 2].item(), K[1, 2].item() # add item() to convert numpy scalar to float (some numpy version issue) self.fov_x, self.fov_y = focal2fov(fx, width), focal2fov(fy, height) self.K = K self.D = D self._update_projection_matrix() class OrthographicCamera(Camera): def __init__( self, xmag: float = 1.0, ymag: float | None = None, rot: np.ndarray = None, pos: np.ndarray = None, width: int = 1024, height: int = 1024, cx: float | None = None, cy: float | None = None, znear: float = -100.0, zfar: float = 100.0, ): super(OrthographicCamera, self).__init__( rot=rot, pos=pos, width=width, height=height, znear=znear, zfar=zfar, ) self.xmag = float(xmag) self.ymag = float(xmag if ymag is None else ymag) self._cx = (width - 1) * 0.5 if cx is None else float(cx) self._cy = (height - 1) * 0.5 if cy is None else float(cy) self._update_projection_matrix() def _update_projection_matrix(self) -> torch.Tensor: n, f = self.znear, self.zfar w, h = self.width, self.height P = torch.zeros_like(self._proj) P[0, 0] = 1.0 / self.xmag P[1, 1] = 1.0 / self.ymag P[0, 3] = (w - 2.0 * self.cx) / w P[1, 3] = (h - 2.0 * self.cy) / h P[2, 2] = 1.0 / (f - n) P[2, 3] = -n / (f - n) P[3, 3] = 1.0 self._proj.copy_(P)