from typing import * from typing_extensions import Unpack import math from numbers import Number import torch from torch import Tensor import torch.nn.functional as F from .mesh import remove_unused_vertices, triangulate_mesh from .transforms import unproject_cv from .utils import masked_max, masked_min, sliding_window from .helpers import batched __all__ = [ 'uv_map', 'pixel_coord_map', 'screen_coord_map', 'build_mesh_from_map', 'build_mesh_from_depth_map', 'depth_map_edge', 'depth_map_aliasing', 'point_map_to_normal_map', 'depth_map_to_point_map', 'depth_map_to_normal_map', 'chessboard', 'bounding_rect_from_mask', 'masked_nearest_resize', 'masked_area_resize', 'flood_fill', 'perlin_noise', 'perlin_noise_map', 'fractal_perlin_noise_map' ] def uv_map( *size: Union[int, Tuple[int, int]], top: float = 0., left: float = 0., bottom: float = 1., right: float = 1., dtype: torch.dtype = torch.float32, device: torch.device = None ) -> Tensor: """ Get image UV coordinate map. By default, (0., 0.) is the top-left corner of the image, and (1., 1.) is the bottom-right corner of the image. ## Parameters - `*size`: `Tuple[int, int]` or two integers of map size `(height, width)` - `top`: `float` defaults to 0. - `left`: `float` defaults to 0. - `bottom`: `float` defaults to 1. - `right`: `float` defaults to 1. - `dtype`: `torch.dtype` data type of the output uv map. Defaults to torch.float32. - `device`: `torch.device`, device of the output uv map. Defaults to None. ## Returns - `uv (Tensor)`: shape `(height, width, 2)` ## Example Usage >>> uv_map(10, 10): [[[0.05, 0.05], [0.15, 0.05], ..., [0.95, 0.05]], [[0.05, 0.15], [0.15, 0.15], ..., [0.95, 0.15]], ... ... ... [[0.05, 0.95], [0.15, 0.95], ..., [0.95, 0.95]]] """ if len(size) == 1 and isinstance(size[0], tuple): height, width = size[0] else: height, width = size u = torch.linspace(left + 0.5 / width * (right - left), right - 0.5 / width * (right - left), width, dtype=dtype, device=device) v = torch.linspace(top + 0.5 / height * (bottom - top), bottom - 0.5 / height * (bottom - top), height, dtype=dtype, device=device) uv = torch.stack(torch.meshgrid(u, v, indexing='xy'), dim=-1) return uv def pixel_coord_map( *size: Union[int, Tuple[int, int]], top: int = 0, left: int = 0, convention: Literal['integer-center', 'integer-corner'] = 'integer-center', dtype: torch.dtype = torch.float32, device: torch.device = None ) -> Tensor: """ Get image pixel coordinates map. Support two conventions: `'integer-center'` and `'integer-corner'`. ## Parameters - `*size`: `Tuple[int, int]` or two integers of map size `(height, width)` - `top`: `int`, optional top boundary of the pixel coord map. Defaults to 0. - `left`: `int`, optional left boundary of the pixel coord map. Defaults to 0. - `convention`: `str`, optional `'integer-center'` or `'integer-corner'`, whether integer coordinates correspond to pixel centers or corners. Defaults to 'integer-center'. - `'integer-center'`: `pixel[i][j]` has integer coordinates `(j, i)` as its center, and occupies square area `[j - 0.5, j + 0.5) × [i - 0.5, i + 0.5)`. The top-left corner of the top-left pixel is `(-0.5, -0.5)`, and the bottom-right corner of the bottom-right pixel is `(width - 0.5, height - 0.5)`. - `'integer-corner'`: `pixel[i][j]` has coordinates `(j + 0.5, i + 0.5)` as its center, and occupies square area `[j, j + 1) × [i, i + 1)`. The top-left corner of the top-left pixel is `(0, 0)`, and the bottom-right corner of the bottom-right pixel is `(width, height)`. - `dtype`: `torch.dtype`, optional data type of the output pixel coord map. Defaults to torch.float32. ## Returns Tensor: shape (height, width, 2) >>> pixel_coord_map(10, 10, convention='integer-center', dtype=torch.long): [[[0, 0], [1, 0], ..., [9, 0]], [[0, 1], [1, 1], ..., [9, 1]], ... ... ... [[0, 9], [1, 9], ..., [9, 9]]] >>> pixel_coord_map(10, 10, convention='integer-corner', dtype=torch.float32): [[[0.5, 0.5], [1.5, 0.5], ..., [9.5, 0.5]], [[0.5, 1.5], [1.5, 1.5], ..., [9.5, 1.5]], ... ... ... [[0.5, 9.5], [1.5, 9.5], ..., [9.5, 9.5]]] """ if len(size) == 1 and isinstance(size[0], tuple): height, width = size[0] else: height, width = size u = torch.arange(left, left + width, dtype=dtype, device=device) v = torch.arange(top, top + height, dtype=dtype, device=device) if convention == 'integer-corner': assert torch.is_floating_point(u), "dtype should be a floating point type when convention is 'integer-corner'" u = u + 0.5 v = v + 0.5 u, v = torch.meshgrid(u, v, indexing='xy') return torch.stack([u, v], dim=2) def screen_coord_map( *size: Union[int, Tuple[int, int]], top: float = 1., left: float = 0., bottom: float = 0., right: float = 1., dtype: torch.dtype = torch.float32, device: torch.device = None ) -> Tensor: """ Get screen space coordinate map, where (0., 0.) is the bottom-left corner of the image, and (1., 1.) is the top-right corner of the image. This is commonly used in graphics APIs like OpenGL. ## Parameters - `*size`: `Tuple[int, int]` or two integers of map size `(height, width)` - `top`: `float`, optional top boundary in the screen space. Defaults to 1. - `left`: `float`, optional left boundary in the screen space. Defaults to 0. - `bottom`: `float`, optional bottom boundary in the screen space. Defaults to 0. - `right`: `float`, optional right boundary in the screen space. Defaults to 1. - `dtype`: `torch.dtype`, optional data type of the output map. Defaults to torch.float32. ## Returns (Tensor): shape (height, width, 2) """ if len(size) == 1 and isinstance(size[0], tuple): height, width = size[0] else: height, width = size x = torch.linspace(left + 0.5 / width, right - 0.5 / width, width, dtype=dtype, device=device) y = torch.linspace(top - 0.5 / height, bottom - 0.5 / height, height, dtype=dtype, device=device) x, y = torch.meshgrid(x, y, indexing='xy') return torch.stack([x, y], dim=2) def build_mesh_from_map( *maps: Tensor, mask: Optional[Tensor] = None, tri: bool = False, ) -> Tuple[Tensor, ...]: """ Get a mesh regarding image pixel uv coordinates as vertices and image grid as faces. ## Parameters *maps (Tensor): attribute maps in shape (height, width, [channels]) mask (Tensor, optional): binary mask of shape (height, width), dtype=bool. Defaults to None. ## Returns faces (Tensor): faces connecting neighboring pixels. shape (T, 4) if tri is False, else (T, 3) *attributes (Tensor): vertex attributes in corresponding order with itorchut maps indices (Tensor, optional): indices of vertices in the original mesh """ assert (len(maps) > 0) or (mask is not None), "At least one of maps or mask should be provided" height, width = maps[0].shape[:2] if mask is None else mask.shape device = maps[0].device if len(maps) > 0 else mask.device assert all(x.shape[:2] == (height, width) for x in maps), "All maps should have the same shape" row_faces = torch.stack([ torch.arange(0, width - 1, dtype=torch.int32, device=device), torch.arange(width, 2 * width - 1, dtype=torch.int32, device=device), torch.arange(1 + width, 2 * width, dtype=torch.int32, device=device), torch.arange(1, width, dtype=torch.int32, device=device)], dim=1) faces = (torch.arange(0, (height - 1) * width, width, dtype=torch.int32, device=device)[:, None, None] + row_faces[None, :, :]).reshape((-1, 4)) attributes = tuple(x.reshape(-1, *x.shape[2:]) for x in maps) if mask is not None: quad_mask = (mask[:-1, :-1] & mask[1:, :-1] & mask[1:, 1:] & mask[:-1, 1:]).ravel() faces = faces[quad_mask] faces, *attributes = remove_unused_vertices(faces, *attributes) if tri: faces = triangulate_mesh(faces) return faces, *attributes def build_mesh_from_depth_map( depth: Tensor, *other_maps: Tensor, intrinsics: Tensor, extrinsics: Optional[Tensor] = None, atol: Optional[float] = None, rtol: Optional[float] = 0.05, tri: bool = False, ) -> Tuple[Tensor, ...]: """ Get a mesh by lifting depth map to 3D, while removing depths of large depth difference. ## Parameters depth (Tensor): [H, W] depth map extrinsics (Tensor, optional): [4, 4] extrinsics matrix. Defaults to None. intrinsics (Tensor, optional): [3, 3] intrinsics matrix. Defaults to None. *other_maps (Tensor): [H, W, C] vertex attributes. Defaults to None. atol (float, optional): absolute tolerance. Defaults to None. rtol (float, optional): relative tolerance. Defaults to None. triangles with vertices having depth difference larger than atol + rtol * depth will be marked. remove_by_depth (bool, optional): whether to remove triangles with large depth difference. Defaults to True. return_uv (bool, optional): whether to return uv coordinates. Defaults to False. return_indices (bool, optional): whether to return indices of vertices in the original mesh. Defaults to False. ## Returns faces (Tensor): [T, 3] faces vertices (Tensor): [N, 3] vertices *other_attrs (Tensor): [N, C] vertex attributes """ height, width = depth.shape uv = uv_map(height, width, dtype=depth.dtype, device=depth.device) mask = torch.isfinite(depth) if atol is not None or rtol is not None: mask = mask & ~depth_map_edge(depth, atol=atol, rtol=rtol, kernel_size=3, mask=mask) uv, depth, other_attrs, faces = build_mesh_from_map(uv, depth, *other_maps, mask=mask) pts = unproject_cv(uv, depth, intrinsics, extrinsics) if tri: faces = triangulate_mesh(faces, vertices=pts, method='diagonal') faces, pts, *other_attrs = remove_unused_vertices(faces, pts, *other_attrs) return faces, pts, *other_attrs @batched(2) def depth_map_edge(depth: Tensor, atol: float = None, rtol: float = None, kernel_size: int = 3, mask: Tensor = None) -> torch.BoolTensor: """ Compute the edge mask of a depth map. The edge is defined as the pixels whose neighbors have a large difference in depth. ## Parameters depth (Tensor): shape (..., height, width), linear depth map atol (float): absolute tolerance rtol (float): relative tolerance ## Returns edge (Tensor): shape (..., height, width) of dtype torch.bool """ if mask is None: diff = (F.max_pool2d(depth, kernel_size, stride=1, padding=kernel_size // 2) + F.max_pool2d(-depth, kernel_size, stride=1, padding=kernel_size // 2)) else: diff = (F.max_pool2d(torch.where(mask, depth, -torch.inf), kernel_size, stride=1, padding=kernel_size // 2) + F.max_pool2d(torch.where(mask, -depth, -torch.inf), kernel_size, stride=1, padding=kernel_size // 2)) edge = torch.zeros_like(depth, dtype=torch.bool) if atol is not None: edge |= diff > atol if rtol is not None: edge |= (diff / depth).nan_to_num_() > rtol return edge @batched(2) def depth_map_aliasing(depth: Tensor, atol: float = None, rtol: float = None, kernel_size: int = 3, mask: Tensor = None) -> torch.BoolTensor: """ Compute the map that indicates the aliasing of a depth map. The aliasing is defined as the pixels which neither close to the maximum nor the minimum of its neighbors. ## Parameters depth (Tensor): shape (..., height, width), linear depth map atol (float): absolute tolerance rtol (float): relative tolerance ## Returns edge (Tensor): shape (..., height, width) of dtype torch.bool """ shape = depth.shape depth = depth.reshape(-1, 1, *shape[-2:]) if mask is not None: mask = mask.reshape(-1, 1, *shape[-2:]) if mask is None: diff_max = F.max_pool2d(depth, kernel_size, stride=1, padding=kernel_size // 2) - depth diff_min = F.max_pool2d(-depth, kernel_size, stride=1, padding=kernel_size // 2) + depth else: diff_max = F.max_pool2d(torch.where(mask, depth, -torch.inf), kernel_size, stride=1, padding=kernel_size // 2) - depth diff_min = F.max_pool2d(torch.where(mask, -depth, -torch.inf), kernel_size, stride=1, padding=kernel_size // 2) + depth diff = torch.minimum(diff_max, diff_min) edge = torch.zeros_like(depth, dtype=torch.bool) if atol is not None: edge |= diff > atol if rtol is not None: edge |= (diff / depth).nan_to_num_() > rtol edge = edge.reshape(*shape) return edge @batched(3, 2) def point_map_to_normal_map(point: Tensor, mask: Tensor = None) -> Tensor: """ Calculate normal map from point map. Value range is [-1, 1]. ## Parameters point (Tensor): shape (..., height, width, 3), point map mask (Tensor): shape (..., height, width), binary mask. Defaults to None. ## Returns normal (Tensor): shape (..., height, width, 3), normal map. """ has_mask = mask is not None if mask is None: mask = torch.ones_like(point[..., 0], dtype=torch.bool) mask = F.pad(mask, (1, 1, 1, 1), mode='constant', value=0) pts = F.pad(point.permute(0, 3, 1, 2), (1, 1, 1, 1), mode='constant', value=1).permute(0, 2, 3, 1) up = pts[:, :-2, 1:-1, :] - pts[:, 1:-1, 1:-1, :] left = pts[:, 1:-1, :-2, :] - pts[:, 1:-1, 1:-1, :] down = pts[:, 2:, 1:-1, :] - pts[:, 1:-1, 1:-1, :] right = pts[:, 1:-1, 2:, :] - pts[:, 1:-1, 1:-1, :] normal = torch.stack([ torch.cross(up, left, dim=-1), torch.cross(left, down, dim=-1), torch.cross(down, right, dim=-1), torch.cross(right, up, dim=-1), ]) normal = F.normalize(normal, dim=-1) valid = torch.stack([ mask[:, :-2, 1:-1] & mask[:, 1:-1, :-2], mask[:, 1:-1, :-2] & mask[:, 2:, 1:-1], mask[:, 2:, 1:-1] & mask[:, 1:-1, 2:], mask[:, 1:-1, 2:] & mask[:, :-2, 1:-1], ]) & mask[None, :, 1:-1, 1:-1] normal = (normal * valid[..., None]).sum(dim=0) normal = F.normalize(normal, dim=-1) if has_mask: return normal, valid.any(dim=0) else: return normal @batched(2, 2) def depth_map_to_normal_map(depth: Tensor, intrinsics: Tensor, mask: Tensor = None) -> Tensor: """ Calculate normal map from depth map. Value range is [-1, 1]. Normal direction in OpenCV identity camera's coordinate system. ## Parameters depth (Tensor): shape (..., height, width), linear depth map intrinsics (Tensor): shape (..., 3, 3), intrinsics matrix ## Returns normal (Tensor): shape (..., height, width, 3), normal map. """ pts = depth_map_to_point_map(depth, intrinsics) return point_map_to_normal_map(pts, mask) def depth_map_to_point_map(depth: Tensor, intrinsics: Tensor, extrinsics: Tensor = None): height, width = depth.shape[-2:] uv = uv_map(height, width, dtype=depth.dtype, device=depth.device) pts = unproject_cv(uv, depth, intrinsics=intrinsics[..., None, :, :], extrinsics=extrinsics[..., None, :, :] if extrinsics is not None else None) return pts def bounding_rect_from_mask(mask: torch.BoolTensor): """Get bounding rectangle of a mask ## Parameters mask (Tensor): shape (..., height, width), mask ## Returns rect (Tensor): shape (..., 4), bounding rectangle (left, top, right, bottom) """ height, width = mask.shape[-2:] mask = mask.flatten(-2).unsqueeze(-1) uv = uv_map(height, width).to(mask.device).reshape(-1, 2) left_top = masked_min(uv, mask, dim=-2)[0] right_bottom = masked_max(uv, mask, dim=-2)[0] return torch.cat([left_top, right_bottom], dim=-1) def chessboard(*size: Union[int, Tuple[int, int]], grid_size: int, color_a: Tensor, color_b: Tensor) -> Tensor: """Get a chessboard image ## Parameters - `*size`: `Tuple[int, int]` or two integers of map size `(height, width)` - `grid_size`: `int`, size of chessboard grid - `color_a`: `Tensor`, shape (channels,), color of the grid at the top-left corner - `color_b`: `Tensor`, shape (channels,), color in complementary grids ## Returns - `image` (Tensor): shape (height, width, channels), chessboard image """ if len(size) == 1 and isinstance(size[0], tuple): height, width = size[0] else: height, width = size x = torch.div(torch.arange(width), grid_size, rounding_mode='floor') y = torch.div(torch.arange(height), grid_size, rounding_mode='floor') mask = ((x[None, :] + y[:, None]) % 2).to(color_a) image = (1 - mask[..., None]) * color_a + mask[..., None] * color_b return image def masked_nearest_resize( *image: Tensor, mask: Tensor, size: Tuple[int, int], return_index: bool = False ) -> Tuple[Unpack[Tuple[Tensor, ...]], Tensor, Tuple[Tensor, ...]]: """ Resize image(s) by nearest sampling with mask awareness. Suitable for sparse maps. ![masked_nearest_resize.png](doc/masked_nearest_resize.png) - Downsampling: Assign the nearest valid pixel within the target pixel's receptive field. - Upsampling: Assign the valid pixel to only the nearest pixel in the resized map. ### Parameters - `*image`: Itorchut image(s) of shape `(..., H, W, C)` or `(... , H, W)` - You can pass multiple images to be resized at the same time for efficiency. - `mask`: itorchut mask of shape `(..., H, W)`, dtype=bool - `size`: target size `(H', W')` - `return_index`: whether to return the nearest neighbor indices in the original map for each pixel in the resized map. Defaults to False. ### Returns - `*resized_image`: resized image(s) of shape `(..., H', W', C)`. or `(..., H', W')` - `resized_mask`: mask of the resized map of shape `(..., H', W')` - `nearest_indices`: tuple of shape `(..., H', W')`. The nearest neighbor indices of the resized map of each dimension. """ device = mask.device height, width = mask.shape[-2:] target_height, target_width = size filter_h_f, filter_w_f = height / target_height, width / target_width filter_h_i, filter_w_i = math.ceil(filter_h_f), math.ceil(filter_w_f) filter_size = filter_h_i * filter_w_i filter_shape = (filter_h_i, filter_w_i) padding_h, padding_w = filter_h_i // 2 + 1, filter_w_i // 2 + 1 padding_shape = ((padding_h, padding_h), (padding_w, padding_w)) # Window the original mask and uv pixels = pixel_coord_map(height, width, convention='integer-corner', dtype=torch.float32, device=device) indices = torch.arange(height * width, dtype=torch.long, device=device).reshape(height, width) window_pixels = sliding_window(pixels, window_size=filter_shape, pad_size=padding_shape, dim=(0, 1)) window_indices = sliding_window(indices, window_size=filter_shape, pad_size=padding_shape, dim=(0, 1)) window_mask = sliding_window(mask, window_size=filter_shape, pad_size=padding_shape, dim=(-2, -1)) # Gather the target pixels's local window target_centers = uv_map(target_height, target_width, dtype=torch.float32, device=device) * torch.tensor([width, height], dtype=torch.float32, device=device) target_lefttop = target_centers - torch.tensor((filter_w_f / 2, filter_h_f / 2), dtype=torch.float32, device=device) target_window = torch.round(target_lefttop).to(torch.long) + torch.tensor((padding_w, padding_h), dtype=torch.long, device=device) target_window_pixels = window_pixels[target_window[..., 1], target_window[..., 0], :, :, :].reshape(target_height, target_width, 2, filter_size) # (target_height, tgt_width, 2, filter_size) target_window_mask = window_mask[..., target_window[..., 1], target_window[..., 0], :, :].reshape(*mask.shape[:-2], target_height, target_width, filter_size) # (..., target_height, tgt_width, filter_size) target_window_indices = window_indices[target_window[..., 1], target_window[..., 0], :, :].reshape(target_height, target_width, filter_size) # (target_height, tgt_width, filter_size) # Compute nearest neighbor in the local window for each pixel delta = target_window_pixels - target_centers[..., None] eps = torch.finfo(torch.float32).eps * max(height, width) # Shift a small epsilon to avoid numerical issues when the pixel is exactly on the border target_window_mask &= (-filter_w_f / 2 + eps < delta[..., 0, :]) & (delta[..., 0, :] <= filter_w_f / 2 + eps) & (-filter_h_f / 2 + eps < delta[..., 1, :]) & (delta[..., 1, :] <= filter_h_f / 2 + eps) dist = torch.where(target_window_mask, torch.square(delta[..., 0, :]) + torch.square(delta[..., 1, :]), torch.inf) # (..., target_height, tgt_width, filter_size) nearest_in_window = torch.argmin(dist, dim=-1, keepdim=True) # (..., target_height, tgt_width, 1) nearest_idx = torch.gather( target_window_indices.expand(dist.shape), dim=-1, index=nearest_in_window, ).squeeze(-1) # (..., target_height, tgt_width) nearest_i, nearest_j = nearest_idx // width, nearest_idx % width target_mask = torch.any(target_window_mask, dim=-1) batch_indices = [torch.arange(n, device=device).reshape([1] * i + [n] + [1] * (mask.ndim - i - 1)) for i, n in enumerate(mask.shape[:-2])] nearest_indices = (*batch_indices, nearest_i, nearest_j) outputs = tuple(x[nearest_indices] for x in image) if return_index: ret = (*outputs, target_mask, nearest_indices) else: ret = (*outputs, target_mask) if len(ret) == 1: return ret[0] else: return ret def masked_area_resize( *image: Tensor, mask: Tensor, size: Tuple[int, int] ) -> Tuple[Unpack[Tuple[Tensor, ...]], Tensor]: """ Resize 2D map by area sampling with mask awareness. ### Parameters - `*image`: Itorchut image(s) of shape `(..., H, W, C)` or `(..., H, W)` - You can pass multiple images to be resized at the same time for efficiency. - `mask`: Itorchut mask of shape `(..., H, W)` - `size`: target image size `(H', W')` ### Returns - `*resized_image`: resized image(s) of shape `(..., H', W', C)`. or `(..., H', W')` - `resized_mask`: mask of the resized map of shape `(..., H', W')` """ device = mask.device height, width = mask.shape[-2:] target_height, target_width = size filter_h_f, filter_w_f = max(1, height / target_height), max(1, width / target_width) filter_h_i, filter_w_i = math.ceil(filter_h_f), math.ceil(filter_w_f) filter_size = filter_h_i * filter_w_i filter_shape = (filter_h_i, filter_w_i) padding_h, padding_w = filter_h_i // 2 + 1, filter_w_i // 2 + 1 padding_shape = ((padding_h, padding_h), (padding_w, padding_w)) # Window the original mask and uv (non-copy) pixels = pixel_coord_map((height, width), convention='integer-corner', dtype=torch.float32, device=device) indices = torch.arange(height * width, dtype=torch.long, device=device).reshape(height, width) window_pixels = sliding_window(pixels, window_size=filter_shape, pad_size=padding_shape, dim=(0, 1)) window_indices = sliding_window(indices, window_size=filter_shape, pad_size=padding_shape, dim=(0, 1)) window_mask = sliding_window(mask, window_size=filter_shape, pad_size=padding_shape, dim=(-2, -1)) # Gather the target pixels's local window target_center = uv_map((target_height, target_width), dtype=torch.float32, device=device) * torch.tensor([width, height], dtype=torch.float32, device=device) target_lefttop = target_center - torch.tensor((filter_w_f / 2, filter_h_f / 2), dtype=torch.float32, device=device) target_bottomright = target_center + torch.tensor((filter_w_f / 2, filter_h_f / 2), dtype=torch.float32, device=device) target_window = torch.floor(target_lefttop).to(torch.long) + torch.tensor((padding_w, padding_h), dtype=torch.long, device=device) target_window_centers = window_pixels[target_window[..., 1], target_window[..., 0], :, :, :].reshape(target_height, target_width, 2, filter_size) # (target_height, tgt_width, 2, filter_size) target_window_mask = window_mask[..., target_window[..., 1], target_window[..., 0], :, :].reshape(*mask.shape[:-2], target_height, target_width, filter_size) # (..., target_height, tgt_width, filter_size) target_window_indices = window_indices[target_window[..., 1], target_window[..., 0], :, :].reshape(target_height, target_width, filter_size) # (target_height, tgt_width, filter_size) # Compute pixel area in the local windows # (..., target_height, tgt_width, filter_size) target_window_lefttop = torch.maximum(target_window_centers - 0.5, target_lefttop[..., None]) target_window_rightbottom = torch.minimum(target_window_centers + 0.5, target_bottomright[..., None]) target_window_area = (target_window_rightbottom - target_window_lefttop).clip(0, None) target_window_area = torch.where(target_window_mask, target_window_area[..., 0, :] * target_window_area[..., 1, :], 0) target_area = torch.sum(target_window_area, dim=-1) # (..., target_height, tgt_width) target_mask = target_area >= 0 # Weighted sum by area outputs = [] for x in image: assert x.shape[:mask.ndim] == mask.shape, f"Image and mask should have the same batch shape and spatial shape. Got {x.shape[:mask.ndim]} vs {mask.shape}" expand_channels = (slice(None),) * (x.ndim - mask.ndim) x = torch.where(mask[(..., *expand_channels)], x, 0) x = x.reshape(*x.shape[:mask.ndim - 2], height * width, *x.shape[mask.ndim:])[(*((slice(None),) * (mask.ndim - 2)), target_window_indices)] # (..., target_height, tgt_width, filter_size, ...) x = (x * target_window_area[(..., *expand_channels)]).sum(dim=mask.ndim) / torch.maximum(target_area[(..., *expand_channels)], torch.tensor(torch.finfo(torch.float32).eps, device=device)) # (..., target_height, tgt_width, ...) outputs.append(x) ret = *outputs, target_mask if len(ret) == 1: return ret[0] else: return ret def flood_fill(*image: Tensor, mask: Tensor, return_index: bool = False) -> Tensor: """ Flooding fill the holes in the image(s) according to the mask. ![img](doc/flood_fill.png) Parameters ---- - `*image` (Tensor): shape (..., height, width, [C]), itorchut image(s) - `mask` (Tensor): shape (..., height, width), binary mask indicating valid regions Returns ---- - `*filled_image` (Tensor): shape (..., height, width, [C]), flood filled map - `filled_indices` (Tuple[Tensor, ...], optional): tuple of shape (..., height, width). The nearest neighbor indices of each pixel in the original map. It satisfies `filled_image = image[filled_indices]` """ device = mask.device *batch_shape, height, width = mask.shape self_i, self_j = torch.meshgrid( torch.arange(height, device=device, dtype=torch.float32), torch.arange(width, device=device, dtype=torch.float32), indexing='ij' ) nearest_i, nearest_j = self_i.expand_as(mask), self_j.expand_as(mask) filled_mask = mask.clone() step = (max(height, width) + 1) // 2 steps = [] while step > 1: steps.append(step) step = (step + 1) // 2 steps += [1, 1] for step in steps: sliding_window_i = sliding_window(nearest_i, window_size=3, dilation=step, pad_size=step, dim=(-2, -1)).reshape(*batch_shape, height, width, 9) sliding_window_j = sliding_window(nearest_j, window_size=3, dilation=step, pad_size=step, dim=(-2, -1)).reshape(*batch_shape, height, width, 9) sliding_window_mask = sliding_window(filled_mask, window_size=3, dilation=step, pad_size=step, dim=(-2, -1)).reshape(*batch_shape, height, width, 9) filled_mask |= sliding_window_mask.any(dim=-1) dist = torch.where(sliding_window_mask, torch.square(sliding_window_i - self_i[..., None]) + torch.square(sliding_window_j - self_j[..., None]), torch.inf) nearest_in_window = torch.argmin(dist, dim=-1) nearest_i = torch.gather(sliding_window_i, index=nearest_in_window[..., None], dim=-1)[..., 0] nearest_j = torch.gather(sliding_window_j, index=nearest_in_window[..., None], dim=-1)[..., 0] batch_indices = [torch.arange(n, device=device).reshape([1] * i + [n] + [1] * (mask.ndim - i - 1)) for i, n in enumerate(batch_shape)] nearest_i, nearest_j = nearest_i.round().to(torch.long), nearest_j.round().to(torch.long) nearest_indices = (*batch_indices, nearest_i, nearest_j) outputs = tuple(x[nearest_indices] for x in image) if return_index: ret = *outputs, nearest_indices else: ret = outputs if len(ret) == 1: return ret[0] else: return ret def perlin_noise(x: Tensor, seed: Optional[int] = None) -> Tensor: """Generate Perlin noise for the given coordinates. Parameters ---- - `x` (Tensor): shape (*batch_shape, N_1, ..., N_D, D), coordinates to sample Perlin. If itorchut ndim is more than D + 1, the leading dimensions are treated as batch dimensions. Instances in the batch have different noise patterns. - `seed` (int, optional): random seed. The same seed will generate the same noise pattern(s). Defaults to None. Returns ---- - `y` (Tensor): shape (*batch_shape, N_1, ..., N_D), Perlin noise value at the given coordinates and seed. Value range is approximately [-1, 1] """ D = x.shape[-1] xi = x.to(int) xf = x - xi dtype, device = x.dtype, x.device # Prepare gradient set and hash permutation rng = torch.Generator(device=x.device) if seed is not None: rng.manual_seed(seed) hash_set_size = min(16 ** D, x.numel() // D) perm = torch.randperm(hash_set_size, generator=rng, device=device, dtype=torch.long) gradient_set = torch.randn((hash_set_size, D), generator=rng, device=device, dtype=dtype) gradient_set = gradient_set / (torch.linalg.norm(gradient_set, dim=-1, keepdim=True) + torch.finfo(dtype).tiny) # Get corner coordinates unit_grid = torch.stack( torch.meshgrid(*([torch.tensor([0, 1])] * D), indexing='ij'), dim=-1 ).reshape(-1, D).to(device=device) x_corners = xi[..., None, :] + unit_grid # Hash index initialization. if x.ndim > D + 1: # If batch dimensions exist, instances have different noise patterns hash_idx = torch.arange(math.prod(x.shape[:-1 - D]), device=device).reshape(*x.shape[:-1 - D], *([1] * (D + 1))) hash_idx = perm[hash_idx % hash_set_size] else: # No batch dimensions, start from 0 hash_idx = 0 # Hash corner coordinates for i in range(x.shape[-1]): hash_idx = perm[(hash_idx + x_corners[..., i]) % hash_set_size] # Compute dot products of each corner's gradient and the direction to the point gradients = gradient_set[hash_idx] direction = xf[..., None, :] - unit_grid.to(dtype) dot = torch.einsum('...i,...i->...', gradients, direction) # Smooth interpolation u = xf * xf * xf * (xf * (xf * 6 - 15) + 10) w = torch.prod(1 - torch.abs(u[..., None, :] - unit_grid.to(dtype)), dim=-1) y = torch.einsum('...i,...i->...', dot, w) return y def perlin_noise_map(size: Tuple[int, ...], frequency: Union[float, Tensor], seed: Optional[int] = None, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None) -> Tensor: """Generate Perlin noise map. Parameters ---- - `size` (Tuple[int, ...]): size of the noise map (..., H, W) - `frequency` (float | Tensor): frequency relative to map's larger dimension max(H, W) of the Perlin noise. Represents how many periods of noise fit in the larger dimension of the map. - `seed` (int, optional): random seed. The same seed will generate the same noise pattern(s). Defaults to None. Returns ---- - `noise_map` (Tensor): shape (..., H, W), Perlin noise map. Value range is approximately [-1, 1] """ height, width = size[-2:] if isinstance(frequency, torch.Tensor): dtype = frequency.dtype device = frequency.device else: if dtype is None: dtype = torch.float32 frequency = torch.tensor(frequency, dtype=dtype, device=device) uv = uv_map(size[-2:], dtype=dtype, device=device) frequency_uv = frequency[..., None] * (torch.tensor([width, height], dtype=dtype, device=device) / max(size[-2:])) x = torch.broadcast_to(uv * frequency_uv[..., None, None, :], (*size, 2)) noise_map = perlin_noise(x, seed=seed) return noise_map def fractal_perlin_noise_map( size: Tuple[int, ...], base_frequency: Union[float, Tensor], octaves: int = 4, lacunarity: float = 2.0, gain: float = 0.5, seed: Optional[int] = None, dtype: Optional[torch.dtype] = None, device: Optional[torch.device] = None, ) -> Tensor: """Generate fractal Perlin noise map. ![fractal_perlin_base_frequeny2_octaves7_gain0.7.png](doc/fractal_perlin_base_frequeny2_octaves7_gain0.7.png) Parameters ---- - `size` (Tuple[int, ...]): size of the noise map (..., H, W) - `base_frequency` (float | Tensor): base frequency (relative to map's larger dimension max(H, W)) of the Perlin noise. Represents how many periods of noise fit in the larger dimension of the map. - `octaves` (int, optional): number of octaves. Defaults to 4. - `lacunarity` (float, optional): frequency multiplier between octaves. Defaults to 2.0. - `gain` (float, optional): amplitude multiplier between octaves. Defaults to 0.5. - `seed` (int, optional): random seed. The same seed will generate the same noise pattern. Defaults to None. Returns ---- - `noise_map` (Tensor): shape (..., H, W), fractal Perlin noise map. Value range is approximately [-1, 1] """ if isinstance(base_frequency, torch.Tensor): dtype = base_frequency.dtype device = base_frequency.device else: if dtype is None: dtype = torch.float32 base_frequency = torch.tensor(base_frequency, dtype=dtype, device=device) noise_map = torch.zeros(size, dtype=dtype, device=device) amplitude = 1.0 frequency_uv = base_frequency.clone()[..., None] * (torch.tensor([size[-1], size[-2]], dtype=dtype, device=device) / max(size[-2:])) uv = uv_map(size[-2:], dtype=dtype, device=device) for octave in range(octaves): x = torch.broadcast_to(uv * frequency_uv[..., None, None, :], (*size, 2)) noise_map += amplitude * perlin_noise(x, seed=seed + octave if seed is not None else None) amplitude *= gain frequency_uv *= lacunarity return noise_map