Download code/tt_moge/reference/utils3d/numpy/rasterization.py from changh95/moge-2-p150: direct link, hf CLI and curl.
- Browser
- Download file 72.9 kB
-
https://huggingface.co/changh95/moge-2-p150/resolve/main/code/tt_moge/reference/utils3d/numpy/rasterization.py
- Command line
-
hf download hf://changh95/moge-2-p150/code/tt_moge/reference/utils3d/numpy/rasterization.py
-
curl -L -o rasterization.py https://huggingface.co/changh95/moge-2-p150/resolve/main/code/tt_moge/reference/utils3d/numpy/rasterization.py
72.9 kB
| import os | |
| import time | |
| from typing import * | |
| from contextlib import contextmanager | |
| import threading | |
| import weakref | |
| import numpy as np | |
| from numpy import ndarray | |
| import moderngl | |
| from .transforms import extrinsics_to_view, intrinsics_to_perspective | |
| __all__ = [ | |
| 'RastContext', | |
| 'rasterize_triangles', | |
| 'rasterize_triangles_peeling', | |
| 'rasterize_lines', | |
| 'rasterize_point_cloud', | |
| 'sample_texture', | |
| 'test_rasterization', | |
| ] | |
| def map_np_dtype(dtype) -> str: | |
| if dtype == int: | |
| return 'i4' | |
| elif dtype == np.uint8: | |
| return 'u1' | |
| elif dtype == np.uint32: | |
| return 'u2' | |
| elif dtype == np.float16: | |
| return 'f2' | |
| elif dtype == np.float32: | |
| return 'f4' | |
| def one_value(dtype): | |
| if dtype == 'u1': | |
| return 255 | |
| elif dtype == 'u2': | |
| return 65535 | |
| else: | |
| return 1 | |
| RASTERIZE_TRIANGLES_VS = """ | |
| #version 430 | |
| uniform mat4 uViewMat; | |
| uniform mat4 uProjectionMat; | |
| in vec3 inVert; | |
| in vec4 inAttr; | |
| {return_interpolation} in vec2 inUV; | |
| {flat} out vec4 vAttr; | |
| out vec3 vViewPos; | |
| {return_interpolation} out vec2 vUV; | |
| void main() { | |
| vec4 viewPos = uViewMat * vec4(inVert, 1.0); | |
| vAttr = inAttr; | |
| vViewPos = viewPos.xyz; | |
| gl_Position = uProjectionMat * viewPos; | |
| {return_interpolation} vUV = inUV; | |
| } | |
| """ | |
| RASTERIZE_TRIANGLES_FS = """ | |
| #version 430 | |
| {flat} in vec4 vAttr; | |
| in vec2 vUV; | |
| in vec3 vViewPos; | |
| out vec4 outAttr; | |
| out float outMask; | |
| {return_interpolation} out int outID; | |
| {return_interpolation} out vec2 outUV; | |
| void main() { | |
| float currDepth = log2(-vViewPos.z) / 64.f + 0.5f; | |
| outAttr = vAttr; | |
| outMask = 1.0; | |
| {return_interpolation} outID = gl_PrimitiveID; | |
| {return_interpolation} outUV = vUV; | |
| gl_FragDepth = currDepth; | |
| } | |
| """ | |
| RASTERIZE_TRIANGLES_PEELING_FS = """ | |
| #version 430 | |
| uniform sampler2D prevBufferDepthMap; | |
| uniform vec2 uScreenSize; | |
| {flat} in vec4 vAttr; | |
| in vec2 vUV; | |
| in vec3 vViewPos; | |
| out vec4 outAttr; | |
| out float outMask; | |
| {return_interpolation} out int outID; | |
| {return_interpolation} out vec2 outUV; | |
| void main() { | |
| float currDepth = log2(-vViewPos.z) / 64.f + 0.5f; | |
| float prevDepth = texture(prevBufferDepthMap, gl_FragCoord.xy / uScreenSize).r; | |
| if (currDepth <= prevDepth) { | |
| discard; | |
| } | |
| outAttr = vAttr; | |
| outMask = 1.0; | |
| {return_interpolation} outID = gl_PrimitiveID; | |
| {return_interpolation} outUV = vUV; | |
| gl_FragDepth = currDepth; | |
| } | |
| """ | |
| RASTERIZE_POINT_CLOUD_VS = """ | |
| #version 430 | |
| uniform mat4 uViewMat; | |
| uniform mat4 uProjectionMat; | |
| in vec3 inVert; | |
| in vec4 inAttr; | |
| in float inPointSize; | |
| {return_point_id} out int vPointID; | |
| out vec3 vViewPos; | |
| out vec4 vAttr; | |
| out float vPointSize; | |
| void main() { | |
| vec4 viewPos = uViewMat * vec4(inVert, 1.0); | |
| vAttr = inAttr; | |
| vViewPos = viewPos.xyz; | |
| vPointSize = inPointSize; | |
| {return_point_id} vPointID = gl_VertexID; | |
| gl_Position = uProjectionMat * viewPos; | |
| } | |
| """ | |
| RASTERIZE_POINT_CLOUD_GS = """ | |
| #version 430 core | |
| layout(points) in; | |
| layout(triangle_strip, max_vertices = {num_point_shape_vertices}) out; | |
| in vec3 vViewPos[]; | |
| in vec4 vAttr[]; | |
| in float vPointSize[]; | |
| {return_point_id} in int vPointID[]; | |
| flat out vec4 gAttr; | |
| out vec3 gViewPos; | |
| flat out vec3 gPointViewPos; | |
| flat out vec2 gPointCenter2D; | |
| flat out float gPointSize; | |
| {return_point_id} flat out int gPointID; | |
| uniform mat4 uViewMat; | |
| uniform mat4 uProjectionMat; | |
| uniform vec2 uScreenSize; | |
| uniform bool uIsPointSize3D; | |
| const vec3 triangle[3] = {vec3(0., 0.5, 0.0), vec3(-0.433013, -0.25, 0.0), vec3(0.433013, -0.25, 0.0)}; | |
| const vec3 square[4] = {vec3(-0.5, -0.5, 0.0), vec3(0.5, -0.5, 0.0), vec3(-0.5, 0.5, 0.0), vec3( 0.5, 0.5, 0.0)}; | |
| const vec3 pentagon[5] = {vec3(0.0, 0.5, 0.0), vec3(-0.475528, 0.154508, 0.0), vec3(0.475528, 0.154508, 0.0), vec3(-0.293893, -0.404508, 0.0), vec3(0.293893, -0.404508, 0.0)}; | |
| const vec3 hexagon[6] = {vec3(0.0, 0.5, 0.0), vec3(-0.433013, 0.25, 0.0), vec3(0.433013, 0.25, 0.0), vec3(-0.433013, -0.25, 0.0), vec3(0.433013, -0.25, 0.0), vec3(0.0, -0.5, 0.0)}; | |
| const vec3 circle[4] = {vec3(-0.5, -0.5, 0.0), vec3(0.5, -0.5, 0.0), vec3(-0.5, 0.5, 0.0), vec3( 0.5, 0.5, 0.0)}; | |
| void main() { | |
| mat4 invProjection = inverse(uProjectionMat); | |
| gAttr = vAttr[0]; | |
| {return_point_id} gPointID = vPointID[0]; | |
| gPointViewPos = vViewPos[0]; | |
| gPointCenter2D = (0.5 * gl_in[0].gl_Position.xy / gl_in[0].gl_Position.w + 0.5) * uScreenSize; | |
| gPointSize = vPointSize[0]; | |
| for (int i = 0; i < {point_shape}.length(); ++i) { | |
| vec3 viewPos = uIsPointSize3D ? gPointViewPos + gPointSize * {point_shape}[i] : gPointViewPos; | |
| vec4 clipCoord = uProjectionMat * vec4(viewPos, 1.0); | |
| if (!uIsPointSize3D) clipCoord.xy += 2 * gPointSize * clipCoord.w / uScreenSize * {point_shape}[i].xy; | |
| gViewPos = (invProjection * clipCoord).xyz; | |
| gl_Position = clipCoord; | |
| EmitVertex(); | |
| } | |
| EndPrimitive(); | |
| } | |
| """ | |
| RASTERIZE_POINT_CLOUD_FS = """ | |
| #version 430 core | |
| flat in vec4 gAttr; | |
| in vec3 gViewPos; | |
| flat in vec3 gPointViewPos; | |
| flat in vec2 gPointCenter2D; | |
| flat in float gPointSize; | |
| {return_point_id} flat in int gPointID; | |
| out vec4 outAttr; | |
| out float outMask; | |
| {return_point_id} out int outPointID; | |
| uniform bool uIsPointSize3D; | |
| void main() { | |
| outAttr = gAttr; | |
| outMask = 1.0; | |
| {return_point_id} outPointID = gPointID; | |
| gl_FragDepth = log2(-gViewPos.z) / 64.f + 0.5f; | |
| {circle_begin} | |
| float distSquare; | |
| if (uIsPointSize3D) { | |
| vec3 x = gViewPos - gPointViewPos; | |
| distSquare = dot(x, x); | |
| } | |
| else { | |
| vec2 x = gl_FragCoord.xy - gPointCenter2D; | |
| distSquare = dot(x, x); | |
| } | |
| if (distSquare > 0.25 * gPointSize * gPointSize) | |
| discard; | |
| {circle_end} | |
| } | |
| """ | |
| FULL_SCREEN_VS = """ | |
| #version 430 core | |
| out vec2 screenXY; | |
| void main() { | |
| screenXY = vec2((gl_VertexID << 1) & 2, gl_VertexID & 2); | |
| gl_Position = vec4(screenXY * 2.0 - 1.0, 0.0, 1.0); | |
| } | |
| """ | |
| CLEAR_TEXTURE_FS = """ | |
| #version 430 | |
| uniform {type} uCleanColor; | |
| in vec2 screenXY; | |
| out {type} outColor; | |
| void main() { | |
| outColor = uCleanColor; | |
| } | |
| """ | |
| SAMPLE_TEXTURE_FS = """ | |
| #version 430 | |
| uniform sampler2D texMap; | |
| uniform sampler2D uvMap; | |
| in vec2 screenXY; | |
| out vec4 outColor; | |
| void main() { | |
| outColor = vec4(texture(texMap, texture(uvMap, screenXY).xy)); | |
| } | |
| """ | |
| DEPTH_LINEAR_TO_BUFFER_FS = """ | |
| #version 430 | |
| uniform sampler2D linearDepthMap; | |
| in vec2 screenXY; | |
| void main() { | |
| float d = texture(linearDepthMap, screenXY).r; | |
| gl_FragDepth = min(1.f, log2(d) / 64.f + 0.5f); | |
| } | |
| """ | |
| DEPTH_BUFFER_TO_LINEAR_FS = """ | |
| #version 430 | |
| uniform sampler2D bufferDepthMap; | |
| in vec2 screenXY; | |
| out float outLinearDepth; | |
| void main() { | |
| float d = texture(bufferDepthMap, screenXY).r; | |
| outLinearDepth = d == 1.f ? 1.f / 0.f : exp2((d - 0.5f) * 64.f); | |
| } | |
| """ | |
| class RastContext: | |
| """ Context for numpy-side rasterization. Based on moderngl. | |
| """ | |
| _threading_local: threading.local = threading.local() | |
| "Thread-local static variable to store default context." | |
| programs: Dict[str, moderngl.Program] | |
| "Cache of compiled moderngl programs." | |
| def __init__(self, *args, **kwargs): | |
| """Create context for numpy-side rasterization. Based on moderngl. | |
| ## Parameters | |
| Leave empty to create context with default settings. Or refer to moderngl.create_context() for other parameters. | |
| """ | |
| if len(args) == 0 and len(kwargs) == 0: | |
| try: | |
| # Default | |
| self.mgl_ctx = moderngl.create_context() | |
| except: | |
| pass | |
| if not hasattr(self, 'mgl_ctx'): | |
| try: | |
| # EGL | |
| self.mgl_ctx = moderngl.create_context(standalone=True, backend='egl',) | |
| except: | |
| pass | |
| if not hasattr(self, 'mgl_ctx'): | |
| try: | |
| # Default standalone | |
| self.mgl_ctx = moderngl.create_context(standalone=True) | |
| except: | |
| pass | |
| if not hasattr(self, 'mgl_ctx'): | |
| try: | |
| # EGL with more specifications | |
| self.mgl_ctx = moderngl.create_context(standalone=True, backend='egl', libgl='libGL.so.1', libegl='libEGL.so.1') | |
| except: | |
| pass | |
| if not hasattr(self, 'mgl_ctx'): | |
| raise RuntimeError( | |
| "Failed to create moderngl context with default settings. " | |
| "Please specify context creation settings following https://moderngl.readthedocs.io/en/latest/topics/context.html. " | |
| "If running on headless ubuntu server, please refer to https://moderngl.readthedocs.io/en/latest/techniques/headless_ubuntu_18_server.html" | |
| ) | |
| elif len(args) == 1 and isinstance(args[0], moderngl.Context): | |
| self.mgl_ctx = args[0] | |
| else: | |
| self.mgl_ctx = moderngl.create_context(*args, **kwargs) | |
| self.programs = {} | |
| def __enter__(self): | |
| self.mgl_ctx.__enter__() | |
| return self | |
| def __exit__(self, exc_type, exc_value, traceback): | |
| self.mgl_ctx.__exit__(exc_type, exc_value, traceback) | |
| def __del__(self): | |
| try: | |
| self.mgl_ctx.release() | |
| for prog_name, prog in self.programs.items(): | |
| prog.release() | |
| except: | |
| pass | |
| def get_default_context(): | |
| """ Get the default RastContext in current thread. Create one if not exists. | |
| """ | |
| if not hasattr(RastContext._threading_local, 'default_context'): | |
| RastContext._threading_local.default_context = RastContext() | |
| return RastContext._threading_local.default_context | |
| def run_full_screen_program( | |
| ctx: RastContext, | |
| prog: moderngl.Program, | |
| in_tex: Union[moderngl.Texture, List[moderngl.Texture]], | |
| out_tex_or_rbo: Union[Union[moderngl.Texture, moderngl.Renderbuffer], Sequence[Union[moderngl.Texture, moderngl.Renderbuffer]]] = None, | |
| out_depth: Optional[moderngl.Texture] = None, | |
| **uniforms | |
| ) -> moderngl.Texture: | |
| vao = ctx.mgl_ctx.vertex_array(prog, []) | |
| if isinstance(in_tex, list): | |
| for i, tex in enumerate(in_tex): | |
| tex.use(location=i) | |
| else: | |
| in_tex.use(location=0) | |
| for k, v in uniforms.items(): | |
| prog[k] = v | |
| if not isinstance(out_tex_or_rbo, list): | |
| out_tex_or_rbo = [out_tex_or_rbo] if out_tex_or_rbo is not None else [] | |
| fbo = ctx.mgl_ctx.framebuffer(color_attachments=out_tex_or_rbo, depth_attachment=out_depth) | |
| fbo.use() | |
| ctx.mgl_ctx.disable(ctx.mgl_ctx.DEPTH_TEST) | |
| ctx.mgl_ctx.disable(ctx.mgl_ctx.CULL_FACE) | |
| ctx.mgl_ctx.disable(ctx.mgl_ctx.BLEND) | |
| vao.render(moderngl.TRIANGLES, vertices=3) | |
| vao.release() | |
| fbo.release() | |
| def clear_texture(ctx: RastContext, tex: moderngl.Texture, value: Union[Tuple[float], Tuple[int], float, int]): | |
| if tex.depth: | |
| fbo = ctx.mgl_ctx.framebuffer(color_attachments=[], depth_attachment=tex) | |
| fbo.clear(depth=value) | |
| fbo.release() | |
| elif tex.dtype.startswith('f'): | |
| fbo = ctx.mgl_ctx.framebuffer(color_attachments=[tex]) | |
| fbo.clear(color=value if isinstance(value, tuple) else (value,) * tex.components) | |
| fbo.release() | |
| elif tex.dtype.startswith('i') or tex.dtype.startswith('u'): | |
| fbo = ctx.mgl_ctx.framebuffer(color_attachments=[tex]) | |
| vec_type = tex.dtype[0] + 'vec4' | |
| program_name = f'clear_texture_{tex.dtype}' | |
| if prog := ctx.programs.get(program_name, None) is None: | |
| prog = ctx.mgl_ctx.program( | |
| vertex_shader=FULL_SCREEN_VS, | |
| fragment_shader=CLEAR_TEXTURE_FS.replace('{type}', vec_type) | |
| ) | |
| ctx.programs[program_name] = prog | |
| vao = ctx.mgl_ctx.vertex_array(prog, []) | |
| prog['uCleanColor'].value = value + (0,) * (4 - len(value)) | |
| fbo.use() | |
| vao.render(moderngl.TRIANGLES, vertices=3) | |
| fbo.release() | |
| vao.release() | |
| def depth_texture_buffer_to_linear(ctx: RastContext, buffer_depth_tex: moderngl.Texture, linear_depth_tex: moderngl.Texture): | |
| program_name = f'depth_buffer_to_linear' | |
| if (prog := ctx.programs.get(program_name, None)) is None: | |
| prog = ctx.mgl_ctx.program( | |
| vertex_shader=FULL_SCREEN_VS, | |
| fragment_shader=DEPTH_BUFFER_TO_LINEAR_FS | |
| ) | |
| prog['bufferDepthMap'] = 0 | |
| ctx.programs[program_name] = prog | |
| run_full_screen_program(ctx, prog, buffer_depth_tex, out_tex_or_rbo=linear_depth_tex) | |
| def depth_texture_linear_to_buffer(ctx: RastContext, linear_depth_tex: moderngl.Texture, buffer_depth_tex: moderngl.Texture): | |
| program_name = f'depth_linear_to_buffer' | |
| if (prog := ctx.programs.get(program_name, None)) is None: | |
| prog = ctx.mgl_ctx.program( | |
| vertex_shader=FULL_SCREEN_VS, | |
| fragment_shader=DEPTH_LINEAR_TO_BUFFER_FS | |
| ) | |
| prog['linearDepthMap'] = 0 | |
| ctx.programs[program_name] = prog | |
| run_full_screen_program(ctx, prog, linear_depth_tex, out_depth=buffer_depth_tex) | |
| def rasterize_triangles( | |
| size: Tuple[int, int], | |
| *, | |
| vertices: ndarray, | |
| attributes: Optional[ndarray] = None, | |
| attributes_domain: Optional[Literal['vertex', 'face']] = 'vertex', | |
| faces: Optional[ndarray] = None, | |
| view: ndarray = None, | |
| projection: ndarray = None, | |
| extrinsics: ndarray = None, | |
| intrinsics: ndarray = None, | |
| near: float = 0.01, | |
| far: float = float('inf'), | |
| cull_backface: bool = False, | |
| return_depth: bool = False, | |
| return_interpolation: bool = False, | |
| background: Optional[Dict[str, ndarray]] = None, | |
| ctx: Optional[RastContext] = None, | |
| ) -> Dict[str, ndarray]: | |
| """ | |
| Rasterize triangles. | |
| Parameters | |
| ---- | |
| - `size` (Tuple[int, int]): (height, width) of the output image | |
| - `vertices` (ndarray): (N, 3) or (T, 3, 3) | |
| - `faces` (Optional[ndarray]): (T, 3) or None. If `None`, the vertices must be an array with shape (T, 3, 3) | |
| - `attributes` (ndarray): (N, C), (T, 3, C) for vertex domain or (T, C) for face domain | |
| - `attributes_domain` (Literal['vertex', 'face']): domain of the attributes | |
| - `view` | `extrinsics` (ndarray): (4, 4) View matrix or extrinsics matrix. Provide either one of them. | |
| - `projection` | `intrinsics` (ndarray): (4, 4) Projection matrix or (3, 3) Intrinsics matrix. Provide either one of them. | |
| - `cull_backface` (bool): whether to cull backface | |
| - `background` (Optional[Dict[str, ndarray]]): background to composite with. Contains the same keys as the return dictionary, each with shape matching the output. | |
| - `ctx` (RastContext): rasterization context. Created by `RastContext()`. Default to the thread-local default context. | |
| Returns | |
| ---- | |
| A dictionary containing | |
| - `mask` (ndarray): (H, W) bool mask of valid pixels | |
| if attributes is not None | |
| - `image` (ndarray): (H, W, C) float32 rendered image corresponding to the input attributes | |
| if return_depth is True | |
| - `depth` (ndarray): (H, W) float32 camera space linear depth. Empty pixels are filled with inf. | |
| if return_interpolation is True | |
| - `interpolation_id` (ndarray): (H, W) int32 triangle ID map | |
| - `interpolation_uv` (ndarray): (H, W, 2) float32 triangle UV (first two channels of barycentric coordinates) | |
| """ | |
| if ctx is None: | |
| ctx = RastContext.get_default_context() | |
| height, width = size | |
| if faces is None: | |
| assert vertices.ndim == 3 and vertices.shape[1] == vertices.shape[2] == 3, "If faces is None, vertices must be an array with shape (T, 3, 3)" | |
| else: | |
| assert faces.ndim == 2 and faces.shape[1] == 3, f"Faces should be a 2D array with shape (T, 3), but got {faces.shape}" | |
| assert vertices.ndim == 2 and vertices.shape[1] == 3 | |
| assert vertices.dtype == np.float32 | |
| if attributes is not None: | |
| assert attributes.dtype == np.float32 | |
| if attributes_domain == 'vertex': | |
| assert (attributes.shape[:-1] == vertices.shape[:-1]), f"Attribute shape {attributes.shape} does not match vertex shape {vertices.shape}" | |
| elif attributes_domain == 'face': | |
| if faces is None: | |
| assert attributes.shape[0] == vertices.shape[0], f"Attribute shape {attributes.shape} does not match vertex shape {vertices.shape}" | |
| else: | |
| assert attributes.shape[0] == faces.shape[0], f"Attribute shape {attributes.shape} does not match face shape {faces.shape}" | |
| else: | |
| raise ValueError(f"Unknown attributes domain: {attributes_domain}") | |
| assert attributes.shape[-1] in [1, 2, 3, 4], f'Vertex attribute only supports channels 1, 2, 3, 4, but got {attributes.shape[-1]}' | |
| assert intrinsics is None or (intrinsics.shape == (3, 3) and intrinsics.dtype == np.float32), f"Intrinsics should be a 3x3 float32 matrix, but got {intrinsics.shape} {intrinsics.dtype}" | |
| assert extrinsics is None or (extrinsics.shape == (4, 4) and extrinsics.dtype == np.float32), f"Extrinsics should be a 4x4 float32 matrix, but got {extrinsics.shape} {extrinsics.dtype}" | |
| assert view is None or (view.shape == (4, 4) and view.dtype == np.float32), f"View should be a 4x4 float32 matrix, but got {view.shape} {view.dtype}" | |
| assert projection is None or (projection.shape == (4, 4) and projection.dtype == np.float32), f"Projection should be a 4x4 float32 matrix, but got {projection.shape} {projection.dtype}" | |
| background_image, background_mask, background_depth, background_interpolation_id, background_interpolation_uv = [background.get(k, None) if background is not None else None for k in ['image', 'mask', 'depth', 'interpolation_id', 'interpolation_uv']] | |
| if background_image is not None: | |
| assert background_image.ndim == 3 and background_image.shape == (height, width, attributes.shape[-1]), f"Image should be a float32 array with shape (H, W, {attributes.shape[1]}), but got {background_image.shape} {background_image.dtype}" | |
| if background_depth is not None: | |
| assert background_depth.dtype == np.float32 and background_depth.ndim == 2 and background_depth.shape == (height, width), f"Depth should be a float32 array with shape (H, W), but got {background_depth.shape} {background_depth.dtype}" | |
| if background_interpolation_id is not None: | |
| assert background_interpolation_id.dtype == np.int32 and background_interpolation_id.ndim == 2 and background_interpolation_id.shape == (height, width), f"Interpolation ID should be a int32 array with shape (H, W), but got {background_interpolation_id.shape} {background_interpolation_id.dtype}" | |
| if background_interpolation_uv is not None: | |
| assert background_interpolation_uv.dtype == np.float32 and background_interpolation_uv.ndim == 3 and background_interpolation_uv.shape == (height, width, 2), f"Interpolation UV should be a float32 array with shape (H, W, 2), but got {background_interpolation_uv.shape} {background_interpolation_uv.dtype}" | |
| if faces is not None: | |
| vertices = vertices[faces] | |
| if attributes is not None: | |
| num_channels = attributes.shape[-1] | |
| attributes = np.concatenate([attributes, np.zeros((*attributes.shape[:-1], 4 - num_channels,), dtype=attributes.dtype)], axis=-1) if num_channels < 4 else attributes | |
| if attributes_domain == 'vertex': | |
| if faces is not None: | |
| attributes = attributes[faces] | |
| elif attributes_domain == 'face': | |
| attributes = attributes[:, None, :].repeat(3, axis=1) | |
| if intrinsics is not None: | |
| assert projection is None, "Cannot specify both intrinsics and projection." | |
| projection = intrinsics_to_perspective(intrinsics, near, far) | |
| if extrinsics is not None: | |
| assert view is None, "Cannot specify both extrinsics and view." | |
| view = extrinsics_to_view(extrinsics) | |
| if view is None: | |
| view = np.eye(4, dtype=np.float32) | |
| if projection is None: | |
| projection = np.eye(4, dtype=np.float32) | |
| if background_image is not None: | |
| background_image = np.concatenate([background_image, np.ones((height, width, 4 - background_image.shape[-1]), dtype=background_image.dtype)], axis=-1) if background_image.shape[-1] < 4 else background_image | |
| # Get program | |
| flat = attributes_domain == 'face' | |
| program_name = f'rasterize_triangles(flat={flat}, return_interpolation={return_interpolation})' | |
| if (prog := ctx.programs.get(program_name, None)) is None: | |
| prog = ctx.mgl_ctx.program( | |
| vertex_shader=RASTERIZE_TRIANGLES_VS.replace('{flat}', 'flat' if flat else '').replace('{return_interpolation}', '' if return_interpolation else '//'), | |
| fragment_shader=RASTERIZE_TRIANGLES_FS.replace('{flat}', 'flat' if flat else '').replace('{return_interpolation}', '' if return_interpolation else '//') | |
| ) | |
| ctx.programs[program_name] = prog | |
| # Create buffers | |
| vbo_vertices = ctx.mgl_ctx.buffer(np.ascontiguousarray(vertices, dtype='f4')) | |
| if attributes is not None: | |
| vbo_attributes = ctx.mgl_ctx.buffer(np.ascontiguousarray(attributes, dtype='f4')) | |
| if return_interpolation: | |
| vbo_uv = ctx.mgl_ctx.buffer(np.ascontiguousarray((np.array([[1., 0.], [0., 1.], [0., 0.]], dtype=np.float32).reshape(1, 3, 2).repeat(vertices.shape[0], axis=0)))) | |
| vao = ctx.mgl_ctx.vertex_array( | |
| prog, | |
| list(filter(lambda x: x is not None, [ | |
| (vbo_vertices, '3f', 'inVert'), | |
| (vbo_attributes, f'4f', 'inAttr') if attributes is not None else None, | |
| (vbo_uv, '2f', 'inUV') if return_interpolation else None, | |
| ])), | |
| mode=moderngl.TRIANGLES, | |
| ) | |
| # Create textures | |
| image_tex = ctx.mgl_ctx.texture((width, height), 4, dtype='f4', data=np.ascontiguousarray(background_image[::-1, :, :]) if background_image is not None else None) | |
| buffer_depth_tex = ctx.mgl_ctx.depth_texture((width, height)) | |
| mask_tex = ctx.mgl_ctx.texture((width, height), 1, dtype='f1', data=np.ascontiguousarray(background_mask[::-1, :]) if background_mask is not None else None) | |
| if return_depth: | |
| linear_depth_tex = ctx.mgl_ctx.texture((width, height), 1, dtype='f4', data=np.ascontiguousarray(background_depth[::-1, :]) if background_depth is not None else None) | |
| if return_interpolation: | |
| interpolation_id_tex = ctx.mgl_ctx.texture((width, height), 1, dtype='i4', data=np.ascontiguousarray(background_interpolation_id[::-1, :]) if background_interpolation_id is not None else None) | |
| interpolation_uv_tex = ctx.mgl_ctx.texture((width, height), 2, dtype='f4', data=np.ascontiguousarray(background_interpolation_uv[::-1, :, :]) if background_interpolation_uv is not None else None) | |
| # Clear textures | |
| if background_image is None: | |
| clear_texture(ctx, image_tex, value=(0.0, 0.0, 0.0, 1.0)) | |
| if background_mask is None: | |
| clear_texture(ctx, mask_tex, value=0.0) | |
| if background_depth is not None: | |
| depth_texture_linear_to_buffer(ctx, linear_depth_tex, buffer_depth_tex) | |
| else: | |
| clear_texture(ctx, buffer_depth_tex, value=1.0) | |
| if return_interpolation: | |
| if background_interpolation_id is None: | |
| clear_texture(ctx, interpolation_id_tex, value=(-1,)) | |
| if background_interpolation_uv is None: | |
| clear_texture(ctx, interpolation_uv_tex, value=(0.0, 0.0)) | |
| # Create framebuffer | |
| if return_interpolation: | |
| fbo = ctx.mgl_ctx.framebuffer( | |
| color_attachments=[image_tex, mask_tex, interpolation_id_tex, interpolation_uv_tex], | |
| depth_attachment=buffer_depth_tex, | |
| ) | |
| else: | |
| fbo = ctx.mgl_ctx.framebuffer( | |
| color_attachments=[image_tex, mask_tex], | |
| depth_attachment=buffer_depth_tex, | |
| ) | |
| fbo.viewport = (0, 0, width, height) | |
| # Set uniforms | |
| prog['uViewMat'].write(np.ascontiguousarray(view.transpose().astype('f4'))) | |
| prog['uProjectionMat'].write(np.ascontiguousarray(projection.transpose().astype('f4'))) | |
| # Set render states | |
| ctx.mgl_ctx.depth_func = '<' | |
| ctx.mgl_ctx.enable(ctx.mgl_ctx.DEPTH_TEST) | |
| if cull_backface: | |
| ctx.mgl_ctx.enable(ctx.mgl_ctx.CULL_FACE) | |
| else: | |
| ctx.mgl_ctx.disable(ctx.mgl_ctx.CULL_FACE) | |
| ctx.mgl_ctx.disable(ctx.mgl_ctx.BLEND) | |
| # Render | |
| fbo.use() | |
| vao.render() | |
| # Read | |
| mask = np.frombuffer(mask_tex.read(), dtype=bool).reshape((height, width)) | |
| mask = np.flip(mask, axis=0) | |
| if attributes is not None: | |
| image = np.frombuffer(image_tex.read(), dtype='f4').reshape((height, width, 4)) | |
| image = np.flip(image, axis=0) | |
| image = image[:, :, :num_channels] | |
| else: | |
| image = None | |
| if return_depth: | |
| depth_texture_buffer_to_linear(ctx, buffer_depth_tex, linear_depth_tex) | |
| depth = np.frombuffer(linear_depth_tex.read(), dtype='f4').reshape((height, width)) | |
| depth = np.flip(depth, axis=0) | |
| else: | |
| depth = None | |
| if return_interpolation: | |
| interpolation_id = np.frombuffer(interpolation_id_tex.read(), dtype='i4').reshape((height, width)) | |
| interpolation_id = np.flip(interpolation_id, axis=0) | |
| interpolation_uv = np.frombuffer(interpolation_uv_tex.read(), dtype='f4').reshape((height, width, 2)) | |
| interpolation_uv = np.flip(interpolation_uv, axis=0) | |
| else: | |
| interpolation_id = None | |
| interpolation_uv = None | |
| # Release | |
| vao.release() | |
| vbo_vertices.release() | |
| if attributes is not None: | |
| vbo_attributes.release() | |
| if return_interpolation: | |
| vbo_uv.release() | |
| fbo.release() | |
| image_tex.release() | |
| mask_tex.release() | |
| buffer_depth_tex.release() | |
| if background_depth is not None or return_depth: | |
| linear_depth_tex.release() | |
| if return_interpolation: | |
| interpolation_id_tex.release() | |
| interpolation_uv_tex.release() | |
| output = { | |
| "image": image, | |
| "mask": mask, | |
| "depth": depth, | |
| "interpolation_id": interpolation_id, | |
| "interpolation_uv": interpolation_uv | |
| } | |
| output = {k: v for k, v in output.items() if v is not None} | |
| return output | |
| def rasterize_triangles_peeling( | |
| size: Tuple[int, int], | |
| *, | |
| vertices: ndarray, | |
| attributes: ndarray, | |
| attributes_domain: Literal['vertex', 'face'] = 'vertex', | |
| faces: Optional[ndarray] = None, | |
| view: ndarray = None, | |
| projection: ndarray = None, | |
| extrinsics: ndarray = None, | |
| intrinsics: ndarray = None, | |
| near: float = 0.01, | |
| far: float = float('inf'), | |
| cull_backface: bool = False, | |
| return_depth: bool = False, | |
| return_interpolation: bool = False, | |
| ctx: Optional[RastContext] = None, | |
| ) -> Iterator[Iterator[Dict[str, ndarray]]]: | |
| """ | |
| Rasterize triangles with depth peeling. | |
| Parameters | |
| ---- | |
| - `size` (Tuple[int, int]): (height, width) of the output image | |
| - `vertices` (ndarray): (N, 3) or (T, 3, 3) | |
| - `faces` (Optional[ndarray]): (T, 3) or None. If `None`, the vertices must be an array with shape (T, 3, 3) | |
| - `attributes` (ndarray): (N, C), (T, 3, C) for vertex domain or (T, C) for face domain | |
| - `attributes_domain` (Literal['vertex', 'face']): domain of the attributes | |
| - `view` | `extrinsics` (ndarray): (4, 4) View matrix or extrinsics matrix. Provide either one of them. | |
| - `projection` | `intrinsics` (ndarray): (4, 4) Projection matrix or (3, 3) Intrinsics matrix. Provide either one of them. | |
| - `near` (float): near clipping plane. Only used for intrinsics. Ignored if projection matrix is provided. | |
| - `far` (float): far clipping plane. Only used for intrinsics. Ignored if projection matrix is provided. | |
| - `cull_backface` (bool): whether to cull backfaces | |
| - `ctx` (RastContext): rasterization context. Created by `RastContext()`. Default to the thread-local default context. | |
| Returns | |
| ---- | |
| A generator that yields dictionaries for each peeling layer containing: | |
| - `mask` (ndarray): (H, W) bool mask of valid pixels in this layer | |
| if attributes is not None | |
| - `image` (ndarray): (H, W, C) float32 rendered image corresponding to the input attributes | |
| if return_depth is True | |
| - `depth` (ndarray): (H, W) float32 camera space linear depth. Empty pixels are filled with inf. | |
| if return_interpolation is True | |
| - `interpolation_id` (ndarray): (H, W) int32 triangle ID map | |
| - `interpolation_uv` (ndarray): (H, W, 2) float32 triangle UV (first two channels of barycentric coordinates) | |
| The last layer yielded will be empty (no pixels covered), then the generator stops. | |
| Example | |
| ---- | |
| ``` | |
| for i, layer_output in enumerate(rasterize_triangles_peeling( | |
| (512, 512), | |
| vertices=vertices, | |
| faces=faces, | |
| attributes=attributes, | |
| view=view, | |
| projection=projection | |
| )): | |
| print(f"Layer {i}:") | |
| for key, value in layer_output.items(): | |
| print(f" {key}: {value.shape}") | |
| if i >= 4: # Stop after 5 layers at most | |
| break | |
| ``` | |
| """ | |
| if ctx is None: | |
| ctx = RastContext.get_default_context() | |
| height, width = size | |
| if faces is None: | |
| assert vertices.ndim == 3 and vertices.shape[1] == vertices.shape[2] == 3, "If faces is None, vertices must be an array with shape (T, 3, 3)" | |
| else: | |
| assert faces.ndim == 2 and faces.shape[1] == 3, f"Faces should be a 2D array with shape (T, 3), but got {faces.shape}" | |
| assert vertices.ndim == 2 and vertices.shape[1] == 3 | |
| assert vertices.dtype == np.float32 | |
| if attributes is not None: | |
| assert attributes.dtype == np.float32 | |
| if attributes_domain == 'vertex': | |
| assert (attributes.shape[:-1] == vertices.shape[:-1]), f"Attribute shape {attributes.shape} does not match vertex shape {vertices.shape}" | |
| elif attributes_domain == 'face': | |
| if faces is None: | |
| assert attributes.shape[0] == vertices.shape[0], f"Attribute shape {attributes.shape} does not match vertex shape {vertices.shape}" | |
| else: | |
| assert attributes.shape[0] == faces.shape[0], f"Attribute shape {attributes.shape} does not match face shape {faces.shape}" | |
| else: | |
| raise ValueError(f"Unknown attributes domain: {attributes_domain}") | |
| assert attributes.shape[-1] in [1, 2, 3, 4], f'Vertex attribute only supports channels 1, 2, 3, 4, but got {attributes.shape[-1]}' | |
| assert intrinsics is None or (intrinsics.shape == (3, 3) and intrinsics.dtype == np.float32), f"Intrinsics should be a 3x3 float32 matrix, but got {intrinsics.shape} {intrinsics.dtype}" | |
| assert extrinsics is None or (extrinsics.shape == (4, 4) and extrinsics.dtype == np.float32), f"Extrinsics should be a 4x4 float32 matrix, but got {extrinsics.shape} {extrinsics.dtype}" | |
| assert view is None or (view.shape == (4, 4) and view.dtype == np.float32), f"View should be a 4x4 float32 matrix, but got {view.shape} {view.dtype}" | |
| assert projection is None or (projection.shape == (4, 4) and projection.dtype == np.float32), f"Projection should be a 4x4 float32 matrix, but got {projection.shape} {projection.dtype}" | |
| if faces is not None: | |
| vertices = vertices[faces] | |
| if attributes is not None: | |
| num_channels = attributes.shape[-1] | |
| attributes = np.concatenate([attributes, np.zeros((*attributes.shape[:-1], 4 - num_channels,), dtype=attributes.dtype)], axis=-1) if num_channels < 4 else attributes | |
| if attributes_domain == 'vertex': | |
| if faces is not None: | |
| attributes = attributes[faces] | |
| elif attributes_domain == 'face': | |
| attributes = attributes[:, None, :].repeat(3, axis=1) | |
| if extrinsics is not None: | |
| assert view is None, "Cannot specify both extrinsics and view." | |
| view = extrinsics_to_view(extrinsics) | |
| if intrinsics is not None: | |
| assert projection is None, "Cannot specify both intrinsics and projection." | |
| projection = intrinsics_to_perspective(intrinsics, near, far) | |
| if view is None: | |
| view = np.eye(4, dtype=np.float32) | |
| if projection is None: | |
| projection = np.eye(4, dtype=np.float32) | |
| # Get program | |
| flat = attributes_domain == 'face' | |
| program_name = f'rasterize_triangles_peeling(flat={flat}, return_interpolation={return_interpolation})' | |
| if (prog := ctx.programs.get(program_name, None)) is None: | |
| prog = ctx.mgl_ctx.program( | |
| vertex_shader=RASTERIZE_TRIANGLES_VS.replace('{flat}', 'flat' if flat else '').replace('{return_interpolation}', '' if return_interpolation else '//'), | |
| fragment_shader=RASTERIZE_TRIANGLES_PEELING_FS.replace('{flat}', 'flat' if flat else '').replace('{return_interpolation}', '' if return_interpolation else '//') | |
| ) | |
| prog['prevBufferDepthMap'] = 0 | |
| ctx.programs[program_name] = prog | |
| # Create buffers | |
| vbo_vertices = ctx.mgl_ctx.buffer(np.ascontiguousarray(vertices, dtype='f4')) | |
| if attributes is not None: | |
| vbo_attributes = ctx.mgl_ctx.buffer(np.ascontiguousarray(attributes, dtype='f4')) | |
| if return_interpolation: | |
| vbo_uv = ctx.mgl_ctx.buffer(np.ascontiguousarray((np.array([[1., 0.], [0., 1.], [0., 0.]], dtype=np.float32).reshape(1, 3, 2).repeat(vertices.shape[0], axis=0)))) | |
| vao = ctx.mgl_ctx.vertex_array( | |
| prog, | |
| list(filter(lambda x: x is not None, [ | |
| (vbo_vertices, '3f', 'inVert'), | |
| (vbo_attributes, f'4f', 'inAttr') if attributes is not None else None, | |
| (vbo_uv, '2f', 'inUV') if return_interpolation else None, | |
| ])), | |
| mode=moderngl.TRIANGLES, | |
| ) | |
| # Create textures | |
| image_tex = ctx.mgl_ctx.texture((width, height), 4, dtype='f4') | |
| mask_tex = ctx.mgl_ctx.texture((width, height), 1, dtype='f1') | |
| buffer_depth_tex_a = ctx.mgl_ctx.depth_texture((width, height)) | |
| buffer_depth_tex_b = ctx.mgl_ctx.depth_texture((width, height)) | |
| if return_depth: | |
| linear_depth_tex = ctx.mgl_ctx.texture((width, height), 1, dtype='f4') | |
| if return_interpolation: | |
| interpolation_id_tex = ctx.mgl_ctx.texture((width, height), 1, dtype='i4') | |
| interpolation_uv_tex = ctx.mgl_ctx.texture((width, height), 2, dtype='f4') | |
| # Create frame buffers | |
| if return_interpolation: | |
| color_attachments=[image_tex, interpolation_id_tex, interpolation_uv_tex] | |
| else: | |
| color_attachments=[image_tex] | |
| fbo_curr = ctx.mgl_ctx.framebuffer( | |
| color_attachments=color_attachments, | |
| depth_attachment=buffer_depth_tex_a, | |
| ) | |
| fbo_curr.viewport = (0, 0, width, height) | |
| fbo_prev = ctx.mgl_ctx.framebuffer( | |
| color_attachments=color_attachments, | |
| depth_attachment=buffer_depth_tex_b, | |
| ) | |
| fbo_prev.viewport = (0, 0, width, height) | |
| # Set uniforms | |
| prog['uViewMat'].write(np.ascontiguousarray(view.transpose().astype('f4'))) | |
| prog['uProjectionMat'].write(np.ascontiguousarray(projection.transpose().astype('f4'))) | |
| prog['uScreenSize'].value = (width, height) | |
| # Rendering settings | |
| ctx.mgl_ctx.depth_func = '<' | |
| ctx.mgl_ctx.enable(ctx.mgl_ctx.DEPTH_TEST) | |
| if cull_backface: | |
| ctx.mgl_ctx.enable(ctx.mgl_ctx.CULL_FACE) | |
| else: | |
| ctx.mgl_ctx.disable(ctx.mgl_ctx.CULL_FACE) | |
| ctx.mgl_ctx.disable(ctx.mgl_ctx.BLEND) | |
| # Initialize prev depth texture = 0 | |
| clear_texture(ctx, fbo_prev.depth_attachment, value=0.0) | |
| try: | |
| while True: | |
| # Clear | |
| clear_texture(ctx, image_tex, value=(0.0, 0.0, 0.0, 1.0)) | |
| clear_texture(ctx, mask_tex, value=0.0) | |
| clear_texture(ctx, fbo_curr.depth_attachment, value=1.0) | |
| if return_interpolation: | |
| clear_texture(ctx, interpolation_id_tex, value=(-1,)) | |
| clear_texture(ctx, interpolation_uv_tex, value=(0.0, 0.0)) | |
| # Render | |
| fbo_prev.depth_attachment.use(location=0) | |
| fbo_curr.use() | |
| vao.render() | |
| # Read | |
| mask = np.frombuffer(mask_tex.read(), dtype=bool).reshape((height, width)) | |
| mask = np.flip(mask, axis=0) | |
| if attributes is not None: | |
| image = np.frombuffer(image_tex.read(), dtype='f4').reshape((height, width, 4)) | |
| image = np.flip(image, axis=0) | |
| image = image[:, :, :num_channels] | |
| else: | |
| image = None | |
| if return_depth: | |
| depth_texture_buffer_to_linear(ctx, fbo_curr.depth_attachment, linear_depth_tex) | |
| depth = np.frombuffer(linear_depth_tex.read(), dtype='f4').reshape((height, width)) | |
| depth = np.flip(depth, axis=0) | |
| else: | |
| depth = None | |
| if return_interpolation: | |
| interpolation_id = np.frombuffer(interpolation_id_tex.read(), dtype='i4').reshape((height, width)) | |
| interpolation_id = np.flip(interpolation_id, axis=0) | |
| interpolation_uv = np.frombuffer(interpolation_uv_tex.read(), dtype='f4').reshape((height, width, 2)) | |
| interpolation_uv = np.flip(interpolation_uv, axis=0) | |
| else: | |
| interpolation_id = None | |
| interpolation_uv = None | |
| # Yield | |
| output = { | |
| "image": image, | |
| "mask": mask, | |
| "depth": depth, | |
| "interpolation_id": interpolation_id, | |
| "interpolation_uv": interpolation_uv | |
| } | |
| output = {k: v for k, v in output.items() if v is not None} | |
| yield output | |
| # Check termination | |
| if not np.any(mask): | |
| break # No more layers | |
| # Swap curr and prev fbos | |
| fbo_curr, fbo_prev = fbo_prev, fbo_curr | |
| finally: | |
| # Release | |
| vao.release() | |
| vbo_vertices.release() | |
| vbo_attributes.release() | |
| fbo_curr.release() | |
| fbo_prev.release() | |
| image_tex.release() | |
| mask_tex.release() | |
| buffer_depth_tex_a.release() | |
| buffer_depth_tex_b.release() | |
| if return_depth: | |
| linear_depth_tex.release() | |
| if return_interpolation: | |
| interpolation_id_tex.release() | |
| interpolation_uv_tex.release() | |
| def rasterize_lines( | |
| size: Tuple[int, int], | |
| *, | |
| vertices: ndarray, | |
| attributes: Optional[ndarray], | |
| attributes_domain: Literal['vertex', 'line'] = 'vertex', | |
| lines: ndarray | None = None, | |
| view: Optional[ndarray] = None, | |
| projection: Optional[ndarray] = None, | |
| extrinsics: Optional[ndarray] = None, | |
| intrinsics: Optional[ndarray] = None, | |
| near: float = 0.01, | |
| far: float = float('inf'), | |
| line_width: float = 1.0, | |
| return_depth: bool = False, | |
| return_interpolation: bool = False, | |
| background: Optional[Dict[str, ndarray]] = None, | |
| ctx: Optional[RastContext] = None, | |
| ) -> Tuple[ndarray, ...]: | |
| """ | |
| Rasterize lines. | |
| Parameters | |
| ---- | |
| - `size` (Tuple[int, int]): (height, width) of the output image | |
| - `vertices` (ndarray): (N, 3) or (L, 2, 3) | |
| - `attributes` (ndarray): (N, C), (T, 3, C) for vertex domain or (T, C) for face domain | |
| - `attributes_domain` (Literal['vertex', 'face']): domain of the attributes | |
| - `lines` (ndarray): (L, 2) int32 array of line vertex indices. If None, vertices shape must be (L, 2, 3) and will be interpreted as line segments directly. | |
| - `view` | `extrinsics` (ndarray): (4, 4) View matrix or extrinsics matrix. Provide either one of them. | |
| - `projection` | `intrinsics` (ndarray): (4, 4) Projection matrix or (3, 3) Intrinsics matrix. Provide either one of them. | |
| - `near` (float): near clipping plane. Only used for intrinsics. Ignored if projection matrix is provided. | |
| - `far` (float): far clipping plane. Only used for intrinsics. Ignored if projection matrix is provided. | |
| - `background` (Optional[Dict[str, ndarray]]): background to composite with. Contains the same keys as the return dictionary, each with shape matching the output. | |
| - `ctx` (RastContext): rasterization context. Created by `RastContext()`. Defaults to the current default context. | |
| Returns | |
| ---- | |
| A dictionary containing | |
| - `mask` (ndarray): (H, W) bool mask of valid pixels | |
| if attributes is not None | |
| - `image` (ndarray): (H, W, C) float32 rendered image corresponding to the input attributes | |
| if return_depth is True | |
| - `depth` (ndarray): (H, W) float32 camera space linear depth, ranging from 0 to 1. | |
| if return_interpolation is True | |
| - `interpolation_id` (ndarray): (H, W) int32 triangle ID map | |
| - `interpolation_uv` (ndarray): (H, W, 2) float32 triangle UV (first two channels of barycentric coordinates) | |
| """ | |
| if ctx is None: | |
| ctx = RastContext.get_default_context() | |
| height, width = size | |
| if lines is None: | |
| assert vertices.ndim == 3 and vertices.shape[1] == 2 and vertices.shape[2] == 3, "If lines is None, vertices must be an array with shape (T, 2, 3)" | |
| else: | |
| assert lines.ndim == 2 and lines.shape[1] == 2, f"Lines should be a int32 or uint32 array with shape (T, 2), but got {lines.shape} {lines.dtype}" | |
| assert lines.dtype == np.uint32 or lines.dtype == np.int32 | |
| assert vertices.ndim == 2 and vertices.shape[1] == 3 | |
| assert vertices.dtype == np.float32 | |
| if attributes is not None: | |
| assert attributes.dtype == np.float32 | |
| if attributes_domain == 'vertex': | |
| assert (attributes.shape[:-1] == vertices.shape[:-1]), f"Attribute shape {attributes.shape} does not match vertex shape {vertices.shape}" | |
| elif attributes_domain == 'line': | |
| if lines is None: | |
| assert attributes.shape[0] == vertices.shape[0], f"Attribute shape {attributes.shape} does not match vertex shape {vertices.shape}" | |
| else: | |
| assert attributes.shape[0] == lines.shape[0], f"Attribute shape {attributes.shape} does not match line shape {lines.shape}" | |
| else: | |
| raise ValueError(f"Unknown attributes domain: {attributes_domain}") | |
| assert attributes.shape[-1] in [1, 2, 3, 4], f'Vertex attribute only supports channels 1, 2, 3, 4, but got {attributes.shape[-1]}' | |
| assert intrinsics is None or (intrinsics.shape == (3, 3) and intrinsics.dtype == np.float32), f"Intrinsics should be a 3x3 float32 matrix, but got {intrinsics.shape} {intrinsics.dtype}" | |
| assert extrinsics is None or (extrinsics.shape == (4, 4) and extrinsics.dtype == np.float32), f"Extrinsics should be a 4x4 float32 matrix, but got {extrinsics.shape} {extrinsics.dtype}" | |
| assert view is None or (view.shape == (4, 4) and view.dtype == np.float32), f"View should be a 4x4 float32 matrix, but got {view.shape} {view.dtype}" | |
| assert projection is None or (projection.shape == (4, 4) and projection.dtype == np.float32), f"Projection should be a 4x4 float32 matrix, but got {projection.shape} {projection.dtype}" | |
| background_image, background_mask, background_depth, background_interpolation_id, background_interpolation_uv = [background.get(k, None) if background is not None else None for k in ['image', 'mask', 'depth', 'interpolation_id', 'interpolation_uv']] | |
| if background_image is not None: | |
| assert background_image.ndim == 3 and background_image.shape == (height, width, attributes.shape[-1]), f"Image should be a float32 array with shape (H, W, {attributes.shape[1]}), but got {background_image.shape} {background_image.dtype}" | |
| if background_mask is not None: | |
| assert background_mask.dtype == np.bool_ and background_mask.ndim == 2 and background_mask.shape == (height, width), f"Mask should be a bool array with shape (H, W), but got {background_mask.shape} {background_mask.dtype}" | |
| if background_depth is not None: | |
| assert background_depth.dtype == np.float32 and background_depth.ndim == 2 and background_depth.shape == (height, width), f"Depth should be a float32 array with shape (H, W), but got {background_depth.shape} {background_depth.dtype}" | |
| if background_interpolation_id is not None: | |
| assert background_interpolation_id.dtype == np.int32 and background_interpolation_id.ndim == 2 and background_interpolation_id.shape == (height, width), f"Interpolation ID should be a int32 array with shape (H, W), but got {background_interpolation_id.shape} {background_interpolation_id.dtype}" | |
| if background_interpolation_uv is not None: | |
| assert background_interpolation_uv.dtype == np.float32 and background_interpolation_uv.ndim == 3 and background_interpolation_uv.shape == (height, width, 2), f"Interpolation UV should be a float32 array with shape (H, W, 2), but got {background_interpolation_uv.shape} {background_interpolation_uv.dtype}" | |
| if lines is not None: | |
| vertices = vertices[lines] | |
| if attributes is not None: | |
| num_channels = attributes.shape[-1] | |
| attributes = np.concatenate([attributes, np.zeros((*attributes.shape[:-1], 4 - num_channels,), dtype=attributes.dtype)], axis=-1) if num_channels < 4 else attributes | |
| if attributes_domain == 'vertex': | |
| if lines is not None: | |
| attributes = attributes[lines] | |
| elif attributes_domain == 'line': | |
| attributes = attributes[:, None, :].repeat(2, axis=1) | |
| if extrinsics is not None: | |
| assert view is None, "Cannot specify both extrinsics and view." | |
| view = extrinsics_to_view(extrinsics) | |
| if intrinsics is not None: | |
| assert projection is None, "Cannot specify both intrinsics and projection." | |
| projection = intrinsics_to_perspective(intrinsics, near, far) | |
| if view is None: | |
| view = np.eye(4, dtype=np.float32) | |
| if projection is None: | |
| projection = np.eye(4, dtype=np.float32) | |
| if background_image is not None: | |
| background_image = np.concatenate([background_image, np.ones((height, width, 4 - background_image.shape[-1]), dtype=background_image.dtype)], axis=-1) if background_image.shape[-1] < 4 else background_image | |
| # Get program (shared with triangle rasterization) | |
| flat = attributes_domain == 'line' | |
| program_name = f'rasterize_triangles(flat={flat}, return_interpolation={return_interpolation})' | |
| if (prog := ctx.programs.get(program_name, None)) is None: | |
| prog = ctx.mgl_ctx.program( | |
| vertex_shader=RASTERIZE_TRIANGLES_VS.replace('{flat}', 'flat' if flat else '').replace('{return_interpolation}', '' if return_interpolation else '//'), | |
| fragment_shader=RASTERIZE_TRIANGLES_FS.replace('{flat}', 'flat' if flat else '').replace('{return_interpolation}', '' if return_interpolation else '//') | |
| ) | |
| ctx.programs[program_name] = prog | |
| # Create buffers | |
| vbo_vertices = ctx.mgl_ctx.buffer(np.ascontiguousarray(vertices, dtype='f4')) | |
| if attributes is not None: | |
| vbo_attributes = ctx.mgl_ctx.buffer(np.ascontiguousarray(attributes, dtype='f4')) | |
| if return_interpolation: | |
| vbo_uv = ctx.mgl_ctx.buffer(np.ascontiguousarray((np.array([[1., 0.], [0., 1.]], dtype=np.float32).reshape(1, 2, 2).repeat(vertices.shape[0], axis=0)))) | |
| vao = ctx.mgl_ctx.vertex_array( | |
| prog, | |
| list(filter(lambda x: x is not None, [ | |
| (vbo_vertices, '3f', 'inVert'), | |
| (vbo_attributes, f'4f', 'inAttr') if attributes is not None else None, | |
| (vbo_uv, '2f', 'inUV') if return_interpolation else None, | |
| ])), | |
| mode=moderngl.LINES, | |
| ) | |
| # Create textures | |
| image_tex = ctx.mgl_ctx.texture((width, height), 4, dtype='f4', data=np.ascontiguousarray(background_image[::-1, :, :]) if background_image is not None else None) | |
| mask_tex = ctx.mgl_ctx.texture((width, height), 1, dtype='f1', data=np.ascontiguousarray(background_mask[::-1, :]) if background_mask is not None else None) | |
| buffer_depth_tex = ctx.mgl_ctx.depth_texture((width, height)) | |
| if return_depth is not None: | |
| linear_depth_tex = ctx.mgl_ctx.texture((width, height), 1, dtype='f4', data=np.ascontiguousarray(background_depth[::-1, :]) if background_depth is not None else None) | |
| if return_interpolation: | |
| interpolation_id_tex = ctx.mgl_ctx.texture((width, height), 1, dtype='i4', data=np.ascontiguousarray(background_interpolation_id[::-1, :]) if background_interpolation_id is not None else None) | |
| interpolation_uv_tex = ctx.mgl_ctx.texture((width, height), 2, dtype='f4', data=np.ascontiguousarray(background_interpolation_uv[::-1, :, :]) if background_interpolation_uv is not None else None) | |
| if background_image is None: | |
| clear_texture(ctx, image_tex, value=(0.0, 0.0, 0.0, 1.0)) | |
| if background_mask is None: | |
| clear_texture(ctx, mask_tex, value=0.0) | |
| if background_depth is None: | |
| clear_texture(ctx, buffer_depth_tex, value=1.0) | |
| else: | |
| depth_texture_linear_to_buffer(ctx, linear_depth_tex, buffer_depth_tex) | |
| if return_interpolation: | |
| if background_interpolation_id is None: | |
| clear_texture(ctx, interpolation_id_tex, value=(-1,)) | |
| if background_interpolation_uv is None: | |
| clear_texture(ctx, interpolation_uv_tex, value=(0.0, 0.0)) | |
| # Create framebuffer | |
| if return_interpolation: | |
| fbo = ctx.mgl_ctx.framebuffer( | |
| color_attachments=[image_tex, mask_tex, interpolation_id_tex, interpolation_uv_tex], | |
| depth_attachment=buffer_depth_tex, | |
| ) | |
| else: | |
| fbo = ctx.mgl_ctx.framebuffer( | |
| color_attachments=[image_tex, mask_tex], | |
| depth_attachment=buffer_depth_tex, | |
| ) | |
| fbo.viewport = (0, 0, width, height) | |
| # Set uniforms | |
| prog['uViewMat'].write(np.ascontiguousarray(view.transpose().astype('f4'))) | |
| prog['uProjectionMat'].write(np.ascontiguousarray(projection.transpose().astype('f4'))) | |
| # Set render states | |
| ctx.mgl_ctx.depth_func = '<' | |
| ctx.mgl_ctx.enable(ctx.mgl_ctx.DEPTH_TEST) | |
| ctx.mgl_ctx.disable(ctx.mgl_ctx.BLEND) | |
| ctx.mgl_ctx.line_width = line_width | |
| # Render | |
| fbo.use() | |
| vao.render() | |
| # Read | |
| mask = np.frombuffer(mask_tex.read(), dtype=bool).reshape((height, width)) | |
| mask = np.flip(mask, axis=0) | |
| if attributes is not None: | |
| image = np.frombuffer(image_tex.read(), dtype='f4').reshape((height, width, 4)) | |
| image = np.flip(image, axis=0) | |
| image = image[:, :, :num_channels] | |
| else: | |
| image = None | |
| if return_depth: | |
| depth_texture_buffer_to_linear(ctx, buffer_depth_tex, linear_depth_tex) | |
| depth = np.frombuffer(linear_depth_tex.read(), dtype='f4').reshape((height, width)) | |
| depth = np.flip(depth, axis=0) | |
| else: | |
| depth = None | |
| if return_interpolation: | |
| interpolation_id = np.frombuffer(interpolation_id_tex.read(), dtype='i4').reshape((height, width)) | |
| interpolation_id = np.flip(interpolation_id, axis=0) | |
| interpolation_uv = np.frombuffer(interpolation_uv_tex.read(), dtype='f4').reshape((height, width, 2)) | |
| interpolation_uv = np.flip(interpolation_uv, axis=0) | |
| else: | |
| interpolation_id = None | |
| interpolation_uv = None | |
| # Release | |
| vao.release() | |
| vbo_vertices.release() | |
| if attributes is not None: | |
| vbo_attributes.release() | |
| if return_interpolation: | |
| vbo_uv.release() | |
| fbo.release() | |
| image_tex.release() | |
| mask_tex.release() | |
| buffer_depth_tex.release() | |
| if background_depth is not None or return_depth: | |
| linear_depth_tex.release() | |
| if return_interpolation: | |
| interpolation_id_tex.release() | |
| interpolation_uv_tex.release() | |
| output = { | |
| "image": image, | |
| "mask": mask, | |
| "depth": depth, | |
| "interpolation_id": interpolation_id, | |
| "interpolation_uv": interpolation_uv | |
| } | |
| output = {k: v for k, v in output.items() if v is not None} | |
| return output | |
| def rasterize_point_cloud( | |
| size: Tuple[int, int], | |
| *, | |
| points: ndarray, | |
| point_sizes: Union[float, ndarray] = 10, | |
| point_size_in: Literal['2d', '3d'] = '2d', | |
| point_shape: Literal['triangle', 'square', 'pentagon', 'hexagon', 'circle'] = 'square', | |
| attributes: Optional[ndarray] = None, | |
| view: ndarray = None, | |
| projection: ndarray = None, | |
| extrinsics: ndarray = None, | |
| intrinsics: ndarray = None, | |
| near: float = 0.01, | |
| far: float = float('inf'), | |
| return_depth: bool = False, | |
| return_point_id: bool = False, | |
| background: Optional[Dict[str, ndarray]] = None, | |
| ctx: Optional[RastContext] = None, | |
| ) -> Dict[str, ndarray]: | |
| """ | |
| Rasterize point cloud. | |
| Parameters | |
| ---- | |
| - `size` (Tuple[int, int]): (height, width) of the output image | |
| - `points` (ndarray): (N, 3) | |
| - `point_sizes` (ndarray): (N,) or float | |
| - `point_size_in`: Literal['2d', '3d'] = '2d'. Whether the point sizes are in 2D (size in pixels) or 3D (size in world units). | |
| - `point_shape`: Literal['triangle', 'square', 'pentagon', 'hexagon', 'circle'] = 'square'. The visual shape of the points. | |
| - `attributes` (ndarray): (N, C) | |
| - `view` | `extrinsics` (ndarray): (4, 4) View matrix or extrinsics matrix. Provide either one of them. | |
| - `projection` | `intrinsics` (ndarray): (4, 4) Projection matrix or (3, 3) Intrinsics matrix. Provide either one of them. | |
| - `near` (float): near clipping plane. Only used for intrinsics. Ignored if projection matrix is provided. | |
| - `far` (float): far clipping plane. Only used for intrinsics. Ignored if projection matrix is provided. | |
| - `return_depth` (bool): whether to return depth map | |
| - `return_point_id` (bool): whether to return point ID map | |
| - `background` (Optional[Dict[str, ndarray]]): background to composite with. Contains the same keys as the return dictionary, each with shape matching the output. | |
| - `ctx` (RastContext): rasterization context. Created by `RastContext()`. Defaults to the current default context. | |
| Returns | |
| ---- | |
| A dictionary containing | |
| if attributes is not None | |
| - `image` (ndarray): (H, W, C) float32 rendered image corresponding to the input attributes | |
| if return_depth is True | |
| - `depth` (ndarray): (H, W) float32 camera space linear depth, ranging from 0 to 1. | |
| if return_point_id is True | |
| - `point_id` (ndarray): (H, W) int32 point ID map | |
| """ | |
| if ctx is None: | |
| ctx = RastContext.get_default_context() | |
| height, width = size | |
| assert points.ndim == 2 and points.shape[1] == 3 and points.dtype == np.float32, f"Points should be a float32 array with shape (N, 3), but got {points.shape} {points.dtype}" | |
| if attributes is not None: | |
| assert attributes.dtype == np.float32 | |
| assert (attributes.shape[:-1] == points.shape[:-1]), f"Attribute shape {attributes.shape} does not match point shape {points.shape}" | |
| assert attributes.shape[-1] in [1, 2, 3, 4], f'Vertex attribute only supports channels 1, 2, 3, 4, but got {attributes.shape[-1]}' | |
| assert intrinsics is None or (intrinsics.shape == (3, 3) and intrinsics.dtype == np.float32), f"Intrinsics should be a 3x3 float32 matrix, but got {intrinsics.shape} {intrinsics.dtype}" | |
| assert extrinsics is None or (extrinsics.shape == (4, 4) and extrinsics.dtype == np.float32), f"Extrinsics should be a 4x4 float32 matrix, but got {extrinsics.shape} {extrinsics.dtype}" | |
| assert view is None or (view.shape == (4, 4) and view.dtype == np.float32), f"View should be a 4x4 float32 matrix, but got {view.shape} {view.dtype}" | |
| assert projection is None or (projection.shape == (4, 4) and projection.dtype == np.float32), f"Projection should be a 4x4 float32 matrix, but got {projection.shape} {projection.dtype}" | |
| background_image, background_depth, background_mask, background_point_id = [background.get(k, None) if background is not None else None for k in ['image', 'depth', 'mask', 'point_id']] | |
| if background_image is not None: | |
| assert background_image.ndim == 3 and background_image.shape == (height, width, attributes.shape[-1]), f"Image should be a float32 array with shape (H, W, {attributes.shape[1]}), but got {background_image.shape} {background_image.dtype}" | |
| if background_depth is not None: | |
| assert background_depth.dtype == np.float32 and background_depth.ndim == 2 and background_depth.shape == (height, width), f"Depth should be a float32 array with shape (H, W), but got {background_depth.shape} {background_depth.dtype}" | |
| if background_mask is not None: | |
| assert background_mask.dtype == np.bool_ and background_mask.ndim == 2 and background_mask.shape == (height, width), f"Mask should be a bool array with shape (H, W), but got {background_mask.shape} {background_mask.dtype}" | |
| if background_point_id is not None: | |
| assert background_point_id.dtype == np.int32 and background_point_id.ndim == 2 and background_point_id.shape == (height, width), f"Point ID should be a int32 array with shape (H, W), but got {background_point_id.shape} {background_point_id.dtype}" | |
| if not isinstance(point_sizes, ndarray): | |
| point_sizes = np.full((points.shape[0],), point_sizes, dtype=np.float32) | |
| if attributes is not None: | |
| num_channels = attributes.shape[-1] | |
| attributes = np.concatenate([attributes, np.zeros((*attributes.shape[:-1], 4 - num_channels,), dtype=attributes.dtype)], axis=-1) if num_channels < 4 else attributes | |
| if background_image is not None: | |
| background_image = np.concatenate([background_image, np.ones((height, width, 4 - background_image.shape[-1]), dtype=background_image.dtype)], axis=-1) if background_image.shape[-1] < 4 else background_image | |
| if extrinsics is not None: | |
| assert view is None, "Cannot specify both extrinsics and view." | |
| view = extrinsics_to_view(extrinsics) | |
| if intrinsics is not None: | |
| assert projection is None, "Cannot specify both intrinsics and projection." | |
| projection = intrinsics_to_perspective(intrinsics, near, far) | |
| if view is None: | |
| view = np.eye(4, dtype=np.float32) | |
| if projection is None: | |
| projection = np.eye(4, dtype=np.float32) | |
| # Get program | |
| program_name = f'rasterize_point_cloud(return_point_id={return_point_id}, point_shape={point_shape})' | |
| num_point_shape_vertices = {'triangle': 3, 'square': 4, 'pentagon': 5, 'hexagon': 6, 'circle': 4}.get(point_shape) | |
| if (prog := ctx.programs.get(program_name, None)) is None: | |
| prog = ctx.mgl_ctx.program( | |
| vertex_shader=RASTERIZE_POINT_CLOUD_VS.replace('{return_point_id}', '' if return_point_id else '//'), | |
| geometry_shader=RASTERIZE_POINT_CLOUD_GS\ | |
| .replace('{return_point_id}', '' if return_point_id else '//')\ | |
| .replace('{point_shape}', point_shape)\ | |
| .replace('{num_point_shape_vertices}', str(num_point_shape_vertices)), | |
| fragment_shader=RASTERIZE_POINT_CLOUD_FS\ | |
| .replace('{return_point_id}', '' if return_point_id else '//')\ | |
| .replace('{circle_begin}', '' if point_shape == 'circle' else '/*')\ | |
| .replace('{circle_end}', '' if point_shape == 'circle' else '*/') | |
| ) | |
| ctx.programs[program_name] = prog | |
| # Create buffers | |
| vbo_vertices = ctx.mgl_ctx.buffer(np.ascontiguousarray(points, dtype='f4')) | |
| if attributes is not None: | |
| vbo_attributes = ctx.mgl_ctx.buffer(np.ascontiguousarray(attributes, dtype='f4')) | |
| vbo_point_sizes = ctx.mgl_ctx.buffer(np.ascontiguousarray(point_sizes, dtype='f4')) | |
| vao = ctx.mgl_ctx.vertex_array( | |
| prog, | |
| list(filter(lambda x: x is not None, [ | |
| (vbo_vertices, '3f', 'inVert'), | |
| (vbo_attributes, f'4f', 'inAttr') if attributes is not None else None, | |
| (vbo_point_sizes, '1f', 'inPointSize') if point_sizes is not None else None, | |
| ])) | |
| ) | |
| # Create textures | |
| image_tex = ctx.mgl_ctx.texture((width, height), 4, dtype='f4', data=np.ascontiguousarray(background_image[::-1, :, :]) if background_image is not None else None) | |
| mask_tex = ctx.mgl_ctx.texture((width, height), 1, dtype='f1', data=np.ascontiguousarray(background_mask[::-1, :]) if background_mask is not None else None) | |
| buffer_depth_tex = ctx.mgl_ctx.depth_texture((width, height)) | |
| if return_depth: | |
| linear_depth_tex = ctx.mgl_ctx.texture((width, height), 1, dtype='f4', data=np.ascontiguousarray(background_depth[::-1, :]) if background_depth is not None else None) | |
| if return_point_id: | |
| point_id_tex = ctx.mgl_ctx.texture((width, height), 1, dtype='i4', data=np.ascontiguousarray(background_point_id[::-1, :]) if background_point_id is not None else None) | |
| if background_image is None: | |
| clear_texture(ctx, image_tex, value=(0.0, 0.0, 0.0, 1.0)) | |
| if background_mask is None: | |
| clear_texture(ctx, mask_tex, value=0.0) | |
| if background_depth is None: | |
| clear_texture(ctx, buffer_depth_tex, value=1.0) | |
| else: | |
| depth_texture_linear_to_buffer(ctx, linear_depth_tex, buffer_depth_tex) | |
| if return_point_id: | |
| clear_texture(ctx, point_id_tex, value=(-1,)) | |
| # Create framebuffer | |
| if return_point_id: | |
| fbo = ctx.mgl_ctx.framebuffer( | |
| color_attachments=[image_tex, mask_tex, point_id_tex], | |
| depth_attachment=buffer_depth_tex, | |
| ) | |
| else: | |
| fbo = ctx.mgl_ctx.framebuffer( | |
| color_attachments=[image_tex, mask_tex], | |
| depth_attachment=buffer_depth_tex, | |
| ) | |
| fbo.viewport = (0, 0, width, height) | |
| # Set uniforms | |
| prog['uViewMat'].write(np.ascontiguousarray(view.transpose().astype('f4'))) | |
| prog['uProjectionMat'].write(np.ascontiguousarray(projection.transpose().astype('f4'))) | |
| prog['uScreenSize'].value = (width, height) | |
| prog['uIsPointSize3D'].value = point_size_in == '3d' | |
| # Set render states | |
| ctx.mgl_ctx.depth_func = '<' | |
| ctx.mgl_ctx.enable(ctx.mgl_ctx.DEPTH_TEST) | |
| ctx.mgl_ctx.disable(ctx.mgl_ctx.CULL_FACE) | |
| ctx.mgl_ctx.disable(ctx.mgl_ctx.BLEND) | |
| # Render | |
| fbo.use() | |
| vao.render(mode=moderngl.POINTS) | |
| # Read | |
| mask = np.frombuffer(mask_tex.read(), dtype=bool).reshape((height, width)) | |
| mask = np.flip(mask, axis=0) | |
| if attributes is not None: | |
| image = np.frombuffer(image_tex.read(), dtype='f4').reshape((height, width, 4)) | |
| image = np.flip(image, axis=0) | |
| image = image[:, :, :num_channels] | |
| else: | |
| image = None | |
| if return_depth: | |
| depth_texture_buffer_to_linear(ctx, buffer_depth_tex, linear_depth_tex) | |
| depth = np.frombuffer(linear_depth_tex.read(), dtype='f4').reshape((height, width)) | |
| depth = np.flip(depth, axis=0) | |
| else: | |
| depth = None | |
| if return_point_id: | |
| point_id = np.frombuffer(point_id_tex.read(), dtype='i4').reshape((height, width)) | |
| point_id = np.flip(point_id, axis=0) | |
| else: | |
| point_id = None | |
| # Release | |
| vao.release() | |
| vbo_vertices.release() | |
| if attributes is not None: | |
| vbo_attributes.release() | |
| vbo_point_sizes.release() | |
| fbo.release() | |
| image_tex.release() | |
| mask_tex.release() | |
| buffer_depth_tex.release() | |
| if background_depth is not None or return_depth: | |
| linear_depth_tex.release() | |
| if return_point_id: | |
| point_id_tex.release() | |
| output = { | |
| "image": image, | |
| "depth": depth, | |
| "mask": mask, | |
| "point_id": point_id | |
| } | |
| output = {k: v for k, v in output.items() if v is not None} | |
| return output | |
| def sample_texture( | |
| uv_map: ndarray, | |
| texture_map: ndarray, | |
| interpolation: Literal['linear', 'nearest'] = 'linear', | |
| mipmap_level: Union[int, Tuple[int, int]] = 0, | |
| repeat: Union[bool, Tuple[bool, bool]] = False, | |
| anisotropic: float = 1.0, | |
| ctx: Optional[RastContext] = None | |
| ) -> ndarray: | |
| """ | |
| Sample from a texture map with a UV map. | |
| """ | |
| if ctx is None: | |
| ctx = RastContext.get_default_context() | |
| assert len(texture_map.shape) == 3 and 1 <= texture_map.shape[2] <= 4 | |
| assert uv_map.shape[2] == 2 | |
| height, width = uv_map.shape[:2] | |
| texture_dtype = map_np_dtype(texture_map.dtype) | |
| # Create texture | |
| texture_tex = ctx.mgl_ctx.texture((texture_map.shape[1], texture_map.shape[0]), texture_map.shape[2], dtype=texture_dtype, data=np.ascontiguousarray(texture_map)) | |
| if isinstance(mipmap_level, tuple): | |
| base_mipmap_level, max_mipmap_level = mipmap_level | |
| else: | |
| base_mipmap_level, max_mipmap_level = 0, mipmap_level | |
| if base_mipmap_level > 0 or max_mipmap_level > 0: | |
| use_mipmap = True | |
| texture_tex.build_mipmaps(base_mipmap_level, max_mipmap_level) | |
| else: | |
| use_mipmap = False | |
| if interpolation == 'linear': | |
| texture_tex.filter = (moderngl.LINEAR_MIPMAP_LINEAR if use_mipmap else moderngl.LINEAR, moderngl.LINEAR) | |
| elif interpolation == 'nearest': | |
| texture_tex.filter = (moderngl.NEAREST_MIPMAP_NEAREST if use_mipmap else moderngl.NEAREST, moderngl.NEAREST) | |
| if isinstance(repeat, tuple): | |
| texture_tex.repeat_x, texture_tex.repeat_y = repeat | |
| else: | |
| texture_tex.repeat_x = texture_tex.repeat_y = repeat | |
| texture_tex.anisotropy = anisotropic | |
| uv_tex = ctx.mgl_ctx.texture((width, height), 2, dtype='f4', data=np.ascontiguousarray(uv_map.astype('f4', copy=False))) | |
| uv_tex.filter = (moderngl.NEAREST, moderngl.NEAREST) | |
| # Create render buffer and frame buffer | |
| output_tex = ctx.mgl_ctx.texture((uv_map.shape[1], uv_map.shape[0]), texture_map.shape[2], dtype=texture_dtype) | |
| # Get program | |
| program_name = 'sample_texture' | |
| if (prog := ctx.programs.get(program_name, None)) is None: | |
| prog = ctx.mgl_ctx.program( | |
| vertex_shader=FULL_SCREEN_VS, | |
| fragment_shader=SAMPLE_TEXTURE_FS | |
| ) | |
| prog['texMap'] = 0 | |
| prog['uvMap'] = 1 | |
| ctx.programs[program_name] = prog | |
| # Render | |
| run_full_screen_program(ctx, prog, [texture_tex, uv_tex], output_tex) | |
| # Read buffer | |
| image_buffer = np.frombuffer(output_tex.read(), dtype=texture_dtype).reshape((height, width, texture_tex.shape[2])) | |
| # Release | |
| texture_tex.release() | |
| output_tex.release() | |
| uv_tex.release() | |
| return image_buffer | |
| # def warp_image_by_depth( | |
| # ctx: RastContext, | |
| # src_depth: ndarray, | |
| # src_image: ndarray = None, | |
| # width: int = None, | |
| # height: int = None, | |
| # *, | |
| # extrinsics_src: ndarray = None, | |
| # extrinsics_tgt: ndarray = None, | |
| # intrinsics_src: ndarray = None, | |
| # intrinsics_tgt: ndarray = None, | |
| # near: float = 0.1, | |
| # far: float = 100.0, | |
| # cull_backface: bool = True, | |
| # ssaa: int = 1, | |
| # return_depth: bool = False, | |
| # ) -> Tuple[ndarray, ...]: | |
| # """ | |
| # Warp image by depth map. | |
| # ## Parameters | |
| # ctx (RastContext): rasterizer context | |
| # src_depth (ndarray): [H, W] | |
| # src_image (ndarray, optional): [H, W, C]. The image to warp. Defaults to None (use uv coordinates). | |
| # width (int, optional): width of the output image. None to use depth map width. Defaults to None. | |
| # height (int, optional): height of the output image. None to use depth map height. Defaults to None. | |
| # extrinsics_src (ndarray, optional): extrinsics matrix of the source camera. Defaults to None (identity). | |
| # extrinsics_tgt (ndarray, optional): extrinsics matrix of the target camera. Defaults to None (identity). | |
| # intrinsics_src (ndarray, optional): intrinsics matrix of the source camera. Defaults to None (use the same as intrinsics_tgt). | |
| # intrinsics_tgt (ndarray, optional): intrinsics matrix of the target camera. Defaults to None (use the same as intrinsics_src). | |
| # cull_backface (bool, optional): whether to cull backface. Defaults to True. | |
| # ssaa (int, optional): super sampling anti-aliasing. Defaults to 1. | |
| # ## Returns | |
| # tgt_image (ndarray): [H, W, C] warped image (or uv coordinates if image is None). | |
| # tgt_depth (ndarray): [H, W] screen space depth, ranging from 0 to 1. If return_depth is False, it is None. | |
| # """ | |
| # assert src_depth.ndim == 2 | |
| # if width is None: | |
| # width = src_depth.shape[1] | |
| # if height is None: | |
| # height = src_depth.shape[0] | |
| # if src_image is not None: | |
| # assert src_image.shape[-2:] == src_depth.shape[-2:], f'Shape of source image {src_image.shape} does not match shape of source depth {src_depth.shape}' | |
| # # set up default camera parameters | |
| # extrinsics_src = np.eye(4) if extrinsics_src is None else extrinsics_src | |
| # extrinsics_tgt = np.eye(4) if extrinsics_tgt is None else extrinsics_tgt | |
| # intrinsics_src = intrinsics_tgt if intrinsics_src is None else intrinsics_src | |
| # intrinsics_tgt = intrinsics_src if intrinsics_tgt is None else intrinsics_tgt | |
| # assert all(x is not None for x in [extrinsics_src, extrinsics_tgt, intrinsics_src, intrinsics_tgt]), "Make sure you have provided all the necessary camera parameters." | |
| # # check shapes | |
| # assert extrinsics_src.shape == (4, 4) and extrinsics_tgt.shape == (4, 4) | |
| # assert intrinsics_src.shape == (3, 3) and intrinsics_tgt.shape == (3, 3) | |
| # # convert to view and perspective matrices | |
| # view_tgt = transforms.extrinsics_to_view(extrinsics_tgt) | |
| # perspective_tgt = transforms.intrinsics_to_perspective(intrinsics_tgt, near=near, far=far) | |
| # # unproject depth map | |
| # uv, faces = utils.image_mesh(*src_depth.shape[-2:]) | |
| # pts = transforms.unproject_cv(uv, src_depth.reshape(-1), extrinsics_src, intrinsics_src) | |
| # faces = mesh.triangulate_mesh(faces, vertices=pts) | |
| # # rasterize attributes | |
| # if src_image is not None: | |
| # attr = src_image.reshape(-1, src_image.shape[-1]) | |
| # else: | |
| # attr = uv | |
| # tgt_image, tgt_depth = rasterize_triangles( | |
| # ctx, | |
| # width * ssaa, | |
| # height * ssaa, | |
| # vertices=pts, | |
| # faces=faces, | |
| # attr=attr, | |
| # transform=perspective_tgt @ view_tgt, | |
| # cull_backface=cull_backface, | |
| # return_depth=return_depth, | |
| # ) | |
| # if ssaa > 1: | |
| # tgt_image = tgt_image.reshape(height, ssaa, width, ssaa, -1).mean(axis=(1, 3)) | |
| # tgt_depth = tgt_depth.reshape(height, ssaa, width, ssaa, -1).mean(axis=(1, 3)) if return_depth else None | |
| # return tgt_image, tgt_depth | |
| def test_rasterization(ctx: Optional[RastContext] = None): | |
| """ | |
| Test if rasterization works. It will render a cube with random colors and save it as a CHECKME.png file. | |
| """ | |
| from .mesh import create_cube_mesh | |
| from .transforms import perspective_from_fov, view_look_at | |
| vertices, faces = create_cube_mesh(tri=True) | |
| attributes = np.random.rand(len(vertices), 3).astype(np.float32) | |
| projection = perspective_from_fov(fov_x=np.deg2rad(60), aspect_ratio=1, near=1e-8, far=100000) | |
| view = view_look_at([1, 2, 2], [0, 0, 0], [0, 1, 0]) | |
| out = rasterize_triangles( | |
| (512, 512), | |
| vertices=vertices, | |
| attributes=attributes, | |
| faces=faces, | |
| view=view, | |
| projection=projection, | |
| cull_backface=True, | |
| return_depth=True, | |
| return_interpolation=False, | |
| ctx=ctx, | |
| ) | |
| image, mask = out['image'], out['mask'] | |
| image = np.concatenate([image, mask[:, :, None].astype(np.float32)], axis=-1) | |
| import cv2 | |
| cv2.imwrite('CHECKME.png', cv2.cvtColor((image.clip(0, 1) * 255).astype(np.uint8), cv2.COLOR_RGBA2BGRA)) | |
| print("An image has been saved as ./CHECKME.png") | |
| # def test_rasterization(ctx: RastContext): | |
| # """ | |
| # Test if rasterization works. It will render a cube with random colors and save it as a CHECKME.png file. | |
| # """ | |
| # from .mesh import cube | |
| # from .transforms import perspective_from_fov, view_look_at, project, unproject | |
| # vertices, faces = cube(tri=True) | |
| # attributes = np.random.rand(len(vertices), 3).astype(np.float32) | |
| # projection = perspective_from_fov(fov_x=np.deg2rad(60), aspect_ratio=1, near=0.1, far=100000) | |
| # view = view_look_at([1, 2, 2], [0, 0, 0], [0, 1, 0]) | |
| # for _ in range(20): | |
| # with timeit(): | |
| # out = rasterize_point_cloud( | |
| # ctx, | |
| # 512, 512, | |
| # points=vertices, | |
| # point_sizes=1, | |
| # point_size_in='3d', | |
| # point_shape='circle', | |
| # attributes=attributes, | |
| # view=view, | |
| # projection=projection, | |
| # return_depth=True, | |
| # return_point_id=True, | |
| # ) | |
| # image = out['image'] | |
| # print(np.unique(image)) | |
| # image = np.concatenate([image, np.zeros((*image.shape[:2], 4 - image.shape[2]), dtype=image.dtype)], axis=-1) | |
| # import cv2 | |
| # cv2.imwrite('CHECKME.png', cv2.cvtColor((image.clip(0, 1) * 255).astype(np.uint8), cv2.COLOR_RGB2BGR)) | |
| # print("An image has been saved as ./CHECKME.png") |