Download panogs/rendering/torch/rasterizer.py from SiddhuG01/PanoGS: direct link, hf CLI and curl.
- Browser
- Download file 10.6 kB
-
https://huggingface.co/spaces/SiddhuG01/PanoGS/resolve/main/panogs/rendering/torch/rasterizer.py
- Command line
-
hf download hf://spaces/SiddhuG01/PanoGS/panogs/rendering/torch/rasterizer.py
-
curl -L -o rasterizer.py https://huggingface.co/spaces/SiddhuG01/PanoGS/resolve/main/panogs/rendering/torch/rasterizer.py
10.6 kB
| """ | |
| PyTorch Differentiable 3D Gaussian Splatting Rasterizer. | |
| Provides TorchGaussianModel with autograd parameter tensors and | |
| differentiable perspective projection, covariance computation, and alpha compositing. | |
| """ | |
| from typing import Optional, Tuple | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| import numpy as np | |
| from panogs.core.gaussian.model import GaussianModel, SH_C0, rgb_to_sh0, sh0_to_rgb | |
| from panogs.rendering.camera import Camera | |
| def quaternion_to_rotation_matrix_torch(quats: torch.Tensor) -> torch.Tensor: | |
| """ | |
| Convert (N, 4) unit quaternions (qw, qx, qy, qz) into (N, 3, 3) rotation matrices in PyTorch. | |
| """ | |
| q = F.normalize(quats, p=2, dim=-1, eps=1e-8) | |
| r = q[:, 0] | |
| x = q[:, 1] | |
| y = q[:, 2] | |
| z = q[:, 3] | |
| N = quats.shape[0] | |
| R = torch.zeros((N, 3, 3), dtype=quats.dtype, device=quats.device) | |
| R[:, 0, 0] = 1.0 - 2.0 * (y * y + z * z) | |
| R[:, 0, 1] = 2.0 * (x * y - r * z) | |
| R[:, 0, 2] = 2.0 * (x * z + r * y) | |
| R[:, 1, 0] = 2.0 * (x * y + r * z) | |
| R[:, 1, 1] = 1.0 - 2.0 * (x * x + z * z) | |
| R[:, 1, 2] = 2.0 * (y * z - r * x) | |
| R[:, 2, 0] = 2.0 * (x * z - r * y) | |
| R[:, 2, 1] = 2.0 * (y * z + r * x) | |
| R[:, 2, 2] = 1.0 - 2.0 * (x * x + y * y) | |
| return R | |
| class TorchGaussianModel(nn.Module): | |
| """ | |
| Differentiable PyTorch container for 3D Gaussian Splats. | |
| """ | |
| def __init__( | |
| self, | |
| xyz: torch.Tensor, | |
| scaling_log: torch.Tensor, | |
| rotation_quats: torch.Tensor, | |
| opacity_logits: torch.Tensor, | |
| features_dc: torch.Tensor, | |
| ): | |
| super().__init__() | |
| self._xyz = nn.Parameter(xyz.float()) | |
| self._scaling_log = nn.Parameter(scaling_log.float()) | |
| self._rotation_quats = nn.Parameter(rotation_quats.float()) | |
| self._opacity_logits = nn.Parameter(opacity_logits.float()) | |
| self._features_dc = nn.Parameter(features_dc.float()) | |
| def num_gaussians(self) -> int: | |
| return self._xyz.shape[0] | |
| def from_numpy(cls, model: GaussianModel, device: str = "cpu") -> "TorchGaussianModel": | |
| """Construct a TorchGaussianModel from a NumPy GaussianModel.""" | |
| return cls( | |
| xyz=torch.tensor(model._xyz, device=device), | |
| scaling_log=torch.tensor(model._scaling_log, device=device), | |
| rotation_quats=torch.tensor(model._rotation_quats, device=device), | |
| opacity_logits=torch.tensor(model._opacity_logits, device=device), | |
| features_dc=torch.tensor(model._features_dc, device=device), | |
| ) | |
| def to_numpy_model(self) -> GaussianModel: | |
| """Export current PyTorch parameters to a NumPy GaussianModel.""" | |
| with torch.no_grad(): | |
| return GaussianModel( | |
| xyz=self._xyz.detach().cpu().numpy(), | |
| scaling_log=self._scaling_log.detach().cpu().numpy(), | |
| rotation_quats=self._rotation_quats.detach().cpu().numpy(), | |
| opacity_logits=self._opacity_logits.detach().cpu().numpy(), | |
| features_dc=self._features_dc.detach().cpu().numpy(), | |
| ) | |
| def get_xyz(self) -> torch.Tensor: | |
| return self._xyz | |
| def get_scaling(self) -> torch.Tensor: | |
| """Standard deviation scales sigma = exp(s_log).""" | |
| return torch.exp(self._scaling_log) | |
| def get_rotation_quats(self) -> torch.Tensor: | |
| return F.normalize(self._rotation_quats, p=2, dim=-1, eps=1e-8) | |
| def get_rotation_matrices(self) -> torch.Tensor: | |
| return quaternion_to_rotation_matrix_torch(self._rotation_quats) | |
| def get_opacity(self) -> torch.Tensor: | |
| """Alpha opacity in (0, 1).""" | |
| return torch.sigmoid(self._opacity_logits) | |
| def get_rgb(self) -> torch.Tensor: | |
| """Nonnegative linear RGB radiance from SH0; HDR values may exceed 1.""" | |
| return torch.clamp(self._features_dc * SH_C0 + 0.5, min=0.0) | |
| def get_covariance_3d(self) -> torch.Tensor: | |
| """ | |
| Compute (N, 3, 3) 3D covariance Sigma = R S S^T R^T. | |
| """ | |
| R = self.get_rotation_matrices() # (N, 3, 3) | |
| scales = self.get_scaling() # (N, 3) | |
| # M = R * S | |
| M = R * scales.unsqueeze(1) # (N, 3, 3) | |
| cov3d = torch.bmm(M, M.transpose(1, 2)) | |
| return cov3d | |
| def render_gaussians_torch( | |
| model: TorchGaussianModel, | |
| camera: Camera, | |
| bg_color: Tuple[float, float, float] = (1.0, 1.0, 1.0), | |
| min_depth: float = 0.05, | |
| low_pass_filter: float = 0.3, | |
| ) -> Tuple[torch.Tensor, dict]: | |
| """ | |
| Differentiable rendering of 3D Gaussians for a camera viewpoint using PyTorch. | |
| Args: | |
| model: TorchGaussianModel instance. | |
| camera: Camera viewpoint. | |
| bg_color: RGB background color. | |
| min_depth: Near plane clipping. | |
| low_pass_filter: Antialiasing constant. | |
| Returns: | |
| Tuple: | |
| - rendered_image: (H, W, 3) float32 linear RGB radiance tensor (may exceed 1 for HDR). | |
| - render_info: Dictionary containing 'screen_pts', 'vis_idx' for gradient tracking. | |
| """ | |
| device = model._xyz.device | |
| dtype = model._xyz.dtype | |
| H, W = camera.height, camera.width | |
| xyz = model.get_xyz() | |
| cov3d = model.get_covariance_3d() | |
| colors = model.get_rgb() | |
| opacities = model.get_opacity().squeeze(-1) | |
| # 1. Transform positions to camera coordinate space | |
| R_cam = torch.tensor(camera.R, device=device, dtype=dtype) # (3, 3) | |
| t_cam = torch.tensor(camera.t, device=device, dtype=dtype) # (3,) | |
| # p_cam = xyz @ R_cam^T + t_cam | |
| p_cam = torch.matmul(xyz, R_cam.t()) + t_cam # (N, 3) | |
| tx = p_cam[:, 0] | |
| ty = p_cam[:, 1] | |
| tz = p_cam[:, 2] | |
| # 2. Frustum near plane filter | |
| vis_mask = tz > min_depth | |
| if not torch.any(vis_mask): | |
| bg = torch.tensor(bg_color, device=device, dtype=dtype).view(1, 1, 3).expand(H, W, 3) | |
| return bg, {"screen_pts": None, "vis_idx": None} | |
| vis_idx = torch.nonzero(vis_mask, as_tuple=False).squeeze(-1) | |
| tx_vis = tx[vis_idx] | |
| ty_vis = ty[vis_idx] | |
| tz_vis = tz[vis_idx] | |
| cov3d_vis = cov3d[vis_idx] | |
| colors_vis = colors[vis_idx] | |
| opacities_vis = opacities[vis_idx] | |
| fx = camera.fx | |
| fy = camera.fy | |
| cx = camera.cx | |
| cy = camera.cy | |
| # 3. Perspective projection | |
| xs = (fx * tx_vis / tz_vis) + cx | |
| ys = (fy * ty_vis / tz_vis) + cy | |
| screen_pts = torch.stack([xs, ys], dim=-1) # (M, 2) | |
| # Enable grad tracking on screen points for densification | |
| if screen_pts.requires_grad: | |
| screen_pts.retain_grad() | |
| # 4. Transform 3D covariance to camera view frame: cov_cam = R_cam @ cov3d @ R_cam^T | |
| cov3d_cam = torch.matmul( | |
| R_cam.unsqueeze(0), | |
| torch.matmul(cov3d_vis, R_cam.t().unsqueeze(0)) | |
| ) | |
| # 5. Jacobian of projection J: (M, 2, 3) | |
| M = vis_idx.shape[0] | |
| tz2 = tz_vis * tz_vis | |
| J = torch.zeros((M, 2, 3), device=device, dtype=dtype) | |
| J[:, 0, 0] = fx / tz_vis | |
| J[:, 0, 2] = - (fx * tx_vis) / tz2 | |
| J[:, 1, 1] = fy / tz_vis | |
| J[:, 1, 2] = - (fy * ty_vis) / tz2 | |
| # 6. 2D Covariance: Sigma_2D = J @ cov3d_cam @ J^T + low_pass * I | |
| cov2d = torch.bmm(J, torch.bmm(cov3d_cam, J.transpose(1, 2))) | |
| cov2d[:, 0, 0] = cov2d[:, 0, 0] + low_pass_filter | |
| cov2d[:, 1, 1] = cov2d[:, 1, 1] + low_pass_filter | |
| # Covariance components | |
| a = cov2d[:, 0, 0] | |
| b = cov2d[:, 0, 1] | |
| c = cov2d[:, 1, 1] | |
| det = a * c - b * b | |
| valid_det = det > 1e-6 | |
| r_x = 3.0 * torch.sqrt(torch.clamp(a, min=1e-4)) | |
| r_y = 3.0 * torch.sqrt(torch.clamp(c, min=1e-4)) | |
| radii = torch.maximum(r_x, r_y) | |
| # Filter invalid Gaussians | |
| valid_mask = valid_det & (radii > 0.5) | |
| if not torch.any(valid_mask): | |
| bg = torch.tensor(bg_color, device=device, dtype=dtype).view(1, 1, 3).expand(H, W, 3) | |
| return bg, {"screen_pts": screen_pts, "vis_idx": vis_idx} | |
| # 7. Depth sorting (front-to-back) | |
| sort_order = torch.argsort(tz_vis) | |
| sort_order = sort_order[valid_mask[sort_order]] | |
| sorted_screen_pts = screen_pts[sort_order] | |
| sorted_a = a[sort_order] | |
| sorted_b = b[sort_order] | |
| sorted_c = c[sort_order] | |
| sorted_det = det[sort_order] | |
| sorted_radii = radii[sort_order] | |
| sorted_colors = colors_vis[sort_order] | |
| sorted_opacities = opacities_vis[sort_order] | |
| # Initialize compositing buffers | |
| acc_color = torch.zeros((H, W, 3), device=device, dtype=dtype) | |
| transmittance = torch.ones((H, W), device=device, dtype=dtype) | |
| num_to_render = sorted_screen_pts.shape[0] | |
| # Compositing loop | |
| for i in range(num_to_render): | |
| cx_i = sorted_screen_pts[i, 0] | |
| cy_i = sorted_screen_pts[i, 1] | |
| rad_i = sorted_radii[i].item() | |
| u_min = max(0, int(np.floor(cx_i.item() - rad_i))) | |
| u_max = min(W - 1, int(np.ceil(cx_i.item() + rad_i))) | |
| v_min = max(0, int(np.floor(cy_i.item() - rad_i))) | |
| v_max = min(H - 1, int(np.ceil(cy_i.item() + rad_i))) | |
| if u_min > u_max or v_min > v_max: | |
| continue | |
| if transmittance[v_min:v_max + 1, u_min:u_max + 1].max().item() < 1e-4: | |
| continue | |
| # Pixel coordinate grid | |
| u_range = torch.arange(u_min, u_max + 1, device=device, dtype=dtype) | |
| v_range = torch.arange(v_min, v_max + 1, device=device, dtype=dtype) | |
| v_grid, u_grid = torch.meshgrid(v_range, u_range, indexing="ij") | |
| dx = u_grid - cx_i | |
| dy = v_grid - cy_i | |
| # Quadratic form: Delta^T * Sigma_2D^-1 * Delta | |
| det_i = sorted_det[i] | |
| inv_a = sorted_a[i] | |
| inv_b = sorted_b[i] | |
| inv_c = sorted_c[i] | |
| power = -0.5 * (inv_c * dx * dx - 2.0 * inv_b * dx * dy + inv_a * dy * dy) / det_i | |
| power_clamped = torch.clamp(power, max=0.0) | |
| G = torch.exp(power_clamped) | |
| alpha_box = sorted_opacities[i] * G | |
| # Pad alpha_box to full image (H, W) out-of-place for autograd integrity | |
| pad_left = u_min | |
| pad_right = W - 1 - u_max | |
| pad_top = v_min | |
| pad_bottom = H - 1 - v_max | |
| alpha_full = F.pad(alpha_box, (pad_left, pad_right, pad_top, pad_bottom), mode="constant", value=0.0) | |
| # Out-of-place accumulation | |
| w = alpha_full * transmittance | |
| color_i = sorted_colors[i] # (3,) | |
| acc_color = acc_color + w.unsqueeze(-1) * color_i.view(1, 1, 3) | |
| transmittance = transmittance * (1.0 - alpha_full) | |
| # Composite background | |
| bg = torch.tensor(bg_color, device=device, dtype=dtype).view(1, 1, 3) | |
| final_image = acc_color + transmittance.unsqueeze(-1) * bg | |
| render_info = { | |
| "screen_pts": screen_pts, | |
| "vis_idx": vis_idx, | |
| } | |
| return final_image, render_info | |