PanoGS / panogs /rendering /torch /rasterizer.py
SiddhuG01's picture
Deploy PanoGS Gradio proof of concept
42bed50
Raw History Blame Contribute Delete
10.6 kB
"""
PyTorch Differentiable 3D Gaussian Splatting Rasterizer.
Provides TorchGaussianModel with autograd parameter tensors and
differentiable perspective projection, covariance computation, and alpha compositing.
"""
from typing import Optional, Tuple
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from panogs.core.gaussian.model import GaussianModel, SH_C0, rgb_to_sh0, sh0_to_rgb
from panogs.rendering.camera import Camera
def quaternion_to_rotation_matrix_torch(quats: torch.Tensor) -> torch.Tensor:
"""
Convert (N, 4) unit quaternions (qw, qx, qy, qz) into (N, 3, 3) rotation matrices in PyTorch.
"""
q = F.normalize(quats, p=2, dim=-1, eps=1e-8)
r = q[:, 0]
x = q[:, 1]
y = q[:, 2]
z = q[:, 3]
N = quats.shape[0]
R = torch.zeros((N, 3, 3), dtype=quats.dtype, device=quats.device)
R[:, 0, 0] = 1.0 - 2.0 * (y * y + z * z)
R[:, 0, 1] = 2.0 * (x * y - r * z)
R[:, 0, 2] = 2.0 * (x * z + r * y)
R[:, 1, 0] = 2.0 * (x * y + r * z)
R[:, 1, 1] = 1.0 - 2.0 * (x * x + z * z)
R[:, 1, 2] = 2.0 * (y * z - r * x)
R[:, 2, 0] = 2.0 * (x * z - r * y)
R[:, 2, 1] = 2.0 * (y * z + r * x)
R[:, 2, 2] = 1.0 - 2.0 * (x * x + y * y)
return R
class TorchGaussianModel(nn.Module):
"""
Differentiable PyTorch container for 3D Gaussian Splats.
"""
def __init__(
self,
xyz: torch.Tensor,
scaling_log: torch.Tensor,
rotation_quats: torch.Tensor,
opacity_logits: torch.Tensor,
features_dc: torch.Tensor,
):
super().__init__()
self._xyz = nn.Parameter(xyz.float())
self._scaling_log = nn.Parameter(scaling_log.float())
self._rotation_quats = nn.Parameter(rotation_quats.float())
self._opacity_logits = nn.Parameter(opacity_logits.float())
self._features_dc = nn.Parameter(features_dc.float())
@property
def num_gaussians(self) -> int:
return self._xyz.shape[0]
@classmethod
def from_numpy(cls, model: GaussianModel, device: str = "cpu") -> "TorchGaussianModel":
"""Construct a TorchGaussianModel from a NumPy GaussianModel."""
return cls(
xyz=torch.tensor(model._xyz, device=device),
scaling_log=torch.tensor(model._scaling_log, device=device),
rotation_quats=torch.tensor(model._rotation_quats, device=device),
opacity_logits=torch.tensor(model._opacity_logits, device=device),
features_dc=torch.tensor(model._features_dc, device=device),
)
def to_numpy_model(self) -> GaussianModel:
"""Export current PyTorch parameters to a NumPy GaussianModel."""
with torch.no_grad():
return GaussianModel(
xyz=self._xyz.detach().cpu().numpy(),
scaling_log=self._scaling_log.detach().cpu().numpy(),
rotation_quats=self._rotation_quats.detach().cpu().numpy(),
opacity_logits=self._opacity_logits.detach().cpu().numpy(),
features_dc=self._features_dc.detach().cpu().numpy(),
)
def get_xyz(self) -> torch.Tensor:
return self._xyz
def get_scaling(self) -> torch.Tensor:
"""Standard deviation scales sigma = exp(s_log)."""
return torch.exp(self._scaling_log)
def get_rotation_quats(self) -> torch.Tensor:
return F.normalize(self._rotation_quats, p=2, dim=-1, eps=1e-8)
def get_rotation_matrices(self) -> torch.Tensor:
return quaternion_to_rotation_matrix_torch(self._rotation_quats)
def get_opacity(self) -> torch.Tensor:
"""Alpha opacity in (0, 1)."""
return torch.sigmoid(self._opacity_logits)
def get_rgb(self) -> torch.Tensor:
"""Nonnegative linear RGB radiance from SH0; HDR values may exceed 1."""
return torch.clamp(self._features_dc * SH_C0 + 0.5, min=0.0)
def get_covariance_3d(self) -> torch.Tensor:
"""
Compute (N, 3, 3) 3D covariance Sigma = R S S^T R^T.
"""
R = self.get_rotation_matrices() # (N, 3, 3)
scales = self.get_scaling() # (N, 3)
# M = R * S
M = R * scales.unsqueeze(1) # (N, 3, 3)
cov3d = torch.bmm(M, M.transpose(1, 2))
return cov3d
def render_gaussians_torch(
model: TorchGaussianModel,
camera: Camera,
bg_color: Tuple[float, float, float] = (1.0, 1.0, 1.0),
min_depth: float = 0.05,
low_pass_filter: float = 0.3,
) -> Tuple[torch.Tensor, dict]:
"""
Differentiable rendering of 3D Gaussians for a camera viewpoint using PyTorch.
Args:
model: TorchGaussianModel instance.
camera: Camera viewpoint.
bg_color: RGB background color.
min_depth: Near plane clipping.
low_pass_filter: Antialiasing constant.
Returns:
Tuple:
- rendered_image: (H, W, 3) float32 linear RGB radiance tensor (may exceed 1 for HDR).
- render_info: Dictionary containing 'screen_pts', 'vis_idx' for gradient tracking.
"""
device = model._xyz.device
dtype = model._xyz.dtype
H, W = camera.height, camera.width
xyz = model.get_xyz()
cov3d = model.get_covariance_3d()
colors = model.get_rgb()
opacities = model.get_opacity().squeeze(-1)
# 1. Transform positions to camera coordinate space
R_cam = torch.tensor(camera.R, device=device, dtype=dtype) # (3, 3)
t_cam = torch.tensor(camera.t, device=device, dtype=dtype) # (3,)
# p_cam = xyz @ R_cam^T + t_cam
p_cam = torch.matmul(xyz, R_cam.t()) + t_cam # (N, 3)
tx = p_cam[:, 0]
ty = p_cam[:, 1]
tz = p_cam[:, 2]
# 2. Frustum near plane filter
vis_mask = tz > min_depth
if not torch.any(vis_mask):
bg = torch.tensor(bg_color, device=device, dtype=dtype).view(1, 1, 3).expand(H, W, 3)
return bg, {"screen_pts": None, "vis_idx": None}
vis_idx = torch.nonzero(vis_mask, as_tuple=False).squeeze(-1)
tx_vis = tx[vis_idx]
ty_vis = ty[vis_idx]
tz_vis = tz[vis_idx]
cov3d_vis = cov3d[vis_idx]
colors_vis = colors[vis_idx]
opacities_vis = opacities[vis_idx]
fx = camera.fx
fy = camera.fy
cx = camera.cx
cy = camera.cy
# 3. Perspective projection
xs = (fx * tx_vis / tz_vis) + cx
ys = (fy * ty_vis / tz_vis) + cy
screen_pts = torch.stack([xs, ys], dim=-1) # (M, 2)
# Enable grad tracking on screen points for densification
if screen_pts.requires_grad:
screen_pts.retain_grad()
# 4. Transform 3D covariance to camera view frame: cov_cam = R_cam @ cov3d @ R_cam^T
cov3d_cam = torch.matmul(
R_cam.unsqueeze(0),
torch.matmul(cov3d_vis, R_cam.t().unsqueeze(0))
)
# 5. Jacobian of projection J: (M, 2, 3)
M = vis_idx.shape[0]
tz2 = tz_vis * tz_vis
J = torch.zeros((M, 2, 3), device=device, dtype=dtype)
J[:, 0, 0] = fx / tz_vis
J[:, 0, 2] = - (fx * tx_vis) / tz2
J[:, 1, 1] = fy / tz_vis
J[:, 1, 2] = - (fy * ty_vis) / tz2
# 6. 2D Covariance: Sigma_2D = J @ cov3d_cam @ J^T + low_pass * I
cov2d = torch.bmm(J, torch.bmm(cov3d_cam, J.transpose(1, 2)))
cov2d[:, 0, 0] = cov2d[:, 0, 0] + low_pass_filter
cov2d[:, 1, 1] = cov2d[:, 1, 1] + low_pass_filter
# Covariance components
a = cov2d[:, 0, 0]
b = cov2d[:, 0, 1]
c = cov2d[:, 1, 1]
det = a * c - b * b
valid_det = det > 1e-6
r_x = 3.0 * torch.sqrt(torch.clamp(a, min=1e-4))
r_y = 3.0 * torch.sqrt(torch.clamp(c, min=1e-4))
radii = torch.maximum(r_x, r_y)
# Filter invalid Gaussians
valid_mask = valid_det & (radii > 0.5)
if not torch.any(valid_mask):
bg = torch.tensor(bg_color, device=device, dtype=dtype).view(1, 1, 3).expand(H, W, 3)
return bg, {"screen_pts": screen_pts, "vis_idx": vis_idx}
# 7. Depth sorting (front-to-back)
sort_order = torch.argsort(tz_vis)
sort_order = sort_order[valid_mask[sort_order]]
sorted_screen_pts = screen_pts[sort_order]
sorted_a = a[sort_order]
sorted_b = b[sort_order]
sorted_c = c[sort_order]
sorted_det = det[sort_order]
sorted_radii = radii[sort_order]
sorted_colors = colors_vis[sort_order]
sorted_opacities = opacities_vis[sort_order]
# Initialize compositing buffers
acc_color = torch.zeros((H, W, 3), device=device, dtype=dtype)
transmittance = torch.ones((H, W), device=device, dtype=dtype)
num_to_render = sorted_screen_pts.shape[0]
# Compositing loop
for i in range(num_to_render):
cx_i = sorted_screen_pts[i, 0]
cy_i = sorted_screen_pts[i, 1]
rad_i = sorted_radii[i].item()
u_min = max(0, int(np.floor(cx_i.item() - rad_i)))
u_max = min(W - 1, int(np.ceil(cx_i.item() + rad_i)))
v_min = max(0, int(np.floor(cy_i.item() - rad_i)))
v_max = min(H - 1, int(np.ceil(cy_i.item() + rad_i)))
if u_min > u_max or v_min > v_max:
continue
if transmittance[v_min:v_max + 1, u_min:u_max + 1].max().item() < 1e-4:
continue
# Pixel coordinate grid
u_range = torch.arange(u_min, u_max + 1, device=device, dtype=dtype)
v_range = torch.arange(v_min, v_max + 1, device=device, dtype=dtype)
v_grid, u_grid = torch.meshgrid(v_range, u_range, indexing="ij")
dx = u_grid - cx_i
dy = v_grid - cy_i
# Quadratic form: Delta^T * Sigma_2D^-1 * Delta
det_i = sorted_det[i]
inv_a = sorted_a[i]
inv_b = sorted_b[i]
inv_c = sorted_c[i]
power = -0.5 * (inv_c * dx * dx - 2.0 * inv_b * dx * dy + inv_a * dy * dy) / det_i
power_clamped = torch.clamp(power, max=0.0)
G = torch.exp(power_clamped)
alpha_box = sorted_opacities[i] * G
# Pad alpha_box to full image (H, W) out-of-place for autograd integrity
pad_left = u_min
pad_right = W - 1 - u_max
pad_top = v_min
pad_bottom = H - 1 - v_max
alpha_full = F.pad(alpha_box, (pad_left, pad_right, pad_top, pad_bottom), mode="constant", value=0.0)
# Out-of-place accumulation
w = alpha_full * transmittance
color_i = sorted_colors[i] # (3,)
acc_color = acc_color + w.unsqueeze(-1) * color_i.view(1, 1, 3)
transmittance = transmittance * (1.0 - alpha_full)
# Composite background
bg = torch.tensor(bg_color, device=device, dtype=dtype).view(1, 1, 3)
final_image = acc_color + transmittance.unsqueeze(-1) * bg
render_info = {
"screen_pts": screen_pts,
"vis_idx": vis_idx,
}
return final_image, render_info