Spaces:
Running on Zero
Running on Zero
Initial commit for brand new AnySplat ZeroGPU Space (part 5)
Browse files- src/model/encoder/visualization/encoder_visualizer_epipolar_cfg.py +10 -0
- src/model/encodings/positional_encoding.py +36 -0
- src/model/model/__init__.py +21 -0
- src/model/model/anysplat.py +126 -0
- src/model/model_wrapper.py +847 -0
- src/model/ply_export.py +74 -0
- src/model/transformer/attention.py +70 -0
- src/model/transformer/feed_forward.py +40 -0
- src/model/transformer/pre_norm.py +35 -0
- src/model/transformer/transformer.py +71 -0
- src/model/types.py +15 -0
- src/post_opt/datasets/colmap.py +490 -0
- src/post_opt/datasets/normalize.py +143 -0
- src/post_opt/datasets/traj.py +254 -0
- src/post_opt/exporter.py +553 -0
- src/post_opt/gsplat_viewer.py +249 -0
- src/post_opt/image_fitting.py +189 -0
- src/post_opt/lib_bilagrid.py +573 -0
- src/post_opt/simple_trainer.py +1433 -0
- src/post_opt/simple_viewer.py +275 -0
- src/post_opt/utils.py +224 -0
- src/utils/__init.py +0 -0
- src/utils/ba.py +407 -0
- src/utils/cropping.py +163 -0
- src/utils/device.py +95 -0
- src/utils/geometry.py +428 -0
- src/utils/image.py +185 -0
- src/utils/misc.py +531 -0
- src/utils/point.py +47 -0
- src/utils/pose.py +218 -0
- src/utils/render.py +284 -0
- src/utils/tensor_to_pycolmap.py +320 -0
- src/utils/transforms.py +18 -0
- src/utils/viz.py +377 -0
- src/visualization/annotation.py +49 -0
- src/visualization/camera_trajectory/interpolation.py +255 -0
- src/visualization/camera_trajectory/spin.py +37 -0
- src/visualization/camera_trajectory/wobble.py +32 -0
- src/visualization/color_map.py +48 -0
- src/visualization/colors.py +32 -0
- src/visualization/drawing/cameras.py +195 -0
- src/visualization/drawing/coordinate_conversion.py +44 -0
- src/visualization/drawing/lines.py +83 -0
- src/visualization/drawing/points.py +59 -0
- src/visualization/drawing/rendering.py +152 -0
- src/visualization/drawing/types.py +67 -0
- src/visualization/layout.py +228 -0
- src/visualization/validation_in_3d.py +115 -0
- src/visualization/video_render.py +162 -0
src/model/encoder/visualization/encoder_visualizer_epipolar_cfg.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass
|
| 2 |
+
|
| 3 |
+
# This is in a separate file to avoid circular imports.
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
@dataclass
|
| 7 |
+
class EncoderVisualizerEpipolarCfg:
|
| 8 |
+
num_samples: int
|
| 9 |
+
min_resolution: int
|
| 10 |
+
export_ply: bool
|
src/model/encodings/positional_encoding.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
from einops import einsum, rearrange, repeat
|
| 4 |
+
from jaxtyping import Float
|
| 5 |
+
from torch import Tensor
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class PositionalEncoding(nn.Module):
|
| 9 |
+
"""For the sake of simplicity, this encodes values in the range [0, 1]."""
|
| 10 |
+
|
| 11 |
+
frequencies: Float[Tensor, "frequency phase"]
|
| 12 |
+
phases: Float[Tensor, "frequency phase"]
|
| 13 |
+
|
| 14 |
+
def __init__(self, num_octaves: int):
|
| 15 |
+
super().__init__()
|
| 16 |
+
octaves = torch.arange(num_octaves).float()
|
| 17 |
+
|
| 18 |
+
# The lowest frequency has a period of 1.
|
| 19 |
+
frequencies = 2 * torch.pi * 2**octaves
|
| 20 |
+
frequencies = repeat(frequencies, "f -> f p", p=2)
|
| 21 |
+
self.register_buffer("frequencies", frequencies, persistent=False)
|
| 22 |
+
|
| 23 |
+
# Choose the phases to match sine and cosine.
|
| 24 |
+
phases = torch.tensor([0, 0.5 * torch.pi], dtype=torch.float32)
|
| 25 |
+
phases = repeat(phases, "p -> f p", f=num_octaves)
|
| 26 |
+
self.register_buffer("phases", phases, persistent=False)
|
| 27 |
+
|
| 28 |
+
def forward(
|
| 29 |
+
self,
|
| 30 |
+
samples: Float[Tensor, "*batch dim"],
|
| 31 |
+
) -> Float[Tensor, "*batch embedded_dim"]:
|
| 32 |
+
samples = einsum(samples, self.frequencies, "... d, f p -> ... d f p")
|
| 33 |
+
return rearrange(torch.sin(samples + self.phases), "... d f p -> ... (d f p)")
|
| 34 |
+
|
| 35 |
+
def d_out(self, dimensionality: int):
|
| 36 |
+
return self.frequencies.numel() * dimensionality
|
src/model/model/__init__.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional, Union
|
| 2 |
+
|
| 3 |
+
from ..encoder import Encoder
|
| 4 |
+
from ..encoder.visualization.encoder_visualizer import EncoderVisualizer
|
| 5 |
+
from ..encoder.anysplat import EncoderAnySplat, EncoderAnySplatCfg
|
| 6 |
+
from ..decoder.decoder_splatting_cuda import DecoderSplattingCUDACfg
|
| 7 |
+
from torch import nn
|
| 8 |
+
from .anysplat import AnySplat
|
| 9 |
+
|
| 10 |
+
MODELS = {
|
| 11 |
+
"anysplat": AnySplat,
|
| 12 |
+
}
|
| 13 |
+
|
| 14 |
+
EncoderCfg = Union[EncoderAnySplatCfg]
|
| 15 |
+
DecoderCfg = DecoderSplattingCUDACfg
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
# hard code for now
|
| 19 |
+
def get_model(encoder_cfg: EncoderCfg, decoder_cfg: DecoderCfg) -> nn.Module:
|
| 20 |
+
model = MODELS['anysplat'](encoder_cfg, decoder_cfg)
|
| 21 |
+
return model
|
src/model/model/anysplat.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from copy import deepcopy
|
| 3 |
+
import time
|
| 4 |
+
from typing import Optional
|
| 5 |
+
from einops import rearrange
|
| 6 |
+
import huggingface_hub
|
| 7 |
+
from omegaconf import DictConfig, OmegaConf
|
| 8 |
+
import torch
|
| 9 |
+
import torch.distributed
|
| 10 |
+
import torch.nn as nn
|
| 11 |
+
import numpy as np
|
| 12 |
+
import torch.nn.functional as F
|
| 13 |
+
from dataclasses import dataclass
|
| 14 |
+
|
| 15 |
+
from src.model.encoder.common.gaussian_adapter import GaussianAdapterCfg
|
| 16 |
+
from src.model.decoder.decoder_splatting_cuda import DecoderSplattingCUDA, DecoderSplattingCUDACfg
|
| 17 |
+
from src.model.encoder.anysplat import EncoderAnySplat, EncoderAnySplatCfg, OpacityMappingCfg
|
| 18 |
+
|
| 19 |
+
class AnySplat(nn.Module, huggingface_hub.PyTorchModelHubMixin):
|
| 20 |
+
def __init__(
|
| 21 |
+
self,
|
| 22 |
+
encoder_cfg: EncoderAnySplatCfg,
|
| 23 |
+
decoder_cfg: DecoderSplattingCUDACfg,
|
| 24 |
+
):
|
| 25 |
+
super(AnySplat, self).__init__()
|
| 26 |
+
self.encoder_cfg = encoder_cfg
|
| 27 |
+
self.decoder_cfg = decoder_cfg
|
| 28 |
+
self.build_encoder(encoder_cfg)
|
| 29 |
+
self.build_decoder(decoder_cfg)
|
| 30 |
+
|
| 31 |
+
def convert_nested_config(self, cfg_dict: dict, target_class: type):
|
| 32 |
+
"""Convert nested dictionary config to dataclass instance
|
| 33 |
+
|
| 34 |
+
Args:
|
| 35 |
+
cfg_dict: Configuration dictionary or already converted object
|
| 36 |
+
target_class: Target dataclass type to convert to
|
| 37 |
+
|
| 38 |
+
Returns:
|
| 39 |
+
Instance of target_class
|
| 40 |
+
"""
|
| 41 |
+
if isinstance(cfg_dict, dict):
|
| 42 |
+
# Convert dict to dataclass
|
| 43 |
+
return target_class(**cfg_dict)
|
| 44 |
+
elif isinstance(cfg_dict, target_class):
|
| 45 |
+
# Already converted, return as is
|
| 46 |
+
return cfg_dict
|
| 47 |
+
elif cfg_dict is None:
|
| 48 |
+
# Handle None case
|
| 49 |
+
return None
|
| 50 |
+
else:
|
| 51 |
+
raise ValueError(f"Cannot convert {type(cfg_dict)} to {target_class}")
|
| 52 |
+
|
| 53 |
+
def convert_config_recursively(self, cfg_obj, conversion_map: dict):
|
| 54 |
+
"""Convert nested configurations recursively using a conversion map
|
| 55 |
+
|
| 56 |
+
Args:
|
| 57 |
+
cfg_obj: Configuration object to convert
|
| 58 |
+
conversion_map: Dict mapping field names to their target classes
|
| 59 |
+
e.g., {'gaussian_adapter': GaussianAdapterCfg}
|
| 60 |
+
|
| 61 |
+
Returns:
|
| 62 |
+
Converted configuration object
|
| 63 |
+
"""
|
| 64 |
+
if not hasattr(cfg_obj, '__dict__'):
|
| 65 |
+
return cfg_obj
|
| 66 |
+
|
| 67 |
+
cfg_dict = cfg_obj.__dict__.copy()
|
| 68 |
+
|
| 69 |
+
for field_name, target_class in conversion_map.items():
|
| 70 |
+
if field_name in cfg_dict:
|
| 71 |
+
cfg_dict[field_name] = self.convert_nested_config(
|
| 72 |
+
cfg_dict[field_name],
|
| 73 |
+
target_class
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
# Return new instance of the same type
|
| 77 |
+
return type(cfg_obj)(**cfg_dict)
|
| 78 |
+
|
| 79 |
+
def convert_encoder_config(self, encoder_cfg: EncoderAnySplatCfg) -> EncoderAnySplatCfg:
|
| 80 |
+
"""Convert all nested configurations in encoder_cfg"""
|
| 81 |
+
conversion_map = {
|
| 82 |
+
'gaussian_adapter': GaussianAdapterCfg,
|
| 83 |
+
'opacity_mapping': OpacityMappingCfg,
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
return self.convert_config_recursively(encoder_cfg, conversion_map)
|
| 87 |
+
|
| 88 |
+
def build_encoder(self, encoder_cfg: EncoderAnySplatCfg):
|
| 89 |
+
# Convert nested configurations using the helper method
|
| 90 |
+
encoder_cfg = self.convert_encoder_config(encoder_cfg)
|
| 91 |
+
self.encoder = EncoderAnySplat(encoder_cfg)
|
| 92 |
+
|
| 93 |
+
def build_decoder(self, decoder_cfg: DecoderSplattingCUDACfg):
|
| 94 |
+
self.decoder = DecoderSplattingCUDA(decoder_cfg)
|
| 95 |
+
|
| 96 |
+
@torch.no_grad()
|
| 97 |
+
def inference(self,
|
| 98 |
+
context_image: torch.Tensor,
|
| 99 |
+
):
|
| 100 |
+
self.encoder.distill = False
|
| 101 |
+
encoder_output = self.encoder(context_image, global_step=0, visualization_dump=None)
|
| 102 |
+
gaussians, pred_context_pose = encoder_output.gaussians, encoder_output.pred_context_pose
|
| 103 |
+
return gaussians, pred_context_pose
|
| 104 |
+
|
| 105 |
+
def forward(self,
|
| 106 |
+
context_image: torch.Tensor,
|
| 107 |
+
global_step: int = 0,
|
| 108 |
+
visualization_dump: Optional[dict] = None,
|
| 109 |
+
near: float = 0.01,
|
| 110 |
+
far: float = 100.0,
|
| 111 |
+
):
|
| 112 |
+
b, v, c, h, w = context_image.shape
|
| 113 |
+
device = context_image.device
|
| 114 |
+
encoder_output = self.encoder(context_image, global_step, visualization_dump=visualization_dump)
|
| 115 |
+
gaussians, pred_context_pose = encoder_output.gaussians, encoder_output.pred_context_pose
|
| 116 |
+
output = self.decoder.forward(
|
| 117 |
+
gaussians,
|
| 118 |
+
pred_context_pose['extrinsic'],
|
| 119 |
+
pred_context_pose["intrinsic"],
|
| 120 |
+
torch.ones(1, v, device=device) * near,
|
| 121 |
+
torch.ones(1, v, device=device) * far,
|
| 122 |
+
(h, w),
|
| 123 |
+
"depth",
|
| 124 |
+
)
|
| 125 |
+
return encoder_output, output
|
| 126 |
+
|
src/model/model_wrapper.py
ADDED
|
@@ -0,0 +1,847 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
import gc
|
| 4 |
+
import random
|
| 5 |
+
from typing import Literal, Optional, Protocol, runtime_checkable, Any
|
| 6 |
+
|
| 7 |
+
import moviepy.editor as mpy
|
| 8 |
+
import torch
|
| 9 |
+
import torchvision
|
| 10 |
+
import wandb
|
| 11 |
+
from einops import pack, rearrange, repeat
|
| 12 |
+
from jaxtyping import Float
|
| 13 |
+
from lightning.pytorch import LightningModule
|
| 14 |
+
from lightning.pytorch.loggers.wandb import WandbLogger
|
| 15 |
+
from lightning.pytorch.utilities import rank_zero_only
|
| 16 |
+
from tabulate import tabulate
|
| 17 |
+
from torch import Tensor, nn, optim
|
| 18 |
+
import torch.nn.functional as F
|
| 19 |
+
|
| 20 |
+
from loss.loss_lpips import LossLpips
|
| 21 |
+
from loss.loss_mse import LossMse
|
| 22 |
+
from model.encoder.vggt.utils.pose_enc import pose_encoding_to_extri_intri
|
| 23 |
+
|
| 24 |
+
from ..loss.loss_distill import DistillLoss
|
| 25 |
+
from src.utils.render import generate_path
|
| 26 |
+
from src.utils.point import get_normal_map
|
| 27 |
+
|
| 28 |
+
from ..loss.loss_huber import HuberLoss, extri_intri_to_pose_encoding
|
| 29 |
+
|
| 30 |
+
# from model.types import Gaussians
|
| 31 |
+
|
| 32 |
+
from ..dataset.data_module import get_data_shim
|
| 33 |
+
from ..dataset.types import BatchedExample
|
| 34 |
+
from ..evaluation.metrics import compute_lpips, compute_psnr, compute_ssim, abs_relative_difference, delta1_acc
|
| 35 |
+
from ..global_cfg import get_cfg
|
| 36 |
+
from ..loss import Loss
|
| 37 |
+
from ..loss.loss_point import Regr3D
|
| 38 |
+
from ..loss.loss_ssim import ssim
|
| 39 |
+
from ..misc.benchmarker import Benchmarker
|
| 40 |
+
from ..misc.cam_utils import update_pose, get_pnp_pose, rotation_6d_to_matrix
|
| 41 |
+
from ..misc.image_io import prep_image, save_image, save_video
|
| 42 |
+
from ..misc.LocalLogger import LOG_PATH, LocalLogger
|
| 43 |
+
from ..misc.nn_module_tools import convert_to_buffer
|
| 44 |
+
from ..misc.step_tracker import StepTracker
|
| 45 |
+
from ..misc.utils import inverse_normalize, vis_depth_map, confidence_map, get_overlap_tag
|
| 46 |
+
from ..visualization.annotation import add_label
|
| 47 |
+
from ..visualization.camera_trajectory.interpolation import (
|
| 48 |
+
interpolate_extrinsics,
|
| 49 |
+
interpolate_intrinsics,
|
| 50 |
+
)
|
| 51 |
+
from ..visualization.camera_trajectory.wobble import (
|
| 52 |
+
generate_wobble,
|
| 53 |
+
generate_wobble_transformation,
|
| 54 |
+
)
|
| 55 |
+
from ..visualization.color_map import apply_color_map_to_image
|
| 56 |
+
from ..visualization.layout import add_border, hcat, vcat
|
| 57 |
+
# from ..visualization.validation_in_3d import render_cameras, render_projections
|
| 58 |
+
from .decoder.decoder import Decoder, DepthRenderingMode
|
| 59 |
+
from .encoder import Encoder
|
| 60 |
+
from .encoder.visualization.encoder_visualizer import EncoderVisualizer
|
| 61 |
+
from .ply_export import export_ply
|
| 62 |
+
|
| 63 |
+
@dataclass
|
| 64 |
+
class OptimizerCfg:
|
| 65 |
+
lr: float
|
| 66 |
+
warm_up_steps: int
|
| 67 |
+
backbone_lr_multiplier: float
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
@dataclass
|
| 71 |
+
class TestCfg:
|
| 72 |
+
output_path: Path
|
| 73 |
+
align_pose: bool
|
| 74 |
+
pose_align_steps: int
|
| 75 |
+
rot_opt_lr: float
|
| 76 |
+
trans_opt_lr: float
|
| 77 |
+
compute_scores: bool
|
| 78 |
+
save_image: bool
|
| 79 |
+
save_video: bool
|
| 80 |
+
save_compare: bool
|
| 81 |
+
generate_video: bool
|
| 82 |
+
mode: Literal["inference", "evaluation"]
|
| 83 |
+
image_folder: str
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
@dataclass
|
| 87 |
+
class TrainCfg:
|
| 88 |
+
output_path: Path
|
| 89 |
+
depth_mode: DepthRenderingMode | None
|
| 90 |
+
extended_visualization: bool
|
| 91 |
+
print_log_every_n_steps: int
|
| 92 |
+
distiller: str
|
| 93 |
+
distill_max_steps: int
|
| 94 |
+
pose_loss_alpha: float = 1.0
|
| 95 |
+
pose_loss_delta: float = 1.0
|
| 96 |
+
cxt_depth_weight: float = 0.01
|
| 97 |
+
weight_pose: float = 1.0
|
| 98 |
+
weight_depth: float = 1.0
|
| 99 |
+
weight_normal: float = 1.0
|
| 100 |
+
render_ba: bool = False
|
| 101 |
+
render_ba_after_step: int = 0
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
@runtime_checkable
|
| 105 |
+
class TrajectoryFn(Protocol):
|
| 106 |
+
def __call__(
|
| 107 |
+
self,
|
| 108 |
+
t: Float[Tensor, " t"],
|
| 109 |
+
) -> tuple[
|
| 110 |
+
Float[Tensor, "batch view 4 4"], # extrinsics
|
| 111 |
+
Float[Tensor, "batch view 3 3"], # intrinsics
|
| 112 |
+
]:
|
| 113 |
+
pass
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
class ModelWrapper(LightningModule):
|
| 117 |
+
logger: Optional[WandbLogger]
|
| 118 |
+
model: nn.Module
|
| 119 |
+
losses: nn.ModuleList
|
| 120 |
+
optimizer_cfg: OptimizerCfg
|
| 121 |
+
test_cfg: TestCfg
|
| 122 |
+
train_cfg: TrainCfg
|
| 123 |
+
step_tracker: StepTracker | None
|
| 124 |
+
|
| 125 |
+
def __init__(
|
| 126 |
+
self,
|
| 127 |
+
optimizer_cfg: OptimizerCfg,
|
| 128 |
+
test_cfg: TestCfg,
|
| 129 |
+
train_cfg: TrainCfg,
|
| 130 |
+
model: nn.Module,
|
| 131 |
+
losses: list[Loss],
|
| 132 |
+
step_tracker: StepTracker | None
|
| 133 |
+
) -> None:
|
| 134 |
+
super().__init__()
|
| 135 |
+
self.optimizer_cfg = optimizer_cfg
|
| 136 |
+
self.test_cfg = test_cfg
|
| 137 |
+
self.train_cfg = train_cfg
|
| 138 |
+
self.step_tracker = step_tracker
|
| 139 |
+
|
| 140 |
+
# Set up the model.
|
| 141 |
+
self.encoder_visualizer = None
|
| 142 |
+
self.model = model
|
| 143 |
+
self.data_shim = get_data_shim(self.model.encoder)
|
| 144 |
+
self.losses = nn.ModuleList(losses)
|
| 145 |
+
|
| 146 |
+
if self.model.encoder.pred_pose:
|
| 147 |
+
self.loss_pose = HuberLoss(alpha=self.train_cfg.pose_loss_alpha, delta=self.train_cfg.pose_loss_delta)
|
| 148 |
+
|
| 149 |
+
if self.model.encoder.distill:
|
| 150 |
+
self.loss_distill = DistillLoss(
|
| 151 |
+
delta=self.train_cfg.pose_loss_delta,
|
| 152 |
+
weight_pose=self.train_cfg.weight_pose,
|
| 153 |
+
weight_depth=self.train_cfg.weight_depth,
|
| 154 |
+
weight_normal=self.train_cfg.weight_normal
|
| 155 |
+
)
|
| 156 |
+
|
| 157 |
+
# This is used for testing.
|
| 158 |
+
self.benchmarker = Benchmarker()
|
| 159 |
+
|
| 160 |
+
def on_train_epoch_start(self) -> None:
|
| 161 |
+
# our custom dataset and sampler has to have epoch set by calling set_epoch
|
| 162 |
+
if hasattr(self.trainer.datamodule.train_loader.dataset, "set_epoch"):
|
| 163 |
+
self.trainer.datamodule.train_loader.dataset.set_epoch(self.current_epoch)
|
| 164 |
+
if hasattr(self.trainer.datamodule.train_loader.sampler, "set_epoch"):
|
| 165 |
+
self.trainer.datamodule.train_loader.sampler.set_epoch(self.current_epoch)
|
| 166 |
+
|
| 167 |
+
def on_validation_epoch_start(self) -> None:
|
| 168 |
+
print(f"Validation epoch start on rank {self.trainer.global_rank}")
|
| 169 |
+
# our custom dataset and sampler has to have epoch set by calling set_epoch
|
| 170 |
+
if hasattr(self.trainer.datamodule.val_loader.dataset, "set_epoch"):
|
| 171 |
+
self.trainer.datamodule.val_loader.dataset.set_epoch(self.current_epoch)
|
| 172 |
+
if hasattr(self.trainer.datamodule.val_loader.sampler, "set_epoch"):
|
| 173 |
+
self.trainer.datamodule.val_loader.sampler.set_epoch(self.current_epoch)
|
| 174 |
+
|
| 175 |
+
def training_step(self, batch, batch_idx):
|
| 176 |
+
# combine batch from different dataloaders
|
| 177 |
+
# torch.cuda.empty_cache()
|
| 178 |
+
if isinstance(batch, list):
|
| 179 |
+
batch_combined = None
|
| 180 |
+
for batch_per_dl in batch:
|
| 181 |
+
if batch_combined is None:
|
| 182 |
+
batch_combined = batch_per_dl
|
| 183 |
+
else:
|
| 184 |
+
for k in batch_combined.keys():
|
| 185 |
+
if isinstance(batch_combined[k], list):
|
| 186 |
+
batch_combined[k] += batch_per_dl[k]
|
| 187 |
+
elif isinstance(batch_combined[k], dict):
|
| 188 |
+
for kk in batch_combined[k].keys():
|
| 189 |
+
batch_combined[k][kk] = torch.cat([batch_combined[k][kk], batch_per_dl[k][kk]], dim=0)
|
| 190 |
+
else:
|
| 191 |
+
raise NotImplementedError
|
| 192 |
+
batch = batch_combined
|
| 193 |
+
|
| 194 |
+
batch: BatchedExample = self.data_shim(batch)
|
| 195 |
+
b, v, c, h, w = batch["context"]["image"].shape
|
| 196 |
+
context_image = (batch["context"]["image"] + 1) / 2
|
| 197 |
+
|
| 198 |
+
# Run the model.
|
| 199 |
+
visualization_dump = None
|
| 200 |
+
|
| 201 |
+
encoder_output, output = self.model(context_image, self.global_step, visualization_dump=visualization_dump)
|
| 202 |
+
gaussians, pred_pose_enc_list, depth_dict = encoder_output.gaussians, encoder_output.pred_pose_enc_list, encoder_output.depth_dict
|
| 203 |
+
pred_context_pose = encoder_output.pred_context_pose
|
| 204 |
+
infos = encoder_output.infos
|
| 205 |
+
distill_infos = encoder_output.distill_infos
|
| 206 |
+
|
| 207 |
+
num_context_views = pred_context_pose['extrinsic'].shape[1]
|
| 208 |
+
|
| 209 |
+
using_index = torch.arange(num_context_views, device=gaussians.means.device)
|
| 210 |
+
batch["using_index"] = using_index
|
| 211 |
+
|
| 212 |
+
target_gt = (batch["context"]["image"] + 1) / 2
|
| 213 |
+
scene_scale = infos["scene_scale"]
|
| 214 |
+
self.log("train/scene_scale", infos["scene_scale"])
|
| 215 |
+
self.log("train/voxelize_ratio", infos["voxelize_ratio"])
|
| 216 |
+
|
| 217 |
+
# Compute metrics.
|
| 218 |
+
psnr_probabilistic = compute_psnr(
|
| 219 |
+
rearrange(target_gt, "b v c h w -> (b v) c h w"),
|
| 220 |
+
rearrange(output.color, "b v c h w -> (b v) c h w"),
|
| 221 |
+
)
|
| 222 |
+
self.log("train/psnr_probabilistic", psnr_probabilistic.mean())
|
| 223 |
+
|
| 224 |
+
consis_absrel = abs_relative_difference(
|
| 225 |
+
rearrange(output.depth, "b v h w -> (b v) h w"),
|
| 226 |
+
rearrange(depth_dict['depth'].squeeze(-1), "b v h w -> (b v) h w"),
|
| 227 |
+
rearrange(distill_infos['conf_mask'], "b v h w -> (b v) h w"),
|
| 228 |
+
)
|
| 229 |
+
self.log("train/consis_absrel", consis_absrel.mean())
|
| 230 |
+
|
| 231 |
+
consis_delta1 = delta1_acc(
|
| 232 |
+
rearrange(output.depth, "b v h w -> (b v) h w"),
|
| 233 |
+
rearrange(depth_dict['depth'].squeeze(-1), "b v h w -> (b v) h w"),
|
| 234 |
+
rearrange(distill_infos['conf_mask'], "b v h w -> (b v) h w"),
|
| 235 |
+
)
|
| 236 |
+
self.log("train/consis_delta1", consis_delta1.mean())
|
| 237 |
+
|
| 238 |
+
# Compute and log loss.
|
| 239 |
+
total_loss = 0
|
| 240 |
+
|
| 241 |
+
depth_dict['distill_infos'] = distill_infos
|
| 242 |
+
with torch.amp.autocast('cuda', enabled=False):
|
| 243 |
+
for loss_fn in self.losses:
|
| 244 |
+
loss = loss_fn.forward(output, batch, gaussians, depth_dict, self.global_step)
|
| 245 |
+
self.log(f"loss/{loss_fn.name}", loss)
|
| 246 |
+
total_loss = total_loss + loss
|
| 247 |
+
|
| 248 |
+
if depth_dict is not None and "depth" in get_cfg()["loss"].keys() and self.train_cfg.cxt_depth_weight > 0:
|
| 249 |
+
depth_loss_idx = list(get_cfg()["loss"].keys()).index("depth")
|
| 250 |
+
depth_loss_fn = self.losses[depth_loss_idx].ctx_depth_loss
|
| 251 |
+
loss_depth = depth_loss_fn(depth_dict["depth_map"], depth_dict["depth_conf"], batch, cxt_depth_weight=self.train_cfg.cxt_depth_weight)
|
| 252 |
+
self.log("loss/ctx_depth", loss_depth)
|
| 253 |
+
total_loss = total_loss + loss_depth
|
| 254 |
+
|
| 255 |
+
if distill_infos is not None:
|
| 256 |
+
# distill ctx pred_pose & depth & normal
|
| 257 |
+
loss_distill_list = self.loss_distill(distill_infos, pred_pose_enc_list, output, batch)
|
| 258 |
+
self.log("loss/distill", loss_distill_list['loss_distill'])
|
| 259 |
+
self.log("loss/distill_pose", loss_distill_list['loss_pose'])
|
| 260 |
+
self.log("loss/distill_depth", loss_distill_list['loss_depth'])
|
| 261 |
+
self.log("loss/distill_normal", loss_distill_list['loss_normal'])
|
| 262 |
+
total_loss = total_loss + loss_distill_list['loss_distill']
|
| 263 |
+
|
| 264 |
+
self.log("loss/total", total_loss)
|
| 265 |
+
print(f"total_loss: {total_loss}")
|
| 266 |
+
|
| 267 |
+
# Skip batch if loss is too high after certain step
|
| 268 |
+
SKIP_AFTER_STEP = 1000
|
| 269 |
+
LOSS_THRESHOLD = 0.2
|
| 270 |
+
if self.global_step > SKIP_AFTER_STEP and total_loss > LOSS_THRESHOLD:
|
| 271 |
+
print(f"Skipping batch with high loss ({total_loss:.6f}) at step {self.global_step} on Rank {self.global_rank}")
|
| 272 |
+
# set to a really small number
|
| 273 |
+
return total_loss * 1e-10
|
| 274 |
+
|
| 275 |
+
if (
|
| 276 |
+
self.global_rank == 0
|
| 277 |
+
and self.global_step % self.train_cfg.print_log_every_n_steps == 0
|
| 278 |
+
):
|
| 279 |
+
print(
|
| 280 |
+
f"train step {self.global_step}; "
|
| 281 |
+
f"scene = {[x[:20] for x in batch['scene']]}; "
|
| 282 |
+
f"context = {batch['context']['index'].tolist()}; "
|
| 283 |
+
f"loss = {total_loss:.6f}; "
|
| 284 |
+
)
|
| 285 |
+
|
| 286 |
+
self.log("info/global_step", self.global_step) # hack for ckpt monitor
|
| 287 |
+
|
| 288 |
+
# Tell the data loader processes about the current step.
|
| 289 |
+
if self.step_tracker is not None:
|
| 290 |
+
self.step_tracker.set_step(self.global_step)
|
| 291 |
+
|
| 292 |
+
del batch
|
| 293 |
+
if self.global_step % 50 == 0:
|
| 294 |
+
gc.collect()
|
| 295 |
+
torch.cuda.empty_cache()
|
| 296 |
+
|
| 297 |
+
return total_loss
|
| 298 |
+
|
| 299 |
+
def on_after_backward(self):
|
| 300 |
+
total_norm = 0.0
|
| 301 |
+
counter = 0
|
| 302 |
+
for p in self.parameters():
|
| 303 |
+
if p.grad is not None:
|
| 304 |
+
param_norm = p.grad.detach().data.norm(2)
|
| 305 |
+
total_norm += param_norm.item() ** 2
|
| 306 |
+
counter += 1
|
| 307 |
+
total_norm = (total_norm / counter) ** 0.5
|
| 308 |
+
self.log("loss/grad_norm", total_norm)
|
| 309 |
+
|
| 310 |
+
def test_step(self, batch, batch_idx):
|
| 311 |
+
batch: BatchedExample = self.data_shim(batch)
|
| 312 |
+
b, v, _, h, w = batch["target"]["image"].shape
|
| 313 |
+
assert b == 1
|
| 314 |
+
if batch_idx % 100 == 0:
|
| 315 |
+
print(f"Test step {batch_idx:0>6}.")
|
| 316 |
+
|
| 317 |
+
# Render Gaussians.
|
| 318 |
+
with self.benchmarker.time("encoder"):
|
| 319 |
+
gaussians = self.model.encoder(
|
| 320 |
+
(batch["context"]["image"]+1)/2,
|
| 321 |
+
self.global_step,
|
| 322 |
+
)[0]
|
| 323 |
+
# export_ply(gaussians.means[0], gaussians.scales[0], gaussians.rotations[0], gaussians.harmonics[0], gaussians.opacities[0], Path("gaussians.ply"))
|
| 324 |
+
# align the target pose
|
| 325 |
+
if self.test_cfg.align_pose:
|
| 326 |
+
output = self.test_step_align(batch, gaussians)
|
| 327 |
+
else:
|
| 328 |
+
with self.benchmarker.time("decoder", num_calls=v):
|
| 329 |
+
output = self.model.decoder.forward(
|
| 330 |
+
gaussians,
|
| 331 |
+
batch["target"]["extrinsics"],
|
| 332 |
+
batch["target"]["intrinsics"],
|
| 333 |
+
batch["target"]["near"],
|
| 334 |
+
batch["target"]["far"],
|
| 335 |
+
(h, w),
|
| 336 |
+
)
|
| 337 |
+
|
| 338 |
+
# compute scores
|
| 339 |
+
if self.test_cfg.compute_scores:
|
| 340 |
+
overlap = batch["context"]["overlap"][0]
|
| 341 |
+
overlap_tag = get_overlap_tag(overlap)
|
| 342 |
+
|
| 343 |
+
rgb_pred = output.color[0]
|
| 344 |
+
rgb_gt = batch["target"]["image"][0]
|
| 345 |
+
all_metrics = {
|
| 346 |
+
f"lpips_ours": compute_lpips(rgb_gt, rgb_pred).mean(),
|
| 347 |
+
f"ssim_ours": compute_ssim(rgb_gt, rgb_pred).mean(),
|
| 348 |
+
f"psnr_ours": compute_psnr(rgb_gt, rgb_pred).mean(),
|
| 349 |
+
}
|
| 350 |
+
methods = ['ours']
|
| 351 |
+
|
| 352 |
+
self.log_dict(all_metrics)
|
| 353 |
+
self.print_preview_metrics(all_metrics, methods, overlap_tag=overlap_tag)
|
| 354 |
+
|
| 355 |
+
# Save images.
|
| 356 |
+
(scene,) = batch["scene"]
|
| 357 |
+
name = get_cfg()["wandb"]["name"]
|
| 358 |
+
path = self.test_cfg.output_path / name
|
| 359 |
+
if self.test_cfg.save_image:
|
| 360 |
+
for index, color in zip(batch["target"]["index"][0], output.color[0]):
|
| 361 |
+
save_image(color, path / scene / f"color/{index:0>6}.png")
|
| 362 |
+
|
| 363 |
+
if self.test_cfg.save_video:
|
| 364 |
+
frame_str = "_".join([str(x.item()) for x in batch["context"]["index"][0]])
|
| 365 |
+
save_video(
|
| 366 |
+
[a for a in output.color[0]],
|
| 367 |
+
path / "video" / f"{scene}_frame_{frame_str}.mp4",
|
| 368 |
+
)
|
| 369 |
+
|
| 370 |
+
if self.test_cfg.save_compare:
|
| 371 |
+
# Construct comparison image.
|
| 372 |
+
context_img = inverse_normalize(batch["context"]["image"][0])
|
| 373 |
+
comparison = hcat(
|
| 374 |
+
add_label(vcat(*context_img), "Context"),
|
| 375 |
+
add_label(vcat(*rgb_gt), "Target (Ground Truth)"),
|
| 376 |
+
add_label(vcat(*rgb_pred), "Target (Prediction)"),
|
| 377 |
+
)
|
| 378 |
+
save_image(comparison, path / f"{scene}.png")
|
| 379 |
+
|
| 380 |
+
def test_step_align(self, batch, gaussians):
|
| 381 |
+
self.model.encoder.eval()
|
| 382 |
+
# freeze all parameters
|
| 383 |
+
for param in self.model.encoder.parameters():
|
| 384 |
+
param.requires_grad = False
|
| 385 |
+
|
| 386 |
+
b, v, _, h, w = batch["target"]["image"].shape
|
| 387 |
+
output_c2ws = batch["target"]["extrinsics"]
|
| 388 |
+
with torch.set_grad_enabled(True):
|
| 389 |
+
cam_rot_delta = nn.Parameter(torch.zeros([b, v, 6], requires_grad=True, device=output_c2ws.device))
|
| 390 |
+
cam_trans_delta = nn.Parameter(torch.zeros([b, v, 3], requires_grad=True, device=output_c2ws.device))
|
| 391 |
+
opt_params = []
|
| 392 |
+
self.register_buffer("identity", torch.tensor([1.0, 0.0, 0.0, 0.0, 1.0, 0.0]).to(output_c2ws))
|
| 393 |
+
opt_params.append(
|
| 394 |
+
{
|
| 395 |
+
"params": [cam_rot_delta],
|
| 396 |
+
"lr": 0.005,
|
| 397 |
+
}
|
| 398 |
+
)
|
| 399 |
+
opt_params.append(
|
| 400 |
+
{
|
| 401 |
+
"params": [cam_trans_delta],
|
| 402 |
+
"lr": 0.005,
|
| 403 |
+
}
|
| 404 |
+
)
|
| 405 |
+
pose_optimizer = torch.optim.Adam(opt_params)
|
| 406 |
+
extrinsics = output_c2ws.clone()
|
| 407 |
+
with self.benchmarker.time("optimize"):
|
| 408 |
+
for i in range(self.test_cfg.pose_align_steps):
|
| 409 |
+
pose_optimizer.zero_grad()
|
| 410 |
+
dx, drot = cam_trans_delta, cam_rot_delta
|
| 411 |
+
rot = rotation_6d_to_matrix(
|
| 412 |
+
drot + self.identity.expand(b, v, -1)
|
| 413 |
+
) # (..., 3, 3)
|
| 414 |
+
|
| 415 |
+
transform = torch.eye(4, device=extrinsics.device).repeat((b, v, 1, 1))
|
| 416 |
+
transform[..., :3, :3] = rot
|
| 417 |
+
transform[..., :3, 3] = dx
|
| 418 |
+
|
| 419 |
+
new_extrinsics = torch.matmul(extrinsics, transform)
|
| 420 |
+
output = self.model.decoder.forward(
|
| 421 |
+
gaussians,
|
| 422 |
+
new_extrinsics,
|
| 423 |
+
batch["target"]["intrinsics"],
|
| 424 |
+
batch["target"]["near"],
|
| 425 |
+
batch["target"]["far"],
|
| 426 |
+
(h, w),
|
| 427 |
+
# cam_rot_delta=cam_rot_delta,
|
| 428 |
+
# cam_trans_delta=cam_trans_delta,
|
| 429 |
+
)
|
| 430 |
+
|
| 431 |
+
# Compute and log loss.
|
| 432 |
+
total_loss = 0
|
| 433 |
+
for loss_fn in self.losses:
|
| 434 |
+
loss = loss_fn.forward(output, batch, gaussians, self.global_step)
|
| 435 |
+
total_loss = total_loss + loss
|
| 436 |
+
|
| 437 |
+
total_loss.backward()
|
| 438 |
+
pose_optimizer.step()
|
| 439 |
+
|
| 440 |
+
# Render Gaussians.
|
| 441 |
+
output = self.model.decoder.forward(
|
| 442 |
+
gaussians,
|
| 443 |
+
new_extrinsics,
|
| 444 |
+
batch["target"]["intrinsics"],
|
| 445 |
+
batch["target"]["near"],
|
| 446 |
+
batch["target"]["far"],
|
| 447 |
+
(h, w),
|
| 448 |
+
)
|
| 449 |
+
|
| 450 |
+
return output
|
| 451 |
+
|
| 452 |
+
def on_test_end(self) -> None:
|
| 453 |
+
name = get_cfg()["wandb"]["name"]
|
| 454 |
+
self.benchmarker.dump(self.test_cfg.output_path / name / "benchmark.json")
|
| 455 |
+
self.benchmarker.dump_memory(
|
| 456 |
+
self.test_cfg.output_path / name / "peak_memory.json"
|
| 457 |
+
)
|
| 458 |
+
self.benchmarker.summarize()
|
| 459 |
+
|
| 460 |
+
@rank_zero_only
|
| 461 |
+
def validation_step(self, batch, batch_idx, dataloader_idx=0):
|
| 462 |
+
batch: BatchedExample = self.data_shim(batch)
|
| 463 |
+
|
| 464 |
+
if self.global_rank == 0:
|
| 465 |
+
print(
|
| 466 |
+
f"validation step {self.global_step}; "
|
| 467 |
+
f"scene = {batch['scene']}; "
|
| 468 |
+
f"context = {batch['context']['index'].tolist()}"
|
| 469 |
+
)
|
| 470 |
+
|
| 471 |
+
# Render Gaussians.
|
| 472 |
+
b, v, _, h, w = batch["context"]["image"].shape
|
| 473 |
+
assert b == 1
|
| 474 |
+
visualization_dump = {}
|
| 475 |
+
|
| 476 |
+
encoder_output, output = self.model(batch["context"]["image"], self.global_step, visualization_dump=visualization_dump)
|
| 477 |
+
gaussians, pred_pose_enc_list, depth_dict = encoder_output.gaussians, encoder_output.pred_pose_enc_list, encoder_output.depth_dict
|
| 478 |
+
pred_context_pose, distill_infos = encoder_output.pred_context_pose, encoder_output.distill_infos
|
| 479 |
+
infos = encoder_output.infos
|
| 480 |
+
|
| 481 |
+
GS_num = infos['voxelize_ratio'] * (h*w*v)
|
| 482 |
+
self.log("val/GS_num", GS_num)
|
| 483 |
+
|
| 484 |
+
num_context_views = pred_context_pose['extrinsic'].shape[1]
|
| 485 |
+
num_target_views = batch["target"]["extrinsics"].shape[1]
|
| 486 |
+
rgb_pred = output.color[0].float()
|
| 487 |
+
depth_pred = vis_depth_map(output.depth[0])
|
| 488 |
+
|
| 489 |
+
# direct depth from gaussian means (used for visualization only)
|
| 490 |
+
gaussian_means = visualization_dump["depth"][0].squeeze()
|
| 491 |
+
if gaussian_means.shape[-1] == 3:
|
| 492 |
+
gaussian_means = gaussian_means.mean(dim=-1)
|
| 493 |
+
|
| 494 |
+
# Compute validation metrics.
|
| 495 |
+
rgb_gt = (batch["context"]["image"][0].float() + 1) / 2
|
| 496 |
+
psnr = compute_psnr(rgb_gt, rgb_pred).mean()
|
| 497 |
+
self.log(f"val/psnr", psnr)
|
| 498 |
+
lpips = compute_lpips(rgb_gt, rgb_pred).mean()
|
| 499 |
+
self.log(f"val/lpips", lpips)
|
| 500 |
+
ssim = compute_ssim(rgb_gt, rgb_pred).mean()
|
| 501 |
+
self.log(f"val/ssim", ssim)
|
| 502 |
+
|
| 503 |
+
# depth metrics
|
| 504 |
+
consis_absrel = abs_relative_difference(
|
| 505 |
+
rearrange(output.depth, "b v h w -> (b v) h w"),
|
| 506 |
+
rearrange(depth_dict['depth'].squeeze(-1), "b v h w -> (b v) h w"),
|
| 507 |
+
)
|
| 508 |
+
self.log("val/consis_absrel", consis_absrel.mean())
|
| 509 |
+
|
| 510 |
+
consis_delta1 = delta1_acc(
|
| 511 |
+
rearrange(output.depth, "b v h w -> (b v) h w"),
|
| 512 |
+
rearrange(depth_dict['depth'].squeeze(-1), "b v h w -> (b v) h w"),
|
| 513 |
+
valid_mask=rearrange(torch.ones_like(output.depth, device=output.depth.device, dtype=torch.bool), "b v h w -> (b v) h w"),
|
| 514 |
+
)
|
| 515 |
+
self.log("val/consis_delta1", consis_delta1.mean())
|
| 516 |
+
|
| 517 |
+
diff_map = torch.abs(output.depth - depth_dict['depth'].squeeze(-1))
|
| 518 |
+
self.log("val/consis_mse", diff_map[distill_infos['conf_mask']].mean())
|
| 519 |
+
|
| 520 |
+
# Construct comparison image.
|
| 521 |
+
context_img = inverse_normalize(batch["context"]["image"][0])
|
| 522 |
+
# context_img_depth = vis_depth_map(gaussian_means)
|
| 523 |
+
context = []
|
| 524 |
+
for i in range(context_img.shape[0]):
|
| 525 |
+
context.append(context_img[i])
|
| 526 |
+
# context.append(context_img_depth[i])
|
| 527 |
+
|
| 528 |
+
colored_diff_map = vis_depth_map(diff_map[0], near=torch.tensor(1e-4, device=diff_map.device), far=torch.tensor(1.0, device=diff_map.device))
|
| 529 |
+
model_depth_pred = depth_dict["depth"].squeeze(-1)[0]
|
| 530 |
+
model_depth_pred = vis_depth_map(model_depth_pred)
|
| 531 |
+
|
| 532 |
+
render_normal = (get_normal_map(output.depth.flatten(0, 1), batch["context"]["intrinsics"].flatten(0, 1)).permute(0, 3, 1, 2) + 1) / 2.
|
| 533 |
+
pred_normal = (get_normal_map(depth_dict['depth'].flatten(0, 1).squeeze(-1), batch["context"]["intrinsics"].flatten(0, 1)).permute(0, 3, 1, 2) + 1) / 2.
|
| 534 |
+
|
| 535 |
+
comparison = hcat(
|
| 536 |
+
add_label(vcat(*context), "Context"),
|
| 537 |
+
add_label(vcat(*rgb_gt), "Target (Ground Truth)"),
|
| 538 |
+
add_label(vcat(*rgb_pred), "Target (Prediction)"),
|
| 539 |
+
add_label(vcat(*depth_pred), "Depth (Prediction)"),
|
| 540 |
+
add_label(vcat(*model_depth_pred), "Depth (VGGT Prediction)"),
|
| 541 |
+
add_label(vcat(*render_normal), "Normal (Prediction)"),
|
| 542 |
+
add_label(vcat(*pred_normal), "Normal (VGGT Prediction)"),
|
| 543 |
+
add_label(vcat(*colored_diff_map), "Diff Map"),
|
| 544 |
+
)
|
| 545 |
+
|
| 546 |
+
comparison = torch.nn.functional.interpolate(
|
| 547 |
+
comparison.unsqueeze(0),
|
| 548 |
+
scale_factor=0.5,
|
| 549 |
+
mode='bicubic',
|
| 550 |
+
align_corners=False
|
| 551 |
+
).squeeze(0)
|
| 552 |
+
|
| 553 |
+
self.logger.log_image(
|
| 554 |
+
"comparison",
|
| 555 |
+
[prep_image(add_border(comparison))],
|
| 556 |
+
step=self.global_step,
|
| 557 |
+
caption=batch["scene"],
|
| 558 |
+
)
|
| 559 |
+
|
| 560 |
+
# self.logger.log_image(
|
| 561 |
+
# key="comparison",
|
| 562 |
+
# images=[wandb.Image(prep_image(add_border(comparison)), caption=batch["scene"], file_type="jpg")],
|
| 563 |
+
# step=self.global_step
|
| 564 |
+
# )
|
| 565 |
+
|
| 566 |
+
# Render projections and construct projection image.
|
| 567 |
+
# These are disabled for now, since RE10k scenes are effectively unbounded.
|
| 568 |
+
|
| 569 |
+
# if isinstance(gaussians, Gaussians):
|
| 570 |
+
# projections = hcat(
|
| 571 |
+
# *render_projections(
|
| 572 |
+
# gaussians,
|
| 573 |
+
# 256,
|
| 574 |
+
# extra_label="",
|
| 575 |
+
# )[0]
|
| 576 |
+
# )
|
| 577 |
+
# self.logger.log_image(
|
| 578 |
+
# "projection",
|
| 579 |
+
# [prep_image(add_border(projections))],
|
| 580 |
+
# step=self.global_step,
|
| 581 |
+
# )
|
| 582 |
+
|
| 583 |
+
# Draw cameras.
|
| 584 |
+
# cameras = hcat(*render_cameras(batch, 256))
|
| 585 |
+
# self.logger.log_image(
|
| 586 |
+
# "cameras", [prep_image(add_border(cameras))], step=self.global_step
|
| 587 |
+
# )
|
| 588 |
+
|
| 589 |
+
if self.encoder_visualizer is not None:
|
| 590 |
+
for k, image in self.encoder_visualizer.visualize(
|
| 591 |
+
batch["context"], self.global_step
|
| 592 |
+
).items():
|
| 593 |
+
self.logger.log_image(k, [prep_image(image)], step=self.global_step)
|
| 594 |
+
|
| 595 |
+
# Run video validation step.
|
| 596 |
+
self.render_video_interpolation(batch)
|
| 597 |
+
self.render_video_wobble(batch)
|
| 598 |
+
if self.train_cfg.extended_visualization:
|
| 599 |
+
self.render_video_interpolation_exaggerated(batch)
|
| 600 |
+
|
| 601 |
+
@rank_zero_only
|
| 602 |
+
def render_video_wobble(self, batch: BatchedExample) -> None:
|
| 603 |
+
# Two views are needed to get the wobble radius.
|
| 604 |
+
_, v, _, _ = batch["context"]["extrinsics"].shape
|
| 605 |
+
if v != 2:
|
| 606 |
+
return
|
| 607 |
+
|
| 608 |
+
def trajectory_fn(t):
|
| 609 |
+
origin_a = batch["context"]["extrinsics"][:, 0, :3, 3]
|
| 610 |
+
origin_b = batch["context"]["extrinsics"][:, 1, :3, 3]
|
| 611 |
+
delta = (origin_a - origin_b).norm(dim=-1)
|
| 612 |
+
extrinsics = generate_wobble(
|
| 613 |
+
batch["context"]["extrinsics"][:, 0],
|
| 614 |
+
delta * 0.25,
|
| 615 |
+
t,
|
| 616 |
+
)
|
| 617 |
+
intrinsics = repeat(
|
| 618 |
+
batch["context"]["intrinsics"][:, 0],
|
| 619 |
+
"b i j -> b v i j",
|
| 620 |
+
v=t.shape[0],
|
| 621 |
+
)
|
| 622 |
+
return extrinsics, intrinsics
|
| 623 |
+
|
| 624 |
+
return self.render_video_generic(batch, trajectory_fn, "wobble", num_frames=60)
|
| 625 |
+
|
| 626 |
+
@rank_zero_only
|
| 627 |
+
def render_video_interpolation(self, batch: BatchedExample) -> None:
|
| 628 |
+
_, v, _, _ = batch["context"]["extrinsics"].shape
|
| 629 |
+
|
| 630 |
+
def trajectory_fn(t):
|
| 631 |
+
extrinsics = interpolate_extrinsics(
|
| 632 |
+
batch["context"]["extrinsics"][0, 0],
|
| 633 |
+
(
|
| 634 |
+
batch["context"]["extrinsics"][0, 1]
|
| 635 |
+
if v == 2
|
| 636 |
+
else batch["target"]["extrinsics"][0, 0]
|
| 637 |
+
),
|
| 638 |
+
t,
|
| 639 |
+
)
|
| 640 |
+
intrinsics = interpolate_intrinsics(
|
| 641 |
+
batch["context"]["intrinsics"][0, 0],
|
| 642 |
+
(
|
| 643 |
+
batch["context"]["intrinsics"][0, 1]
|
| 644 |
+
if v == 2
|
| 645 |
+
else batch["target"]["intrinsics"][0, 0]
|
| 646 |
+
),
|
| 647 |
+
t,
|
| 648 |
+
)
|
| 649 |
+
return extrinsics[None], intrinsics[None]
|
| 650 |
+
|
| 651 |
+
return self.render_video_generic(batch, trajectory_fn, "rgb")
|
| 652 |
+
|
| 653 |
+
@rank_zero_only
|
| 654 |
+
def render_video_interpolation_exaggerated(self, batch: BatchedExample) -> None:
|
| 655 |
+
# Two views are needed to get the wobble radius.
|
| 656 |
+
_, v, _, _ = batch["context"]["extrinsics"].shape
|
| 657 |
+
if v != 2:
|
| 658 |
+
return
|
| 659 |
+
|
| 660 |
+
def trajectory_fn(t):
|
| 661 |
+
origin_a = batch["context"]["extrinsics"][:, 0, :3, 3]
|
| 662 |
+
origin_b = batch["context"]["extrinsics"][:, 1, :3, 3]
|
| 663 |
+
delta = (origin_a - origin_b).norm(dim=-1)
|
| 664 |
+
tf = generate_wobble_transformation(
|
| 665 |
+
delta * 0.5,
|
| 666 |
+
t,
|
| 667 |
+
5,
|
| 668 |
+
scale_radius_with_t=False,
|
| 669 |
+
)
|
| 670 |
+
extrinsics = interpolate_extrinsics(
|
| 671 |
+
batch["context"]["extrinsics"][0, 0],
|
| 672 |
+
(
|
| 673 |
+
batch["context"]["extrinsics"][0, 1]
|
| 674 |
+
if v == 2
|
| 675 |
+
else batch["target"]["extrinsics"][0, 0]
|
| 676 |
+
),
|
| 677 |
+
t * 5 - 2,
|
| 678 |
+
)
|
| 679 |
+
intrinsics = interpolate_intrinsics(
|
| 680 |
+
batch["context"]["intrinsics"][0, 0],
|
| 681 |
+
(
|
| 682 |
+
batch["context"]["intrinsics"][0, 1]
|
| 683 |
+
if v == 2
|
| 684 |
+
else batch["target"]["intrinsics"][0, 0]
|
| 685 |
+
),
|
| 686 |
+
t * 5 - 2,
|
| 687 |
+
)
|
| 688 |
+
return extrinsics @ tf, intrinsics[None]
|
| 689 |
+
|
| 690 |
+
return self.render_video_generic(
|
| 691 |
+
batch,
|
| 692 |
+
trajectory_fn,
|
| 693 |
+
"interpolation_exagerrated",
|
| 694 |
+
num_frames=300,
|
| 695 |
+
smooth=False,
|
| 696 |
+
loop_reverse=False,
|
| 697 |
+
)
|
| 698 |
+
|
| 699 |
+
@rank_zero_only
|
| 700 |
+
def render_video_generic(
|
| 701 |
+
self,
|
| 702 |
+
batch: BatchedExample,
|
| 703 |
+
trajectory_fn: TrajectoryFn,
|
| 704 |
+
name: str,
|
| 705 |
+
num_frames: int = 30,
|
| 706 |
+
smooth: bool = True,
|
| 707 |
+
loop_reverse: bool = True,
|
| 708 |
+
) -> None:
|
| 709 |
+
# Render probabilistic estimate of scene.
|
| 710 |
+
encoder_output = self.model.encoder((batch["context"]["image"]+1)/2, self.global_step)
|
| 711 |
+
gaussians, pred_pose_enc_list = encoder_output.gaussians, encoder_output.pred_pose_enc_list
|
| 712 |
+
|
| 713 |
+
t = torch.linspace(0, 1, num_frames, dtype=torch.float32, device=self.device)
|
| 714 |
+
if smooth:
|
| 715 |
+
t = (torch.cos(torch.pi * (t + 1)) + 1) / 2
|
| 716 |
+
|
| 717 |
+
extrinsics, intrinsics = trajectory_fn(t)
|
| 718 |
+
|
| 719 |
+
_, _, _, h, w = batch["context"]["image"].shape
|
| 720 |
+
|
| 721 |
+
# TODO: Interpolate near and far planes?
|
| 722 |
+
near = repeat(batch["context"]["near"][:, 0], "b -> b v", v=num_frames)
|
| 723 |
+
far = repeat(batch["context"]["far"][:, 0], "b -> b v", v=num_frames)
|
| 724 |
+
output = self.model.decoder.forward(
|
| 725 |
+
gaussians, extrinsics, intrinsics, near, far, (h, w), "depth"
|
| 726 |
+
)
|
| 727 |
+
images = [
|
| 728 |
+
vcat(rgb, depth)
|
| 729 |
+
for rgb, depth in zip(output.color[0], vis_depth_map(output.depth[0]))
|
| 730 |
+
]
|
| 731 |
+
|
| 732 |
+
video = torch.stack(images)
|
| 733 |
+
video = (video.clip(min=0, max=1) * 255).type(torch.uint8).cpu().numpy()
|
| 734 |
+
if loop_reverse:
|
| 735 |
+
video = pack([video, video[::-1][1:-1]], "* c h w")[0]
|
| 736 |
+
visualizations = {
|
| 737 |
+
f"video/{name}": wandb.Video(video[None], fps=30, format="mp4")
|
| 738 |
+
}
|
| 739 |
+
|
| 740 |
+
# Since the PyTorch Lightning doesn't support video logging, log to wandb directly.
|
| 741 |
+
try:
|
| 742 |
+
wandb.log(visualizations)
|
| 743 |
+
except Exception:
|
| 744 |
+
assert isinstance(self.logger, LocalLogger)
|
| 745 |
+
for key, value in visualizations.items():
|
| 746 |
+
tensor = value._prepare_video(value.data)
|
| 747 |
+
clip = mpy.ImageSequenceClip(list(tensor), fps=30)
|
| 748 |
+
dir = LOG_PATH / key
|
| 749 |
+
dir.mkdir(exist_ok=True, parents=True)
|
| 750 |
+
clip.write_videofile(
|
| 751 |
+
str(dir / f"{self.global_step:0>6}.mp4"), logger=None
|
| 752 |
+
)
|
| 753 |
+
|
| 754 |
+
def print_preview_metrics(self, metrics: dict[str, float | Tensor], methods: list[str] | None = None, overlap_tag: str | None = None) -> None:
|
| 755 |
+
if getattr(self, "running_metrics", None) is None:
|
| 756 |
+
self.running_metrics = metrics
|
| 757 |
+
self.running_metric_steps = 1
|
| 758 |
+
else:
|
| 759 |
+
s = self.running_metric_steps
|
| 760 |
+
self.running_metrics = {
|
| 761 |
+
k: ((s * v) + metrics[k]) / (s + 1)
|
| 762 |
+
for k, v in self.running_metrics.items()
|
| 763 |
+
}
|
| 764 |
+
self.running_metric_steps += 1
|
| 765 |
+
|
| 766 |
+
if overlap_tag is not None:
|
| 767 |
+
if getattr(self, "running_metrics_sub", None) is None:
|
| 768 |
+
self.running_metrics_sub = {overlap_tag: metrics}
|
| 769 |
+
self.running_metric_steps_sub = {overlap_tag: 1}
|
| 770 |
+
elif overlap_tag not in self.running_metrics_sub:
|
| 771 |
+
self.running_metrics_sub[overlap_tag] = metrics
|
| 772 |
+
self.running_metric_steps_sub[overlap_tag] = 1
|
| 773 |
+
else:
|
| 774 |
+
s = self.running_metric_steps_sub[overlap_tag]
|
| 775 |
+
self.running_metrics_sub[overlap_tag] = {k: ((s * v) + metrics[k]) / (s + 1)
|
| 776 |
+
for k, v in self.running_metrics_sub[overlap_tag].items()}
|
| 777 |
+
self.running_metric_steps_sub[overlap_tag] += 1
|
| 778 |
+
|
| 779 |
+
metric_list = ["psnr", "lpips", "ssim"]
|
| 780 |
+
|
| 781 |
+
def print_metrics(runing_metric, methods=None):
|
| 782 |
+
table = []
|
| 783 |
+
if methods is None:
|
| 784 |
+
methods = ['ours']
|
| 785 |
+
|
| 786 |
+
for method in methods:
|
| 787 |
+
row = [
|
| 788 |
+
f"{runing_metric[f'{metric}_{method}']:.3f}"
|
| 789 |
+
for metric in metric_list
|
| 790 |
+
]
|
| 791 |
+
table.append((method, *row))
|
| 792 |
+
|
| 793 |
+
headers = ["Method"] + metric_list
|
| 794 |
+
table = tabulate(table, headers)
|
| 795 |
+
print(table)
|
| 796 |
+
|
| 797 |
+
print("All Pairs:")
|
| 798 |
+
print_metrics(self.running_metrics, methods)
|
| 799 |
+
if overlap_tag is not None:
|
| 800 |
+
for k, v in self.running_metrics_sub.items():
|
| 801 |
+
print(f"Overlap: {k}")
|
| 802 |
+
print_metrics(v, methods)
|
| 803 |
+
|
| 804 |
+
def configure_optimizers(self):
|
| 805 |
+
new_params, new_param_names = [], []
|
| 806 |
+
pretrained_params, pretrained_param_names = [], []
|
| 807 |
+
for name, param in self.named_parameters():
|
| 808 |
+
if not param.requires_grad:
|
| 809 |
+
continue
|
| 810 |
+
|
| 811 |
+
if "gaussian_param_head" in name or "interm" in name:
|
| 812 |
+
new_params.append(param)
|
| 813 |
+
new_param_names.append(name)
|
| 814 |
+
else:
|
| 815 |
+
pretrained_params.append(param)
|
| 816 |
+
pretrained_param_names.append(name)
|
| 817 |
+
|
| 818 |
+
param_dicts = [
|
| 819 |
+
{
|
| 820 |
+
"params": new_params,
|
| 821 |
+
"lr": self.optimizer_cfg.lr,
|
| 822 |
+
},
|
| 823 |
+
{
|
| 824 |
+
"params": pretrained_params,
|
| 825 |
+
"lr": self.optimizer_cfg.lr * self.optimizer_cfg.backbone_lr_multiplier,
|
| 826 |
+
},
|
| 827 |
+
]
|
| 828 |
+
optimizer = torch.optim.AdamW(param_dicts, lr=self.optimizer_cfg.lr, weight_decay=0.05, betas=(0.9, 0.95))
|
| 829 |
+
warm_up_steps = self.optimizer_cfg.warm_up_steps
|
| 830 |
+
warm_up = torch.optim.lr_scheduler.LinearLR(
|
| 831 |
+
optimizer,
|
| 832 |
+
1 / warm_up_steps,
|
| 833 |
+
1,
|
| 834 |
+
total_iters=warm_up_steps,
|
| 835 |
+
)
|
| 836 |
+
|
| 837 |
+
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=get_cfg()["trainer"]["max_steps"], eta_min=self.optimizer_cfg.lr * 0.1)
|
| 838 |
+
lr_scheduler = torch.optim.lr_scheduler.SequentialLR(optimizer, schedulers=[warm_up, lr_scheduler], milestones=[warm_up_steps])
|
| 839 |
+
|
| 840 |
+
return {
|
| 841 |
+
"optimizer": optimizer,
|
| 842 |
+
"lr_scheduler": {
|
| 843 |
+
"scheduler": lr_scheduler,
|
| 844 |
+
"interval": "step",
|
| 845 |
+
"frequency": 1,
|
| 846 |
+
},
|
| 847 |
+
}
|
src/model/ply_export.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
|
| 3 |
+
import numpy as np
|
| 4 |
+
import torch
|
| 5 |
+
from einops import einsum, rearrange
|
| 6 |
+
from jaxtyping import Float
|
| 7 |
+
from plyfile import PlyData, PlyElement
|
| 8 |
+
from scipy.spatial.transform import Rotation as R
|
| 9 |
+
from torch import Tensor
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def construct_list_of_attributes(num_rest: int) -> list[str]:
|
| 13 |
+
attributes = ["x", "y", "z", "nx", "ny", "nz"]
|
| 14 |
+
for i in range(3):
|
| 15 |
+
attributes.append(f"f_dc_{i}")
|
| 16 |
+
for i in range(num_rest):
|
| 17 |
+
attributes.append(f"f_rest_{i}")
|
| 18 |
+
attributes.append("opacity")
|
| 19 |
+
for i in range(3):
|
| 20 |
+
attributes.append(f"scale_{i}")
|
| 21 |
+
for i in range(4):
|
| 22 |
+
attributes.append(f"rot_{i}")
|
| 23 |
+
return attributes
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def export_ply(
|
| 27 |
+
means: Float[Tensor, "gaussian 3"],
|
| 28 |
+
scales: Float[Tensor, "gaussian 3"],
|
| 29 |
+
rotations: Float[Tensor, "gaussian 4"],
|
| 30 |
+
harmonics: Float[Tensor, "gaussian 3 d_sh"],
|
| 31 |
+
opacities: Float[Tensor, " gaussian"],
|
| 32 |
+
path: Path,
|
| 33 |
+
shift_and_scale: bool = False,
|
| 34 |
+
save_sh_dc_only: bool = True,
|
| 35 |
+
):
|
| 36 |
+
if shift_and_scale:
|
| 37 |
+
# Shift the scene so that the median Gaussian is at the origin.
|
| 38 |
+
means = means - means.median(dim=0).values
|
| 39 |
+
|
| 40 |
+
# Rescale the scene so that most Gaussians are within range [-1, 1].
|
| 41 |
+
scale_factor = means.abs().quantile(0.95, dim=0).max()
|
| 42 |
+
means = means / scale_factor
|
| 43 |
+
scales = scales / scale_factor
|
| 44 |
+
|
| 45 |
+
# Apply the rotation to the Gaussian rotations.
|
| 46 |
+
rotations = R.from_quat(rotations.detach().cpu().numpy()).as_matrix()
|
| 47 |
+
rotations = R.from_matrix(rotations).as_quat()
|
| 48 |
+
x, y, z, w = rearrange(rotations, "g xyzw -> xyzw g")
|
| 49 |
+
rotations = np.stack((w, x, y, z), axis=-1)
|
| 50 |
+
|
| 51 |
+
# Since current model use SH_degree = 4,
|
| 52 |
+
# which require large memory to store, we can only save the DC band to save memory.
|
| 53 |
+
f_dc = harmonics[..., 0]
|
| 54 |
+
f_rest = harmonics[..., 1:].flatten(start_dim=1)
|
| 55 |
+
|
| 56 |
+
dtype_full = [(attribute, "f4") for attribute in construct_list_of_attributes(0 if save_sh_dc_only else f_rest.shape[1])]
|
| 57 |
+
elements = np.empty(means.shape[0], dtype=dtype_full)
|
| 58 |
+
attributes = [
|
| 59 |
+
means.detach().cpu().numpy(),
|
| 60 |
+
torch.zeros_like(means).detach().cpu().numpy(),
|
| 61 |
+
f_dc.detach().cpu().contiguous().numpy(),
|
| 62 |
+
f_rest.detach().cpu().contiguous().numpy(),
|
| 63 |
+
opacities[..., None].detach().cpu().numpy(),
|
| 64 |
+
scales.log().detach().cpu().numpy(),
|
| 65 |
+
rotations,
|
| 66 |
+
]
|
| 67 |
+
if save_sh_dc_only:
|
| 68 |
+
# remove f_rest from attributes
|
| 69 |
+
attributes.pop(3)
|
| 70 |
+
|
| 71 |
+
attributes = np.concatenate(attributes, axis=1)
|
| 72 |
+
elements[:] = list(map(tuple, attributes))
|
| 73 |
+
path.parent.mkdir(exist_ok=True, parents=True)
|
| 74 |
+
PlyData([PlyElement.describe(elements, "vertex")]).write(path)
|
src/model/transformer/attention.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# MIT License
|
| 2 |
+
|
| 3 |
+
# Copyright (c) 2022 Karl Stelzner
|
| 4 |
+
|
| 5 |
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
# of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
# in the Software without restriction, including without limitation the rights
|
| 8 |
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
# copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
# furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
# The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
# copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
# SOFTWARE.
|
| 22 |
+
|
| 23 |
+
# This file comes from https://github.com/stelzner/srt
|
| 24 |
+
|
| 25 |
+
import torch
|
| 26 |
+
from einops import rearrange
|
| 27 |
+
from torch import nn
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class Attention(nn.Module):
|
| 31 |
+
def __init__(
|
| 32 |
+
self, dim, heads=8, dim_head=64, dropout=0.0, selfatt=True, kv_dim=None
|
| 33 |
+
):
|
| 34 |
+
super().__init__()
|
| 35 |
+
inner_dim = dim_head * heads
|
| 36 |
+
project_out = not (heads == 1 and dim_head == dim)
|
| 37 |
+
|
| 38 |
+
self.heads = heads
|
| 39 |
+
self.scale = dim_head**-0.5
|
| 40 |
+
|
| 41 |
+
self.attend = nn.Softmax(dim=-1)
|
| 42 |
+
if selfatt:
|
| 43 |
+
self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False)
|
| 44 |
+
else:
|
| 45 |
+
self.to_q = nn.Linear(dim, inner_dim, bias=False)
|
| 46 |
+
self.to_kv = nn.Linear(kv_dim, inner_dim * 2, bias=False)
|
| 47 |
+
|
| 48 |
+
self.to_out = (
|
| 49 |
+
nn.Sequential(nn.Linear(inner_dim, dim), nn.Dropout(dropout))
|
| 50 |
+
if project_out
|
| 51 |
+
else nn.Identity()
|
| 52 |
+
)
|
| 53 |
+
|
| 54 |
+
def forward(self, x, z=None):
|
| 55 |
+
if z is None:
|
| 56 |
+
qkv = self.to_qkv(x).chunk(3, dim=-1)
|
| 57 |
+
else:
|
| 58 |
+
q = self.to_q(x)
|
| 59 |
+
k, v = self.to_kv(z).chunk(2, dim=-1)
|
| 60 |
+
qkv = (q, k, v)
|
| 61 |
+
|
| 62 |
+
q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b h n d", h=self.heads), qkv)
|
| 63 |
+
|
| 64 |
+
dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale
|
| 65 |
+
|
| 66 |
+
attn = self.attend(dots)
|
| 67 |
+
|
| 68 |
+
out = torch.matmul(attn, v)
|
| 69 |
+
out = rearrange(out, "b h n d -> b n (h d)")
|
| 70 |
+
return self.to_out(out)
|
src/model/transformer/feed_forward.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# MIT License
|
| 2 |
+
|
| 3 |
+
# Copyright (c) 2022 Karl Stelzner
|
| 4 |
+
|
| 5 |
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
# of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
# in the Software without restriction, including without limitation the rights
|
| 8 |
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
# copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
# furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
# The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
# copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
# SOFTWARE.
|
| 22 |
+
|
| 23 |
+
# This file comes from https://github.com/stelzner/srt
|
| 24 |
+
|
| 25 |
+
from torch import nn
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class FeedForward(nn.Module):
|
| 29 |
+
def __init__(self, dim, hidden_dim, dropout=0.0):
|
| 30 |
+
super().__init__()
|
| 31 |
+
self.net = nn.Sequential(
|
| 32 |
+
nn.Linear(dim, hidden_dim),
|
| 33 |
+
nn.GELU(),
|
| 34 |
+
nn.Dropout(dropout),
|
| 35 |
+
nn.Linear(hidden_dim, dim),
|
| 36 |
+
nn.Dropout(dropout),
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
def forward(self, x):
|
| 40 |
+
return self.net(x)
|
src/model/transformer/pre_norm.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# MIT License
|
| 2 |
+
|
| 3 |
+
# Copyright (c) 2022 Karl Stelzner
|
| 4 |
+
|
| 5 |
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
# of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
# in the Software without restriction, including without limitation the rights
|
| 8 |
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
# copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
# furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
# The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
# copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
# SOFTWARE.
|
| 22 |
+
|
| 23 |
+
# This file comes from https://github.com/stelzner/srt
|
| 24 |
+
|
| 25 |
+
from torch import nn
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class PreNorm(nn.Module):
|
| 29 |
+
def __init__(self, dim, fn):
|
| 30 |
+
super().__init__()
|
| 31 |
+
self.norm = nn.LayerNorm(dim)
|
| 32 |
+
self.fn = fn
|
| 33 |
+
|
| 34 |
+
def forward(self, x, **kwargs):
|
| 35 |
+
return self.fn(self.norm(x), **kwargs)
|
src/model/transformer/transformer.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# MIT License
|
| 2 |
+
|
| 3 |
+
# Copyright (c) 2022 Karl Stelzner
|
| 4 |
+
|
| 5 |
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
# of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
# in the Software without restriction, including without limitation the rights
|
| 8 |
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
# copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
# furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
# The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
# copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
# SOFTWARE.
|
| 22 |
+
|
| 23 |
+
# This file comes from https://github.com/stelzner/srt
|
| 24 |
+
|
| 25 |
+
from torch import nn
|
| 26 |
+
|
| 27 |
+
from .attention import Attention
|
| 28 |
+
from .feed_forward import FeedForward
|
| 29 |
+
from .pre_norm import PreNorm
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class Transformer(nn.Module):
|
| 33 |
+
def __init__(
|
| 34 |
+
self,
|
| 35 |
+
dim,
|
| 36 |
+
depth,
|
| 37 |
+
heads,
|
| 38 |
+
dim_head,
|
| 39 |
+
mlp_dim,
|
| 40 |
+
dropout=0.0,
|
| 41 |
+
selfatt=True,
|
| 42 |
+
kv_dim=None,
|
| 43 |
+
feed_forward_layer=FeedForward,
|
| 44 |
+
):
|
| 45 |
+
super().__init__()
|
| 46 |
+
self.layers = nn.ModuleList([])
|
| 47 |
+
for _ in range(depth):
|
| 48 |
+
self.layers.append(
|
| 49 |
+
nn.ModuleList(
|
| 50 |
+
[
|
| 51 |
+
PreNorm(
|
| 52 |
+
dim,
|
| 53 |
+
Attention(
|
| 54 |
+
dim,
|
| 55 |
+
heads=heads,
|
| 56 |
+
dim_head=dim_head,
|
| 57 |
+
dropout=dropout,
|
| 58 |
+
selfatt=selfatt,
|
| 59 |
+
kv_dim=kv_dim,
|
| 60 |
+
),
|
| 61 |
+
),
|
| 62 |
+
PreNorm(dim, feed_forward_layer(dim, mlp_dim, dropout=dropout)),
|
| 63 |
+
]
|
| 64 |
+
)
|
| 65 |
+
)
|
| 66 |
+
|
| 67 |
+
def forward(self, x, z=None, **kwargs):
|
| 68 |
+
for attn, ff in self.layers:
|
| 69 |
+
x = attn(x, z=z) + x
|
| 70 |
+
x = ff(x, **kwargs) + x
|
| 71 |
+
return x
|
src/model/types.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass
|
| 2 |
+
|
| 3 |
+
from jaxtyping import Float
|
| 4 |
+
from torch import Tensor
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
@dataclass
|
| 8 |
+
class Gaussians:
|
| 9 |
+
means: Float[Tensor, "batch gaussian dim"]
|
| 10 |
+
covariances: Float[Tensor, "batch gaussian dim dim"]
|
| 11 |
+
harmonics: Float[Tensor, "batch gaussian 3 d_sh"]
|
| 12 |
+
opacities: Float[Tensor, "batch gaussian"]
|
| 13 |
+
scales: Float[Tensor, "batch gaussian 3"]
|
| 14 |
+
rotations: Float[Tensor, "batch gaussian 4"]
|
| 15 |
+
# levels: Float[Tensor, "batch gaussian"]
|
src/post_opt/datasets/colmap.py
ADDED
|
@@ -0,0 +1,490 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import os
|
| 3 |
+
from typing import Any, Dict, List, Optional
|
| 4 |
+
|
| 5 |
+
import cv2
|
| 6 |
+
import imageio.v2 as imageio
|
| 7 |
+
import numpy as np
|
| 8 |
+
import torch
|
| 9 |
+
from PIL import Image
|
| 10 |
+
from pycolmap import SceneManager
|
| 11 |
+
from tqdm import tqdm
|
| 12 |
+
from typing_extensions import assert_never
|
| 13 |
+
|
| 14 |
+
import sys
|
| 15 |
+
sys.path.append("/cpfs01/user/jianglihan/projects/gsplat/examples/datasets")
|
| 16 |
+
sys.path.append("/cpfs01/user/jianglihan/projects/gsplat/examples")
|
| 17 |
+
sys.path.append("/cpfs01/user/jianglihan/projects/gsplat")
|
| 18 |
+
|
| 19 |
+
from normalize import (
|
| 20 |
+
align_principal_axes,
|
| 21 |
+
similarity_from_cameras,
|
| 22 |
+
transform_cameras,
|
| 23 |
+
transform_points,
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _get_rel_paths(path_dir: str) -> List[str]:
|
| 28 |
+
"""Recursively get relative paths of files in a directory."""
|
| 29 |
+
paths = []
|
| 30 |
+
for dp, dn, fn in os.walk(path_dir):
|
| 31 |
+
for f in fn:
|
| 32 |
+
paths.append(os.path.relpath(os.path.join(dp, f), path_dir))
|
| 33 |
+
return paths
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _resize_image_folder(image_dir: str, resized_dir: str, factor: int) -> str:
|
| 37 |
+
"""Resize image folder."""
|
| 38 |
+
print(f"Downscaling images by {factor}x from {image_dir} to {resized_dir}.")
|
| 39 |
+
os.makedirs(resized_dir, exist_ok=True)
|
| 40 |
+
|
| 41 |
+
image_files = _get_rel_paths(image_dir)
|
| 42 |
+
for image_file in tqdm(image_files):
|
| 43 |
+
image_path = os.path.join(image_dir, image_file)
|
| 44 |
+
resized_path = os.path.join(
|
| 45 |
+
resized_dir, os.path.splitext(image_file)[0] + ".png"
|
| 46 |
+
)
|
| 47 |
+
if os.path.isfile(resized_path):
|
| 48 |
+
continue
|
| 49 |
+
image = imageio.imread(image_path)[..., :3]
|
| 50 |
+
resized_size = (
|
| 51 |
+
int(round(image.shape[1] / factor)),
|
| 52 |
+
int(round(image.shape[0] / factor)),
|
| 53 |
+
)
|
| 54 |
+
resized_image = np.array(
|
| 55 |
+
Image.fromarray(image).resize(resized_size, Image.BICUBIC)
|
| 56 |
+
)
|
| 57 |
+
imageio.imwrite(resized_path, resized_image)
|
| 58 |
+
return resized_dir
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
class Parser:
|
| 62 |
+
"""COLMAP parser."""
|
| 63 |
+
|
| 64 |
+
def __init__(
|
| 65 |
+
self,
|
| 66 |
+
data_dir: str,
|
| 67 |
+
factor: int = 1,
|
| 68 |
+
normalize: bool = False,
|
| 69 |
+
test_every: int = 8,
|
| 70 |
+
):
|
| 71 |
+
self.data_dir = data_dir
|
| 72 |
+
self.factor = factor
|
| 73 |
+
self.normalize = normalize
|
| 74 |
+
self.test_every = test_every
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
colmap_dir = os.path.join(data_dir, "sparse/0/")
|
| 79 |
+
if not os.path.exists(colmap_dir):
|
| 80 |
+
colmap_dir = os.path.join(data_dir, "sparse")
|
| 81 |
+
assert os.path.exists(
|
| 82 |
+
colmap_dir
|
| 83 |
+
), f"COLMAP directory {colmap_dir} does not exist."
|
| 84 |
+
|
| 85 |
+
manager = SceneManager(colmap_dir)
|
| 86 |
+
manager.load_cameras()
|
| 87 |
+
manager.load_images()
|
| 88 |
+
manager.load_points3D()
|
| 89 |
+
|
| 90 |
+
# Extract extrinsic matrices in world-to-camera format.
|
| 91 |
+
imdata = manager.images
|
| 92 |
+
w2c_mats = []
|
| 93 |
+
camera_ids = []
|
| 94 |
+
Ks_dict = dict()
|
| 95 |
+
params_dict = dict()
|
| 96 |
+
imsize_dict = dict() # width, height
|
| 97 |
+
mask_dict = dict()
|
| 98 |
+
bottom = np.array([0, 0, 0, 1]).reshape(1, 4)
|
| 99 |
+
for k in imdata:
|
| 100 |
+
im = imdata[k]
|
| 101 |
+
rot = im.R()
|
| 102 |
+
trans = im.tvec.reshape(3, 1)
|
| 103 |
+
w2c = np.concatenate([np.concatenate([rot, trans], 1), bottom], axis=0)
|
| 104 |
+
w2c_mats.append(w2c)
|
| 105 |
+
|
| 106 |
+
# support different camera intrinsics
|
| 107 |
+
camera_id = im.camera_id
|
| 108 |
+
camera_ids.append(camera_id)
|
| 109 |
+
|
| 110 |
+
# camera intrinsics
|
| 111 |
+
cam = manager.cameras[camera_id]
|
| 112 |
+
fx, fy, cx, cy = cam.fx, cam.fy, cam.cx, cam.cy
|
| 113 |
+
K = np.array([[fx, 0, cx], [0, fy, cy], [0, 0, 1]])
|
| 114 |
+
K[:2, :] /= factor
|
| 115 |
+
Ks_dict[camera_id] = K
|
| 116 |
+
|
| 117 |
+
# Get distortion parameters.
|
| 118 |
+
type_ = cam.camera_type
|
| 119 |
+
if type_ == 0 or type_ == "SIMPLE_PINHOLE":
|
| 120 |
+
params = np.empty(0, dtype=np.float32)
|
| 121 |
+
camtype = "perspective"
|
| 122 |
+
elif type_ == 1 or type_ == "PINHOLE":
|
| 123 |
+
params = np.empty(0, dtype=np.float32)
|
| 124 |
+
camtype = "perspective"
|
| 125 |
+
if type_ == 2 or type_ == "SIMPLE_RADIAL":
|
| 126 |
+
params = np.array([cam.k1, 0.0, 0.0, 0.0], dtype=np.float32)
|
| 127 |
+
camtype = "perspective"
|
| 128 |
+
elif type_ == 3 or type_ == "RADIAL":
|
| 129 |
+
params = np.array([cam.k1, cam.k2, 0.0, 0.0], dtype=np.float32)
|
| 130 |
+
camtype = "perspective"
|
| 131 |
+
elif type_ == 4 or type_ == "OPENCV":
|
| 132 |
+
params = np.array([cam.k1, cam.k2, cam.p1, cam.p2], dtype=np.float32)
|
| 133 |
+
camtype = "perspective"
|
| 134 |
+
elif type_ == 5 or type_ == "OPENCV_FISHEYE":
|
| 135 |
+
params = np.array([cam.k1, cam.k2, cam.k3, cam.k4], dtype=np.float32)
|
| 136 |
+
camtype = "fisheye"
|
| 137 |
+
assert (
|
| 138 |
+
camtype == "perspective" or camtype == "fisheye"
|
| 139 |
+
), f"Only perspective and fisheye cameras are supported, got {type_}"
|
| 140 |
+
|
| 141 |
+
params_dict[camera_id] = params
|
| 142 |
+
imsize_dict[camera_id] = (cam.width // factor, cam.height // factor)
|
| 143 |
+
mask_dict[camera_id] = None
|
| 144 |
+
print(
|
| 145 |
+
f"[Parser] {len(imdata)} images, taken by {len(set(camera_ids))} cameras."
|
| 146 |
+
)
|
| 147 |
+
|
| 148 |
+
if len(imdata) == 0:
|
| 149 |
+
raise ValueError("No images found in COLMAP.")
|
| 150 |
+
if not (type_ == 0 or type_ == 1):
|
| 151 |
+
print("Warning: COLMAP Camera is not PINHOLE. Images have distortion.")
|
| 152 |
+
|
| 153 |
+
w2c_mats = np.stack(w2c_mats, axis=0)
|
| 154 |
+
|
| 155 |
+
# Convert extrinsics to camera-to-world.
|
| 156 |
+
camtoworlds = np.linalg.inv(w2c_mats)
|
| 157 |
+
|
| 158 |
+
# Image names from COLMAP. No need for permuting the poses according to
|
| 159 |
+
# image names anymore.
|
| 160 |
+
image_names = [imdata[k].name for k in imdata]
|
| 161 |
+
|
| 162 |
+
# Previous Nerf results were generated with images sorted by filename,
|
| 163 |
+
# ensure metrics are reported on the same test set.
|
| 164 |
+
inds = np.argsort(image_names)
|
| 165 |
+
image_names = [image_names[i] for i in inds]
|
| 166 |
+
camtoworlds = camtoworlds[inds]
|
| 167 |
+
camera_ids = [camera_ids[i] for i in inds]
|
| 168 |
+
|
| 169 |
+
# Load extended metadata. Used by Bilarf dataset.
|
| 170 |
+
self.extconf = {
|
| 171 |
+
"spiral_radius_scale": 1.0,
|
| 172 |
+
"no_factor_suffix": False,
|
| 173 |
+
}
|
| 174 |
+
extconf_file = os.path.join(data_dir, "ext_metadata.json")
|
| 175 |
+
if os.path.exists(extconf_file):
|
| 176 |
+
with open(extconf_file) as f:
|
| 177 |
+
self.extconf.update(json.load(f))
|
| 178 |
+
|
| 179 |
+
# Load bounds if possible (only used in forward facing scenes).
|
| 180 |
+
self.bounds = np.array([0.01, 1.0])
|
| 181 |
+
posefile = os.path.join(data_dir, "poses_bounds.npy")
|
| 182 |
+
if os.path.exists(posefile):
|
| 183 |
+
self.bounds = np.load(posefile)[:, -2:]
|
| 184 |
+
|
| 185 |
+
# Load images.
|
| 186 |
+
if factor > 1 and not self.extconf["no_factor_suffix"]:
|
| 187 |
+
image_dir_suffix = f"_{factor}"
|
| 188 |
+
else:
|
| 189 |
+
image_dir_suffix = ""
|
| 190 |
+
colmap_image_dir = os.path.join(data_dir, "images")
|
| 191 |
+
image_dir = os.path.join(data_dir, "images" + image_dir_suffix)
|
| 192 |
+
for d in [image_dir, colmap_image_dir]:
|
| 193 |
+
if not os.path.exists(d):
|
| 194 |
+
raise ValueError(f"Image folder {d} does not exist.")
|
| 195 |
+
|
| 196 |
+
# Downsampled images may have different names vs images used for COLMAP,
|
| 197 |
+
# so we need to map between the two sorted lists of files.
|
| 198 |
+
colmap_files = sorted(_get_rel_paths(colmap_image_dir))
|
| 199 |
+
image_files = sorted(_get_rel_paths(image_dir))
|
| 200 |
+
if factor > 1 and os.path.splitext(image_files[0])[1].lower() == ".jpg":
|
| 201 |
+
image_dir = _resize_image_folder(
|
| 202 |
+
colmap_image_dir, image_dir + "_png", factor=factor
|
| 203 |
+
)
|
| 204 |
+
image_files = sorted(_get_rel_paths(image_dir))
|
| 205 |
+
colmap_to_image = dict(zip(colmap_files, image_files))
|
| 206 |
+
image_paths = [os.path.join(image_dir, colmap_to_image[f]) for f in image_names]
|
| 207 |
+
|
| 208 |
+
# 3D points and {image_name -> [point_idx]}
|
| 209 |
+
points = manager.points3D.astype(np.float32)
|
| 210 |
+
points_err = manager.point3D_errors.astype(np.float32)
|
| 211 |
+
points_rgb = manager.point3D_colors.astype(np.uint8)
|
| 212 |
+
point_indices = dict()
|
| 213 |
+
|
| 214 |
+
image_id_to_name = {v: k for k, v in manager.name_to_image_id.items()}
|
| 215 |
+
for point_id, data in manager.point3D_id_to_images.items():
|
| 216 |
+
for image_id, _ in data:
|
| 217 |
+
image_name = image_id_to_name[image_id]
|
| 218 |
+
point_idx = manager.point3D_id_to_point3D_idx[point_id]
|
| 219 |
+
point_indices.setdefault(image_name, []).append(point_idx)
|
| 220 |
+
point_indices = {
|
| 221 |
+
k: np.array(v).astype(np.int32) for k, v in point_indices.items()
|
| 222 |
+
}
|
| 223 |
+
|
| 224 |
+
# Normalize the world space.
|
| 225 |
+
if normalize:
|
| 226 |
+
T1 = similarity_from_cameras(camtoworlds)
|
| 227 |
+
camtoworlds = transform_cameras(T1, camtoworlds)
|
| 228 |
+
points = transform_points(T1, points)
|
| 229 |
+
|
| 230 |
+
T2 = align_principal_axes(points)
|
| 231 |
+
camtoworlds = transform_cameras(T2, camtoworlds)
|
| 232 |
+
points = transform_points(T2, points)
|
| 233 |
+
|
| 234 |
+
transform = T2 @ T1
|
| 235 |
+
|
| 236 |
+
# Fix for up side down. We assume more points towards
|
| 237 |
+
# the bottom of the scene which is true when ground floor is
|
| 238 |
+
# present in the images.
|
| 239 |
+
if np.median(points[:, 2]) > np.mean(points[:, 2]):
|
| 240 |
+
# rotate 180 degrees around x axis such that z is flipped
|
| 241 |
+
T3 = np.array(
|
| 242 |
+
[
|
| 243 |
+
[1.0, 0.0, 0.0, 0.0],
|
| 244 |
+
[0.0, -1.0, 0.0, 0.0],
|
| 245 |
+
[0.0, 0.0, -1.0, 0.0],
|
| 246 |
+
[0.0, 0.0, 0.0, 1.0],
|
| 247 |
+
]
|
| 248 |
+
)
|
| 249 |
+
camtoworlds = transform_cameras(T3, camtoworlds)
|
| 250 |
+
points = transform_points(T3, points)
|
| 251 |
+
transform = T3 @ transform
|
| 252 |
+
else:
|
| 253 |
+
transform = np.eye(4)
|
| 254 |
+
|
| 255 |
+
self.image_names = image_names # List[str], (num_images,)
|
| 256 |
+
self.image_paths = image_paths # List[str], (num_images,)
|
| 257 |
+
self.camtoworlds = camtoworlds # np.ndarray, (num_images, 4, 4)
|
| 258 |
+
self.camera_ids = camera_ids # List[int], (num_images,)
|
| 259 |
+
self.Ks_dict = Ks_dict # Dict of camera_id -> K
|
| 260 |
+
self.params_dict = params_dict # Dict of camera_id -> params
|
| 261 |
+
self.imsize_dict = imsize_dict # Dict of camera_id -> (width, height)
|
| 262 |
+
self.mask_dict = mask_dict # Dict of camera_id -> mask
|
| 263 |
+
self.points = points # np.ndarray, (num_points, 3)
|
| 264 |
+
self.points_err = points_err # np.ndarray, (num_points,)
|
| 265 |
+
self.points_rgb = points_rgb # np.ndarray, (num_points, 3)
|
| 266 |
+
self.point_indices = point_indices # Dict[str, np.ndarray], image_name -> [M,]
|
| 267 |
+
self.transform = transform # np.ndarray, (4, 4)
|
| 268 |
+
|
| 269 |
+
# load one image to check the size. In the case of tanksandtemples dataset, the
|
| 270 |
+
# intrinsics stored in COLMAP corresponds to 2x upsampled images.
|
| 271 |
+
actual_image = imageio.imread(self.image_paths[0])[..., :3]
|
| 272 |
+
actual_height, actual_width = actual_image.shape[:2]
|
| 273 |
+
colmap_width, colmap_height = self.imsize_dict[self.camera_ids[0]]
|
| 274 |
+
s_height, s_width = actual_height / colmap_height, actual_width / colmap_width
|
| 275 |
+
for camera_id, K in self.Ks_dict.items():
|
| 276 |
+
K[0, :] *= s_width
|
| 277 |
+
K[1, :] *= s_height
|
| 278 |
+
self.Ks_dict[camera_id] = K
|
| 279 |
+
width, height = self.imsize_dict[camera_id]
|
| 280 |
+
self.imsize_dict[camera_id] = (int(width * s_width), int(height * s_height))
|
| 281 |
+
|
| 282 |
+
# undistortion
|
| 283 |
+
self.mapx_dict = dict()
|
| 284 |
+
self.mapy_dict = dict()
|
| 285 |
+
self.roi_undist_dict = dict()
|
| 286 |
+
for camera_id in self.params_dict.keys():
|
| 287 |
+
params = self.params_dict[camera_id]
|
| 288 |
+
if len(params) == 0:
|
| 289 |
+
continue # no distortion
|
| 290 |
+
assert camera_id in self.Ks_dict, f"Missing K for camera {camera_id}"
|
| 291 |
+
assert (
|
| 292 |
+
camera_id in self.params_dict
|
| 293 |
+
), f"Missing params for camera {camera_id}"
|
| 294 |
+
K = self.Ks_dict[camera_id]
|
| 295 |
+
width, height = self.imsize_dict[camera_id]
|
| 296 |
+
|
| 297 |
+
if camtype == "perspective":
|
| 298 |
+
K_undist, roi_undist = cv2.getOptimalNewCameraMatrix(
|
| 299 |
+
K, params, (width, height), 0
|
| 300 |
+
)
|
| 301 |
+
mapx, mapy = cv2.initUndistortRectifyMap(
|
| 302 |
+
K, params, None, K_undist, (width, height), cv2.CV_32FC1
|
| 303 |
+
)
|
| 304 |
+
mask = None
|
| 305 |
+
elif camtype == "fisheye":
|
| 306 |
+
fx = K[0, 0]
|
| 307 |
+
fy = K[1, 1]
|
| 308 |
+
cx = K[0, 2]
|
| 309 |
+
cy = K[1, 2]
|
| 310 |
+
grid_x, grid_y = np.meshgrid(
|
| 311 |
+
np.arange(width, dtype=np.float32),
|
| 312 |
+
np.arange(height, dtype=np.float32),
|
| 313 |
+
indexing="xy",
|
| 314 |
+
)
|
| 315 |
+
x1 = (grid_x - cx) / fx
|
| 316 |
+
y1 = (grid_y - cy) / fy
|
| 317 |
+
theta = np.sqrt(x1**2 + y1**2)
|
| 318 |
+
r = (
|
| 319 |
+
1.0
|
| 320 |
+
+ params[0] * theta**2
|
| 321 |
+
+ params[1] * theta**4
|
| 322 |
+
+ params[2] * theta**6
|
| 323 |
+
+ params[3] * theta**8
|
| 324 |
+
)
|
| 325 |
+
mapx = (fx * x1 * r + width // 2).astype(np.float32)
|
| 326 |
+
mapy = (fy * y1 * r + height // 2).astype(np.float32)
|
| 327 |
+
|
| 328 |
+
# Use mask to define ROI
|
| 329 |
+
mask = np.logical_and(
|
| 330 |
+
np.logical_and(mapx > 0, mapy > 0),
|
| 331 |
+
np.logical_and(mapx < width - 1, mapy < height - 1),
|
| 332 |
+
)
|
| 333 |
+
y_indices, x_indices = np.nonzero(mask)
|
| 334 |
+
y_min, y_max = y_indices.min(), y_indices.max() + 1
|
| 335 |
+
x_min, x_max = x_indices.min(), x_indices.max() + 1
|
| 336 |
+
mask = mask[y_min:y_max, x_min:x_max]
|
| 337 |
+
K_undist = K.copy()
|
| 338 |
+
K_undist[0, 2] -= x_min
|
| 339 |
+
K_undist[1, 2] -= y_min
|
| 340 |
+
roi_undist = [x_min, y_min, x_max - x_min, y_max - y_min]
|
| 341 |
+
else:
|
| 342 |
+
assert_never(camtype)
|
| 343 |
+
|
| 344 |
+
self.mapx_dict[camera_id] = mapx
|
| 345 |
+
self.mapy_dict[camera_id] = mapy
|
| 346 |
+
self.Ks_dict[camera_id] = K_undist
|
| 347 |
+
self.roi_undist_dict[camera_id] = roi_undist
|
| 348 |
+
self.imsize_dict[camera_id] = (roi_undist[2], roi_undist[3])
|
| 349 |
+
self.mask_dict[camera_id] = mask
|
| 350 |
+
|
| 351 |
+
# size of the scene measured by cameras
|
| 352 |
+
camera_locations = camtoworlds[:, :3, 3]
|
| 353 |
+
scene_center = np.mean(camera_locations, axis=0)
|
| 354 |
+
dists = np.linalg.norm(camera_locations - scene_center, axis=1)
|
| 355 |
+
self.scene_scale = np.max(dists)
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
class Dataset:
|
| 359 |
+
"""A simple dataset class."""
|
| 360 |
+
|
| 361 |
+
def __init__(
|
| 362 |
+
self,
|
| 363 |
+
# parser: Parser,
|
| 364 |
+
images: np.ndarray,
|
| 365 |
+
camtoworlds: np.ndarray,
|
| 366 |
+
Ks: np.ndarray,
|
| 367 |
+
split: str = "train",
|
| 368 |
+
patch_size: Optional[int] = None,
|
| 369 |
+
load_depths: bool = False,
|
| 370 |
+
):
|
| 371 |
+
# self.parser = parser
|
| 372 |
+
self.split = split
|
| 373 |
+
self.patch_size = patch_size
|
| 374 |
+
self.load_depths = load_depths
|
| 375 |
+
self.images = images
|
| 376 |
+
self.camtoworlds = camtoworlds
|
| 377 |
+
self.Ks = Ks
|
| 378 |
+
H, W = self.images.shape[-2:]
|
| 379 |
+
self.Ks[:, 0, :] *= W
|
| 380 |
+
self.Ks[:, 1, :] *= H
|
| 381 |
+
self.indices = np.arange(len(images))
|
| 382 |
+
# indices = np.arange(len(self.parser.image_names))
|
| 383 |
+
# if split == "train":
|
| 384 |
+
# self.indices = indices[indices % self.parser.test_every != 0]
|
| 385 |
+
# else:
|
| 386 |
+
# self.indices = indices[indices % self.parser.test_every == 0]
|
| 387 |
+
|
| 388 |
+
# if split == "train":
|
| 389 |
+
# self.images = np.load(os.path.join(self.parser.true_data_dir, "context_image.npy"))
|
| 390 |
+
# self.camtoworlds = np.load(os.path.join(self.parser.true_data_dir, "context_extrinsic.npy"))
|
| 391 |
+
# self.Ks = np.load(os.path.join(self.parser.true_data_dir, "context_intrinsic.npy"))
|
| 392 |
+
# H, W = self.images.shape[-2:]
|
| 393 |
+
# self.Ks[:, 0, :] *= W
|
| 394 |
+
# self.Ks[:, 1, :] *= H
|
| 395 |
+
# self.indices = np.arange(len(self.images))
|
| 396 |
+
# else:
|
| 397 |
+
# self.images = np.load(os.path.join(self.parser.true_data_dir, "target_image.npy"))
|
| 398 |
+
# self.camtoworlds = np.load(os.path.join(self.parser.true_data_dir, "target_extrinsic.npy"))
|
| 399 |
+
# self.Ks = np.load(os.path.join(self.parser.true_data_dir, "target_intrinsic.npy"))
|
| 400 |
+
# H, W = self.images.shape[-2:]
|
| 401 |
+
# self.Ks[:, 0, :] *= W
|
| 402 |
+
# self.Ks[:, 1, :] *= H
|
| 403 |
+
# self.indices = np.arange(len(self.images))
|
| 404 |
+
|
| 405 |
+
def __len__(self):
|
| 406 |
+
return len(self.indices)
|
| 407 |
+
|
| 408 |
+
def __getitem__(self, item: int) -> Dict[str, Any]:
|
| 409 |
+
index = self.indices[item]
|
| 410 |
+
image = (self.images[index]*255.0).transpose(1, 2, 0).astype(np.uint8) # (H, W, 3)
|
| 411 |
+
K = self.Ks[index].copy() # undistorted K
|
| 412 |
+
params = None
|
| 413 |
+
camtoworlds = self.camtoworlds[index]
|
| 414 |
+
mask = None
|
| 415 |
+
|
| 416 |
+
if self.patch_size is not None:
|
| 417 |
+
# Random crop.
|
| 418 |
+
h, w = image.shape[:2]
|
| 419 |
+
x = np.random.randint(0, max(w - self.patch_size, 1))
|
| 420 |
+
y = np.random.randint(0, max(h - self.patch_size, 1))
|
| 421 |
+
image = image[y : y + self.patch_size, x : x + self.patch_size]
|
| 422 |
+
K[0, 2] -= x
|
| 423 |
+
K[1, 2] -= y
|
| 424 |
+
|
| 425 |
+
data = {
|
| 426 |
+
"K": torch.from_numpy(K).float(),
|
| 427 |
+
"camtoworld": torch.from_numpy(camtoworlds).float(),
|
| 428 |
+
"image": torch.from_numpy(image).float(),
|
| 429 |
+
"image_id": item, # the index of the image in the dataset
|
| 430 |
+
}
|
| 431 |
+
if mask is not None:
|
| 432 |
+
data["mask"] = torch.from_numpy(mask).bool()
|
| 433 |
+
|
| 434 |
+
if self.load_depths and False:
|
| 435 |
+
# projected points to image plane to get depths
|
| 436 |
+
worldtocams = np.linalg.inv(camtoworlds)
|
| 437 |
+
image_name = self.parser.image_names[index]
|
| 438 |
+
point_indices = self.parser.point_indices[image_name]
|
| 439 |
+
points_world = self.parser.points[point_indices]
|
| 440 |
+
points_cam = (worldtocams[:3, :3] @ points_world.T + worldtocams[:3, 3:4]).T
|
| 441 |
+
points_proj = (K @ points_cam.T).T
|
| 442 |
+
points = points_proj[:, :2] / points_proj[:, 2:3] # (M, 2)
|
| 443 |
+
depths = points_cam[:, 2] # (M,)
|
| 444 |
+
# filter out points outside the image
|
| 445 |
+
selector = (
|
| 446 |
+
(points[:, 0] >= 0)
|
| 447 |
+
& (points[:, 0] < image.shape[1])
|
| 448 |
+
& (points[:, 1] >= 0)
|
| 449 |
+
& (points[:, 1] < image.shape[0])
|
| 450 |
+
& (depths > 0)
|
| 451 |
+
)
|
| 452 |
+
points = points[selector]
|
| 453 |
+
depths = depths[selector]
|
| 454 |
+
data["points"] = torch.from_numpy(points).float()
|
| 455 |
+
data["depths"] = torch.from_numpy(depths).float()
|
| 456 |
+
|
| 457 |
+
return data
|
| 458 |
+
|
| 459 |
+
|
| 460 |
+
if __name__ == "__main__":
|
| 461 |
+
import argparse
|
| 462 |
+
|
| 463 |
+
import imageio.v2 as imageio
|
| 464 |
+
|
| 465 |
+
parser = argparse.ArgumentParser()
|
| 466 |
+
parser.add_argument("--data_dir", type=str, default="data/mipnerf360/garden")
|
| 467 |
+
parser.add_argument("--true_data_dir", type=str, default="/cpfs01/user/jianglihan/projects/anysplat_baselines/demo_data/infer_output/3F_100view/room5")
|
| 468 |
+
parser.add_argument("--factor", type=int, default=4)
|
| 469 |
+
args = parser.parse_args()
|
| 470 |
+
|
| 471 |
+
# Parse COLMAP data.
|
| 472 |
+
parser = Parser(
|
| 473 |
+
data_dir=args.data_dir,
|
| 474 |
+
true_data_dir=args.true_data_dir,
|
| 475 |
+
factor=args.factor,
|
| 476 |
+
normalize=True,
|
| 477 |
+
test_every=8
|
| 478 |
+
)
|
| 479 |
+
dataset = Dataset(parser, split="train", load_depths=True)
|
| 480 |
+
print(f"Dataset: {len(dataset)} images.")
|
| 481 |
+
|
| 482 |
+
writer = imageio.get_writer("results/points.mp4", fps=30)
|
| 483 |
+
for data in tqdm(dataset, desc="Plotting points"):
|
| 484 |
+
image = data["image"].numpy().astype(np.uint8)
|
| 485 |
+
points = data["points"].numpy()
|
| 486 |
+
depths = data["depths"].numpy()
|
| 487 |
+
for x, y in points:
|
| 488 |
+
cv2.circle(image, (int(x), int(y)), 2, (255, 0, 0), -1)
|
| 489 |
+
writer.append_data(image)
|
| 490 |
+
writer.close()
|
src/post_opt/datasets/normalize.py
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def similarity_from_cameras(c2w, strict_scaling=False, center_method="focus"):
|
| 5 |
+
"""
|
| 6 |
+
reference: nerf-factory
|
| 7 |
+
Get a similarity transform to normalize dataset
|
| 8 |
+
from c2w (OpenCV convention) cameras
|
| 9 |
+
:param c2w: (N, 4)
|
| 10 |
+
:return T (4,4) , scale (float)
|
| 11 |
+
"""
|
| 12 |
+
t = c2w[:, :3, 3]
|
| 13 |
+
R = c2w[:, :3, :3]
|
| 14 |
+
|
| 15 |
+
# (1) Rotate the world so that z+ is the up axis
|
| 16 |
+
# we estimate the up axis by averaging the camera up axes
|
| 17 |
+
ups = np.sum(R * np.array([0, -1.0, 0]), axis=-1)
|
| 18 |
+
world_up = np.mean(ups, axis=0)
|
| 19 |
+
world_up /= np.linalg.norm(world_up)
|
| 20 |
+
|
| 21 |
+
up_camspace = np.array([0.0, -1.0, 0.0])
|
| 22 |
+
c = (up_camspace * world_up).sum()
|
| 23 |
+
cross = np.cross(world_up, up_camspace)
|
| 24 |
+
skew = np.array(
|
| 25 |
+
[
|
| 26 |
+
[0.0, -cross[2], cross[1]],
|
| 27 |
+
[cross[2], 0.0, -cross[0]],
|
| 28 |
+
[-cross[1], cross[0], 0.0],
|
| 29 |
+
]
|
| 30 |
+
)
|
| 31 |
+
if c > -1:
|
| 32 |
+
R_align = np.eye(3) + skew + (skew @ skew) * 1 / (1 + c)
|
| 33 |
+
else:
|
| 34 |
+
# In the unlikely case the original data has y+ up axis,
|
| 35 |
+
# rotate 180-deg about x axis
|
| 36 |
+
R_align = np.array([[-1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]])
|
| 37 |
+
|
| 38 |
+
# R_align = np.eye(3) # DEBUG
|
| 39 |
+
R = R_align @ R
|
| 40 |
+
fwds = np.sum(R * np.array([0, 0.0, 1.0]), axis=-1)
|
| 41 |
+
t = (R_align @ t[..., None])[..., 0]
|
| 42 |
+
|
| 43 |
+
# (2) Recenter the scene.
|
| 44 |
+
if center_method == "focus":
|
| 45 |
+
# find the closest point to the origin for each camera's center ray
|
| 46 |
+
nearest = t + (fwds * -t).sum(-1)[:, None] * fwds
|
| 47 |
+
translate = -np.median(nearest, axis=0)
|
| 48 |
+
elif center_method == "poses":
|
| 49 |
+
# use center of the camera positions
|
| 50 |
+
translate = -np.median(t, axis=0)
|
| 51 |
+
else:
|
| 52 |
+
raise ValueError(f"Unknown center_method {center_method}")
|
| 53 |
+
|
| 54 |
+
transform = np.eye(4)
|
| 55 |
+
transform[:3, 3] = translate
|
| 56 |
+
transform[:3, :3] = R_align
|
| 57 |
+
|
| 58 |
+
# (3) Rescale the scene using camera distances
|
| 59 |
+
scale_fn = np.max if strict_scaling else np.median
|
| 60 |
+
scale = 1.0 / scale_fn(np.linalg.norm(t + translate, axis=-1))
|
| 61 |
+
transform[:3, :] *= scale
|
| 62 |
+
|
| 63 |
+
return transform
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def align_principal_axes(point_cloud):
|
| 67 |
+
# Compute centroid
|
| 68 |
+
centroid = np.median(point_cloud, axis=0)
|
| 69 |
+
|
| 70 |
+
# Translate point cloud to centroid
|
| 71 |
+
translated_point_cloud = point_cloud - centroid
|
| 72 |
+
|
| 73 |
+
# Compute covariance matrix
|
| 74 |
+
covariance_matrix = np.cov(translated_point_cloud, rowvar=False)
|
| 75 |
+
|
| 76 |
+
# Compute eigenvectors and eigenvalues
|
| 77 |
+
eigenvalues, eigenvectors = np.linalg.eigh(covariance_matrix)
|
| 78 |
+
|
| 79 |
+
# Sort eigenvectors by eigenvalues (descending order) so that the z-axis
|
| 80 |
+
# is the principal axis with the smallest eigenvalue.
|
| 81 |
+
sort_indices = eigenvalues.argsort()[::-1]
|
| 82 |
+
eigenvectors = eigenvectors[:, sort_indices]
|
| 83 |
+
|
| 84 |
+
# Check orientation of eigenvectors. If the determinant of the eigenvectors is
|
| 85 |
+
# negative, then we need to flip the sign of one of the eigenvectors.
|
| 86 |
+
if np.linalg.det(eigenvectors) < 0:
|
| 87 |
+
eigenvectors[:, 0] *= -1
|
| 88 |
+
|
| 89 |
+
# Create rotation matrix
|
| 90 |
+
rotation_matrix = eigenvectors.T
|
| 91 |
+
|
| 92 |
+
# Create SE(3) matrix (4x4 transformation matrix)
|
| 93 |
+
transform = np.eye(4)
|
| 94 |
+
transform[:3, :3] = rotation_matrix
|
| 95 |
+
transform[:3, 3] = -rotation_matrix @ centroid
|
| 96 |
+
|
| 97 |
+
return transform
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def transform_points(matrix, points):
|
| 101 |
+
"""Transform points using an SE(3) matrix.
|
| 102 |
+
|
| 103 |
+
Args:
|
| 104 |
+
matrix: 4x4 SE(3) matrix
|
| 105 |
+
points: Nx3 array of points
|
| 106 |
+
|
| 107 |
+
Returns:
|
| 108 |
+
Nx3 array of transformed points
|
| 109 |
+
"""
|
| 110 |
+
assert matrix.shape == (4, 4)
|
| 111 |
+
assert len(points.shape) == 2 and points.shape[1] == 3
|
| 112 |
+
return points @ matrix[:3, :3].T + matrix[:3, 3]
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def transform_cameras(matrix, camtoworlds):
|
| 116 |
+
"""Transform cameras using an SE(3) matrix.
|
| 117 |
+
|
| 118 |
+
Args:
|
| 119 |
+
matrix: 4x4 SE(3) matrix
|
| 120 |
+
camtoworlds: Nx4x4 array of camera-to-world matrices
|
| 121 |
+
|
| 122 |
+
Returns:
|
| 123 |
+
Nx4x4 array of transformed camera-to-world matrices
|
| 124 |
+
"""
|
| 125 |
+
assert matrix.shape == (4, 4)
|
| 126 |
+
assert len(camtoworlds.shape) == 3 and camtoworlds.shape[1:] == (4, 4)
|
| 127 |
+
camtoworlds = np.einsum("nij, ki -> nkj", camtoworlds, matrix)
|
| 128 |
+
scaling = np.linalg.norm(camtoworlds[:, 0, :3], axis=1)
|
| 129 |
+
camtoworlds[:, :3, :3] = camtoworlds[:, :3, :3] / scaling[:, None, None]
|
| 130 |
+
return camtoworlds
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def normalize(camtoworlds, points=None):
|
| 134 |
+
T1 = similarity_from_cameras(camtoworlds)
|
| 135 |
+
camtoworlds = transform_cameras(T1, camtoworlds)
|
| 136 |
+
if points is not None:
|
| 137 |
+
points = transform_points(T1, points)
|
| 138 |
+
T2 = align_principal_axes(points)
|
| 139 |
+
camtoworlds = transform_cameras(T2, camtoworlds)
|
| 140 |
+
points = transform_points(T2, points)
|
| 141 |
+
return camtoworlds, points, T2 @ T1
|
| 142 |
+
else:
|
| 143 |
+
return camtoworlds, T1
|
src/post_opt/datasets/traj.py
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Code borrowed from
|
| 3 |
+
|
| 4 |
+
https://github.com/google-research/multinerf/blob/5b4d4f64608ec8077222c52fdf814d40acc10bc1/internal/camera_utils.py
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
import scipy
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def normalize(x: np.ndarray) -> np.ndarray:
|
| 12 |
+
"""Normalization helper function."""
|
| 13 |
+
return x / np.linalg.norm(x)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def viewmatrix(lookdir: np.ndarray, up: np.ndarray, position: np.ndarray) -> np.ndarray:
|
| 17 |
+
"""Construct lookat view matrix."""
|
| 18 |
+
vec2 = normalize(lookdir)
|
| 19 |
+
vec0 = normalize(np.cross(up, vec2))
|
| 20 |
+
vec1 = normalize(np.cross(vec2, vec0))
|
| 21 |
+
m = np.stack([vec0, vec1, vec2, position], axis=1)
|
| 22 |
+
return m
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def focus_point_fn(poses: np.ndarray) -> np.ndarray:
|
| 26 |
+
"""Calculate nearest point to all focal axes in poses."""
|
| 27 |
+
directions, origins = poses[:, :3, 2:3], poses[:, :3, 3:4]
|
| 28 |
+
m = np.eye(3) - directions * np.transpose(directions, [0, 2, 1])
|
| 29 |
+
mt_m = np.transpose(m, [0, 2, 1]) @ m
|
| 30 |
+
focus_pt = np.linalg.inv(mt_m.mean(0)) @ (mt_m @ origins).mean(0)[:, 0]
|
| 31 |
+
return focus_pt
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def average_pose(poses: np.ndarray) -> np.ndarray:
|
| 35 |
+
"""New pose using average position, z-axis, and up vector of input poses."""
|
| 36 |
+
position = poses[:, :3, 3].mean(0)
|
| 37 |
+
z_axis = poses[:, :3, 2].mean(0)
|
| 38 |
+
up = poses[:, :3, 1].mean(0)
|
| 39 |
+
cam2world = viewmatrix(z_axis, up, position)
|
| 40 |
+
return cam2world
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def generate_spiral_path(
|
| 44 |
+
poses,
|
| 45 |
+
bounds,
|
| 46 |
+
n_frames=120,
|
| 47 |
+
n_rots=2,
|
| 48 |
+
zrate=0.5,
|
| 49 |
+
spiral_scale_f=1.0,
|
| 50 |
+
spiral_scale_r=1.0,
|
| 51 |
+
focus_distance=0.75,
|
| 52 |
+
):
|
| 53 |
+
"""Calculates a forward facing spiral path for rendering."""
|
| 54 |
+
# Find a reasonable 'focus depth' for this dataset as a weighted average
|
| 55 |
+
# of conservative near and far bounds in disparity space.
|
| 56 |
+
near_bound = bounds.min()
|
| 57 |
+
far_bound = bounds.max()
|
| 58 |
+
# All cameras will point towards the world space point (0, 0, -focal).
|
| 59 |
+
focal = 1 / (((1 - focus_distance) / near_bound + focus_distance / far_bound))
|
| 60 |
+
focal = focal * spiral_scale_f
|
| 61 |
+
|
| 62 |
+
# Get radii for spiral path using 90th percentile of camera positions.
|
| 63 |
+
positions = poses[:, :3, 3]
|
| 64 |
+
radii = np.percentile(np.abs(positions), 90, 0)
|
| 65 |
+
radii = radii * spiral_scale_r
|
| 66 |
+
radii = np.concatenate([radii, [1.0]])
|
| 67 |
+
|
| 68 |
+
# Generate poses for spiral path.
|
| 69 |
+
render_poses = []
|
| 70 |
+
cam2world = average_pose(poses)
|
| 71 |
+
up = poses[:, :3, 1].mean(0)
|
| 72 |
+
for theta in np.linspace(0.0, 2.0 * np.pi * n_rots, n_frames, endpoint=False):
|
| 73 |
+
t = radii * [np.cos(theta), -np.sin(theta), -np.sin(theta * zrate), 1.0]
|
| 74 |
+
position = cam2world @ t
|
| 75 |
+
lookat = cam2world @ [0, 0, -focal, 1.0]
|
| 76 |
+
z_axis = position - lookat
|
| 77 |
+
render_poses.append(viewmatrix(z_axis, up, position))
|
| 78 |
+
render_poses = np.stack(render_poses, axis=0)
|
| 79 |
+
return render_poses
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def generate_ellipse_path_z(
|
| 83 |
+
poses: np.ndarray,
|
| 84 |
+
n_frames: int = 120,
|
| 85 |
+
# const_speed: bool = True,
|
| 86 |
+
variation: float = 0.0,
|
| 87 |
+
phase: float = 0.0,
|
| 88 |
+
height: float = 0.0,
|
| 89 |
+
) -> np.ndarray:
|
| 90 |
+
"""Generate an elliptical render path based on the given poses."""
|
| 91 |
+
# Calculate the focal point for the path (cameras point toward this).
|
| 92 |
+
center = focus_point_fn(poses)
|
| 93 |
+
# Path height sits at z=height (in middle of zero-mean capture pattern).
|
| 94 |
+
offset = np.array([center[0], center[1], height])
|
| 95 |
+
|
| 96 |
+
# Calculate scaling for ellipse axes based on input camera positions.
|
| 97 |
+
sc = np.percentile(np.abs(poses[:, :3, 3] - offset), 90, axis=0)
|
| 98 |
+
# Use ellipse that is symmetric about the focal point in xy.
|
| 99 |
+
low = -sc + offset
|
| 100 |
+
high = sc + offset
|
| 101 |
+
# Optional height variation need not be symmetric
|
| 102 |
+
z_low = np.percentile((poses[:, :3, 3]), 10, axis=0)
|
| 103 |
+
z_high = np.percentile((poses[:, :3, 3]), 90, axis=0)
|
| 104 |
+
|
| 105 |
+
def get_positions(theta):
|
| 106 |
+
# Interpolate between bounds with trig functions to get ellipse in x-y.
|
| 107 |
+
# Optionally also interpolate in z to change camera height along path.
|
| 108 |
+
return np.stack(
|
| 109 |
+
[
|
| 110 |
+
low[0] + (high - low)[0] * (np.cos(theta) * 0.5 + 0.5),
|
| 111 |
+
low[1] + (high - low)[1] * (np.sin(theta) * 0.5 + 0.5),
|
| 112 |
+
variation
|
| 113 |
+
* (
|
| 114 |
+
z_low[2]
|
| 115 |
+
+ (z_high - z_low)[2]
|
| 116 |
+
* (np.cos(theta + 2 * np.pi * phase) * 0.5 + 0.5)
|
| 117 |
+
)
|
| 118 |
+
+ height,
|
| 119 |
+
],
|
| 120 |
+
-1,
|
| 121 |
+
)
|
| 122 |
+
|
| 123 |
+
theta = np.linspace(0, 2.0 * np.pi, n_frames + 1, endpoint=True)
|
| 124 |
+
positions = get_positions(theta)
|
| 125 |
+
|
| 126 |
+
# if const_speed:
|
| 127 |
+
# # Resample theta angles so that the velocity is closer to constant.
|
| 128 |
+
# lengths = np.linalg.norm(positions[1:] - positions[:-1], axis=-1)
|
| 129 |
+
# theta = stepfun.sample(None, theta, np.log(lengths), n_frames + 1)
|
| 130 |
+
# positions = get_positions(theta)
|
| 131 |
+
|
| 132 |
+
# Throw away duplicated last position.
|
| 133 |
+
positions = positions[:-1]
|
| 134 |
+
|
| 135 |
+
# Set path's up vector to axis closest to average of input pose up vectors.
|
| 136 |
+
avg_up = poses[:, :3, 1].mean(0)
|
| 137 |
+
avg_up = avg_up / np.linalg.norm(avg_up)
|
| 138 |
+
ind_up = np.argmax(np.abs(avg_up))
|
| 139 |
+
up = np.eye(3)[ind_up] * np.sign(avg_up[ind_up])
|
| 140 |
+
|
| 141 |
+
return np.stack([viewmatrix(center - p, up, p) for p in positions])
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def generate_ellipse_path_y(
|
| 145 |
+
poses: np.ndarray,
|
| 146 |
+
n_frames: int = 120,
|
| 147 |
+
# const_speed: bool = True,
|
| 148 |
+
variation: float = 0.0,
|
| 149 |
+
phase: float = 0.0,
|
| 150 |
+
height: float = 0.0,
|
| 151 |
+
) -> np.ndarray:
|
| 152 |
+
"""Generate an elliptical render path based on the given poses."""
|
| 153 |
+
# Calculate the focal point for the path (cameras point toward this).
|
| 154 |
+
center = focus_point_fn(poses)
|
| 155 |
+
# Path height sits at y=height (in middle of zero-mean capture pattern).
|
| 156 |
+
offset = np.array([center[0], height, center[2]])
|
| 157 |
+
|
| 158 |
+
# Calculate scaling for ellipse axes based on input camera positions.
|
| 159 |
+
sc = np.percentile(np.abs(poses[:, :3, 3] - offset), 90, axis=0)
|
| 160 |
+
# Use ellipse that is symmetric about the focal point in xy.
|
| 161 |
+
low = -sc + offset
|
| 162 |
+
high = sc + offset
|
| 163 |
+
# Optional height variation need not be symmetric
|
| 164 |
+
y_low = np.percentile((poses[:, :3, 3]), 10, axis=0)
|
| 165 |
+
y_high = np.percentile((poses[:, :3, 3]), 90, axis=0)
|
| 166 |
+
|
| 167 |
+
def get_positions(theta):
|
| 168 |
+
# Interpolate between bounds with trig functions to get ellipse in x-z.
|
| 169 |
+
# Optionally also interpolate in y to change camera height along path.
|
| 170 |
+
return np.stack(
|
| 171 |
+
[
|
| 172 |
+
low[0] + (high - low)[0] * (np.cos(theta) * 0.5 + 0.5),
|
| 173 |
+
variation
|
| 174 |
+
* (
|
| 175 |
+
y_low[1]
|
| 176 |
+
+ (y_high - y_low)[1]
|
| 177 |
+
* (np.cos(theta + 2 * np.pi * phase) * 0.5 + 0.5)
|
| 178 |
+
)
|
| 179 |
+
+ height,
|
| 180 |
+
low[2] + (high - low)[2] * (np.sin(theta) * 0.5 + 0.5),
|
| 181 |
+
],
|
| 182 |
+
-1,
|
| 183 |
+
)
|
| 184 |
+
|
| 185 |
+
theta = np.linspace(0, 2.0 * np.pi, n_frames + 1, endpoint=True)
|
| 186 |
+
positions = get_positions(theta)
|
| 187 |
+
|
| 188 |
+
# if const_speed:
|
| 189 |
+
# # Resample theta angles so that the velocity is closer to constant.
|
| 190 |
+
# lengths = np.linalg.norm(positions[1:] - positions[:-1], axis=-1)
|
| 191 |
+
# theta = stepfun.sample(None, theta, np.log(lengths), n_frames + 1)
|
| 192 |
+
# positions = get_positions(theta)
|
| 193 |
+
|
| 194 |
+
# Throw away duplicated last position.
|
| 195 |
+
positions = positions[:-1]
|
| 196 |
+
|
| 197 |
+
# Set path's up vector to axis closest to average of input pose up vectors.
|
| 198 |
+
avg_up = poses[:, :3, 1].mean(0)
|
| 199 |
+
avg_up = avg_up / np.linalg.norm(avg_up)
|
| 200 |
+
ind_up = np.argmax(np.abs(avg_up))
|
| 201 |
+
up = np.eye(3)[ind_up] * np.sign(avg_up[ind_up])
|
| 202 |
+
|
| 203 |
+
return np.stack([viewmatrix(p - center, up, p) for p in positions])
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def generate_interpolated_path(
|
| 207 |
+
poses: np.ndarray,
|
| 208 |
+
n_interp: int,
|
| 209 |
+
spline_degree: int = 5,
|
| 210 |
+
smoothness: float = 0.03,
|
| 211 |
+
rot_weight: float = 0.1,
|
| 212 |
+
):
|
| 213 |
+
"""Creates a smooth spline path between input keyframe camera poses.
|
| 214 |
+
|
| 215 |
+
Spline is calculated with poses in format (position, lookat-point, up-point).
|
| 216 |
+
|
| 217 |
+
Args:
|
| 218 |
+
poses: (n, 3, 4) array of input pose keyframes.
|
| 219 |
+
n_interp: returned path will have n_interp * (n - 1) total poses.
|
| 220 |
+
spline_degree: polynomial degree of B-spline.
|
| 221 |
+
smoothness: parameter for spline smoothing, 0 forces exact interpolation.
|
| 222 |
+
rot_weight: relative weighting of rotation/translation in spline solve.
|
| 223 |
+
|
| 224 |
+
Returns:
|
| 225 |
+
Array of new camera poses with shape (n_interp * (n - 1), 3, 4).
|
| 226 |
+
"""
|
| 227 |
+
|
| 228 |
+
def poses_to_points(poses, dist):
|
| 229 |
+
"""Converts from pose matrices to (position, lookat, up) format."""
|
| 230 |
+
pos = poses[:, :3, -1]
|
| 231 |
+
lookat = poses[:, :3, -1] - dist * poses[:, :3, 2]
|
| 232 |
+
up = poses[:, :3, -1] + dist * poses[:, :3, 1]
|
| 233 |
+
return np.stack([pos, lookat, up], 1)
|
| 234 |
+
|
| 235 |
+
def points_to_poses(points):
|
| 236 |
+
"""Converts from (position, lookat, up) format to pose matrices."""
|
| 237 |
+
return np.array([viewmatrix(p - l, u - p, p) for p, l, u in points])
|
| 238 |
+
|
| 239 |
+
def interp(points, n, k, s):
|
| 240 |
+
"""Runs multidimensional B-spline interpolation on the input points."""
|
| 241 |
+
sh = points.shape
|
| 242 |
+
pts = np.reshape(points, (sh[0], -1))
|
| 243 |
+
k = min(k, sh[0] - 1)
|
| 244 |
+
tck, _ = scipy.interpolate.splprep(pts.T, k=k, s=s)
|
| 245 |
+
u = np.linspace(0, 1, n, endpoint=False)
|
| 246 |
+
new_points = np.array(scipy.interpolate.splev(u, tck))
|
| 247 |
+
new_points = np.reshape(new_points.T, (n, sh[1], sh[2]))
|
| 248 |
+
return new_points
|
| 249 |
+
|
| 250 |
+
points = poses_to_points(poses, dist=rot_weight)
|
| 251 |
+
new_points = interp(
|
| 252 |
+
points, n_interp * (points.shape[0] - 1), k=spline_degree, s=smoothness
|
| 253 |
+
)
|
| 254 |
+
return points_to_poses(new_points)
|
src/post_opt/exporter.py
ADDED
|
@@ -0,0 +1,553 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
import struct
|
| 3 |
+
from io import BytesIO
|
| 4 |
+
from typing import Literal, Optional
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def sh2rgb(sh: torch.Tensor) -> torch.Tensor:
|
| 11 |
+
"""Convert Sphere Harmonics to RGB
|
| 12 |
+
|
| 13 |
+
Args:
|
| 14 |
+
sh (torch.Tensor): SH tensor
|
| 15 |
+
|
| 16 |
+
Returns:
|
| 17 |
+
torch.Tensor: RGB tensor
|
| 18 |
+
"""
|
| 19 |
+
C0 = 0.28209479177387814
|
| 20 |
+
return sh * C0 + 0.5
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def part1by2_vec(x: torch.Tensor) -> torch.Tensor:
|
| 24 |
+
"""Interleave bits of x with 0s
|
| 25 |
+
|
| 26 |
+
Args:
|
| 27 |
+
x (torch.Tensor): Input tensor. Shape (N,)
|
| 28 |
+
|
| 29 |
+
Returns:
|
| 30 |
+
torch.Tensor: Output tensor. Shape (N,)
|
| 31 |
+
"""
|
| 32 |
+
|
| 33 |
+
x = x & 0x000003FF
|
| 34 |
+
x = (x ^ (x << 16)) & 0xFF0000FF
|
| 35 |
+
x = (x ^ (x << 8)) & 0x0300F00F
|
| 36 |
+
x = (x ^ (x << 4)) & 0x030C30C3
|
| 37 |
+
x = (x ^ (x << 2)) & 0x09249249
|
| 38 |
+
return x
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def encode_morton3_vec(
|
| 42 |
+
x: torch.Tensor, y: torch.Tensor, z: torch.Tensor
|
| 43 |
+
) -> torch.Tensor:
|
| 44 |
+
"""Compute Morton codes for 3D coordinates
|
| 45 |
+
|
| 46 |
+
Args:
|
| 47 |
+
x (torch.Tensor): X coordinates. Shape (N,)
|
| 48 |
+
y (torch.Tensor): Y coordinates. Shape (N,)
|
| 49 |
+
z (torch.Tensor): Z coordinates. Shape (N,)
|
| 50 |
+
Returns:
|
| 51 |
+
torch.Tensor: Morton codes. Shape (N,)
|
| 52 |
+
"""
|
| 53 |
+
return (part1by2_vec(z) << 2) + (part1by2_vec(y) << 1) + part1by2_vec(x)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def sort_centers(centers: torch.Tensor, indices: torch.Tensor) -> torch.Tensor:
|
| 57 |
+
"""Sort centers based on Morton codes
|
| 58 |
+
|
| 59 |
+
Args:
|
| 60 |
+
centers (torch.Tensor): Centers. Shape (N, 3)
|
| 61 |
+
indices (torch.Tensor): Indices. Shape (N,)
|
| 62 |
+
Returns:
|
| 63 |
+
torch.Tensor: Sorted indices. Shape (N,)
|
| 64 |
+
"""
|
| 65 |
+
# Compute min and max values in a single operation
|
| 66 |
+
min_vals, _ = torch.min(centers, dim=0)
|
| 67 |
+
max_vals, _ = torch.max(centers, dim=0)
|
| 68 |
+
|
| 69 |
+
# Compute the scaling factors
|
| 70 |
+
lengths = max_vals - min_vals
|
| 71 |
+
lengths[lengths == 0] = 1 # Prevent division by zero
|
| 72 |
+
|
| 73 |
+
# Normalize and scale to 10-bit integer range (0-1024)
|
| 74 |
+
scaled_centers = ((centers - min_vals) / lengths * 1024).floor().to(torch.int32)
|
| 75 |
+
|
| 76 |
+
# Extract x, y, z coordinates
|
| 77 |
+
x, y, z = scaled_centers[:, 0], scaled_centers[:, 1], scaled_centers[:, 2]
|
| 78 |
+
|
| 79 |
+
# Compute Morton codes using vectorized operations
|
| 80 |
+
morton = encode_morton3_vec(x, y, z)
|
| 81 |
+
|
| 82 |
+
# Sort indices based on Morton codes
|
| 83 |
+
sorted_indices = indices[torch.argsort(morton).to(indices.device)]
|
| 84 |
+
|
| 85 |
+
return sorted_indices
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def pack_unorm(value: torch.Tensor, bits: int) -> torch.Tensor:
|
| 89 |
+
"""Pack a floating point value into an unsigned integer with a given number of bits.
|
| 90 |
+
|
| 91 |
+
Args:
|
| 92 |
+
value (torch.Tensor): Floating point value to pack. Shape (N,)
|
| 93 |
+
bits (int): Number of bits to pack into.
|
| 94 |
+
|
| 95 |
+
Returns:
|
| 96 |
+
torch.Tensor: Packed value. Shape (N,)
|
| 97 |
+
"""
|
| 98 |
+
|
| 99 |
+
t = (1 << bits) - 1
|
| 100 |
+
packed = torch.clamp((value * t + 0.5).floor(), min=0, max=t)
|
| 101 |
+
# Convert to integer type
|
| 102 |
+
return packed.to(torch.int64)
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def pack_111011(x: torch.Tensor, y: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
|
| 106 |
+
"""Pack three floating point values into a 32-bit integer with 11, 10, and 11 bits.
|
| 107 |
+
|
| 108 |
+
Args:
|
| 109 |
+
x (torch.Tensor): X component. Shape (N,)
|
| 110 |
+
y (torch.Tensor): Y component. Shape (N,)
|
| 111 |
+
z (torch.Tensor): Z component. Shape (N,)
|
| 112 |
+
Returns:
|
| 113 |
+
torch.Tensor: Packed values. Shape (N,)
|
| 114 |
+
"""
|
| 115 |
+
# Pack each component using pack_unorm
|
| 116 |
+
packed_x = pack_unorm(x, 11) << 21
|
| 117 |
+
packed_y = pack_unorm(y, 10) << 11
|
| 118 |
+
packed_z = pack_unorm(z, 11)
|
| 119 |
+
|
| 120 |
+
# Combine the packed values using bitwise OR
|
| 121 |
+
return packed_x | packed_y | packed_z
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def pack_8888(
|
| 125 |
+
x: torch.Tensor, y: torch.Tensor, z: torch.Tensor, w: torch.Tensor
|
| 126 |
+
) -> torch.Tensor:
|
| 127 |
+
"""Pack four floating point values into a 32-bit integer with 8 bits each.
|
| 128 |
+
|
| 129 |
+
Args:
|
| 130 |
+
x (torch.Tensor): X component. Shape (N,)
|
| 131 |
+
y (torch.Tensor): Y component. Shape (N,)
|
| 132 |
+
z (torch.Tensor): Z component. Shape (N,)
|
| 133 |
+
w (torch.Tensor): W component. Shape (N,)
|
| 134 |
+
Returns:
|
| 135 |
+
torch.Tensor: Packed values. Shape (N,)
|
| 136 |
+
"""
|
| 137 |
+
# Pack each component using pack_unorm
|
| 138 |
+
packed_x = pack_unorm(x, 8) << 24
|
| 139 |
+
packed_y = pack_unorm(y, 8) << 16
|
| 140 |
+
packed_z = pack_unorm(z, 8) << 8
|
| 141 |
+
packed_w = pack_unorm(w, 8)
|
| 142 |
+
|
| 143 |
+
# Combine the packed values using bitwise OR
|
| 144 |
+
return packed_x | packed_y | packed_z | packed_w
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def pack_rotation(q: torch.Tensor) -> torch.Tensor:
|
| 148 |
+
"""Pack a quaternion into a 32-bit integer.
|
| 149 |
+
|
| 150 |
+
Args:
|
| 151 |
+
q (torch.Tensor): Quaternions. Shape (N, 4)
|
| 152 |
+
|
| 153 |
+
Returns:
|
| 154 |
+
torch.Tensor: Packed values. Shape (N,)
|
| 155 |
+
"""
|
| 156 |
+
|
| 157 |
+
# Normalize each quaternion
|
| 158 |
+
norms = torch.linalg.norm(q, dim=-1, keepdim=True)
|
| 159 |
+
q = q / norms
|
| 160 |
+
|
| 161 |
+
# Find the largest component index for each quaternion
|
| 162 |
+
largest_components = torch.argmax(torch.abs(q), dim=-1)
|
| 163 |
+
|
| 164 |
+
# Flip quaternions where the largest component is negative
|
| 165 |
+
batch_indices = torch.arange(q.size(0), device=q.device)
|
| 166 |
+
largest_values = q[batch_indices, largest_components]
|
| 167 |
+
flip_mask = largest_values < 0
|
| 168 |
+
q[flip_mask] *= -1
|
| 169 |
+
|
| 170 |
+
# Precomputed indices for the components to pack (excluding largest)
|
| 171 |
+
precomputed_indices = torch.tensor(
|
| 172 |
+
[[1, 2, 3], [0, 2, 3], [0, 1, 3], [0, 1, 2]], dtype=torch.long, device=q.device
|
| 173 |
+
)
|
| 174 |
+
|
| 175 |
+
# Gather components to pack for each quaternion
|
| 176 |
+
pack_indices = precomputed_indices[largest_components]
|
| 177 |
+
components_to_pack = q[batch_indices[:, None], pack_indices]
|
| 178 |
+
|
| 179 |
+
# Scale and pack each component into 10-bit integers
|
| 180 |
+
norm = math.sqrt(2) * 0.5
|
| 181 |
+
scaled = components_to_pack * norm + 0.5
|
| 182 |
+
packed = pack_unorm(scaled, 10) # Assuming pack_unorm is vectorized
|
| 183 |
+
|
| 184 |
+
# Combine into the final 32-bit integer
|
| 185 |
+
largest_packed = largest_components.to(torch.int64) << 30
|
| 186 |
+
c0_packed = packed[:, 0] << 20
|
| 187 |
+
c1_packed = packed[:, 1] << 10
|
| 188 |
+
c2_packed = packed[:, 2]
|
| 189 |
+
|
| 190 |
+
result = largest_packed | c0_packed | c1_packed | c2_packed
|
| 191 |
+
return result
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def splat2ply_bytes_compressed(
|
| 195 |
+
means: torch.Tensor,
|
| 196 |
+
scales: torch.Tensor,
|
| 197 |
+
quats: torch.Tensor,
|
| 198 |
+
opacities: torch.Tensor,
|
| 199 |
+
sh0: torch.Tensor,
|
| 200 |
+
shN: torch.Tensor,
|
| 201 |
+
chunk_max_size: int = 256,
|
| 202 |
+
opacity_threshold: float = 1 / 255,
|
| 203 |
+
) -> bytes:
|
| 204 |
+
"""Return the binary compressed Ply file. Used by Supersplat viewer.
|
| 205 |
+
|
| 206 |
+
Args:
|
| 207 |
+
means (torch.Tensor): Splat means. Shape (N, 3)
|
| 208 |
+
scales (torch.Tensor): Splat scales. Shape (N, 3)
|
| 209 |
+
quats (torch.Tensor): Splat quaternions. Shape (N, 4)
|
| 210 |
+
opacities (torch.Tensor): Splat opacities. Shape (N,)
|
| 211 |
+
sh0 (torch.Tensor): Spherical harmonics. Shape (N, 3)
|
| 212 |
+
shN (torch.Tensor): Spherical harmonics. Shape (N, K*3)
|
| 213 |
+
chunk_max_size (int): Maximum number of splats per chunk. Default: 256
|
| 214 |
+
opacity_threshold (float): Opacity threshold. Default: 1 / 255
|
| 215 |
+
|
| 216 |
+
Returns:
|
| 217 |
+
bytes: Binary compressed Ply file representing the model.
|
| 218 |
+
"""
|
| 219 |
+
|
| 220 |
+
# Filter the splats with too low opacity
|
| 221 |
+
mask = torch.sigmoid(opacities) > opacity_threshold
|
| 222 |
+
means = means[mask]
|
| 223 |
+
scales = scales[mask]
|
| 224 |
+
sh0_colors = sh2rgb(sh0)
|
| 225 |
+
sh0_colors = sh0_colors[mask]
|
| 226 |
+
shN = shN[mask]
|
| 227 |
+
quats = quats[mask]
|
| 228 |
+
opacities = opacities[mask]
|
| 229 |
+
|
| 230 |
+
num_splats = means.shape[0]
|
| 231 |
+
n_chunks = num_splats // chunk_max_size + (num_splats % chunk_max_size != 0)
|
| 232 |
+
indices = torch.arange(num_splats)
|
| 233 |
+
indices = sort_centers(means, indices)
|
| 234 |
+
|
| 235 |
+
float_properties = [
|
| 236 |
+
"min_x",
|
| 237 |
+
"min_y",
|
| 238 |
+
"min_z",
|
| 239 |
+
"max_x",
|
| 240 |
+
"max_y",
|
| 241 |
+
"max_z",
|
| 242 |
+
"min_scale_x",
|
| 243 |
+
"min_scale_y",
|
| 244 |
+
"min_scale_z",
|
| 245 |
+
"max_scale_x",
|
| 246 |
+
"max_scale_y",
|
| 247 |
+
"max_scale_z",
|
| 248 |
+
"min_r",
|
| 249 |
+
"min_g",
|
| 250 |
+
"min_b",
|
| 251 |
+
"max_r",
|
| 252 |
+
"max_g",
|
| 253 |
+
"max_b",
|
| 254 |
+
]
|
| 255 |
+
uint_properties = [
|
| 256 |
+
"packed_position",
|
| 257 |
+
"packed_rotation",
|
| 258 |
+
"packed_scale",
|
| 259 |
+
"packed_color",
|
| 260 |
+
]
|
| 261 |
+
buffer = BytesIO()
|
| 262 |
+
|
| 263 |
+
# Write PLY header
|
| 264 |
+
buffer.write(b"ply\n")
|
| 265 |
+
buffer.write(b"format binary_little_endian 1.0\n")
|
| 266 |
+
buffer.write(f"element chunk {n_chunks}\n".encode())
|
| 267 |
+
for prop in float_properties:
|
| 268 |
+
buffer.write(f"property float {prop}\n".encode())
|
| 269 |
+
buffer.write(f"element vertex {num_splats}\n".encode())
|
| 270 |
+
for prop in uint_properties:
|
| 271 |
+
buffer.write(f"property uint {prop}\n".encode())
|
| 272 |
+
buffer.write(f"element sh {num_splats}\n".encode())
|
| 273 |
+
for j in range(shN.shape[1]):
|
| 274 |
+
buffer.write(f"property uchar f_rest_{j}\n".encode())
|
| 275 |
+
buffer.write(b"end_header\n")
|
| 276 |
+
|
| 277 |
+
chunk_data = []
|
| 278 |
+
splat_data = []
|
| 279 |
+
sh_data = []
|
| 280 |
+
for chunk_idx in range(n_chunks):
|
| 281 |
+
chunk_end_idx = min((chunk_idx + 1) * chunk_max_size, num_splats)
|
| 282 |
+
chunk_start_idx = chunk_idx * chunk_max_size
|
| 283 |
+
splat_idxs = indices[chunk_start_idx:chunk_end_idx]
|
| 284 |
+
|
| 285 |
+
# Bounds
|
| 286 |
+
# Means
|
| 287 |
+
chunk_means = means[splat_idxs]
|
| 288 |
+
min_means = torch.min(chunk_means, dim=0).values
|
| 289 |
+
max_means = torch.max(chunk_means, dim=0).values
|
| 290 |
+
mean_bounds = torch.cat([min_means, max_means])
|
| 291 |
+
# Scales
|
| 292 |
+
chunk_scales = scales[splat_idxs]
|
| 293 |
+
min_scales = torch.min(chunk_scales, dim=0).values
|
| 294 |
+
max_scales = torch.max(chunk_scales, dim=0).values
|
| 295 |
+
min_scales = torch.clamp(min_scales, -20, 20)
|
| 296 |
+
max_scales = torch.clamp(max_scales, -20, 20)
|
| 297 |
+
scale_bounds = torch.cat([min_scales, max_scales])
|
| 298 |
+
# Colors
|
| 299 |
+
chunk_colors = sh0_colors[splat_idxs]
|
| 300 |
+
min_colors = torch.min(chunk_colors, dim=0).values
|
| 301 |
+
max_colors = torch.max(chunk_colors, dim=0).values
|
| 302 |
+
color_bounds = torch.cat([min_colors, max_colors])
|
| 303 |
+
chunk_data.extend([mean_bounds, scale_bounds, color_bounds])
|
| 304 |
+
|
| 305 |
+
# Quantized properties:
|
| 306 |
+
# Means
|
| 307 |
+
normalized_means = (chunk_means - min_means) / (max_means - min_means)
|
| 308 |
+
means_i = pack_111011(
|
| 309 |
+
normalized_means[:, 0],
|
| 310 |
+
normalized_means[:, 1],
|
| 311 |
+
normalized_means[:, 2],
|
| 312 |
+
)
|
| 313 |
+
# Quaternions
|
| 314 |
+
chunk_quats = quats[splat_idxs]
|
| 315 |
+
quat_i = pack_rotation(chunk_quats)
|
| 316 |
+
# Scales
|
| 317 |
+
normalized_scales = (chunk_scales - min_scales) / (max_scales - min_scales)
|
| 318 |
+
scales_i = pack_111011(
|
| 319 |
+
normalized_scales[:, 0],
|
| 320 |
+
normalized_scales[:, 1],
|
| 321 |
+
normalized_scales[:, 2],
|
| 322 |
+
)
|
| 323 |
+
# Colors
|
| 324 |
+
normalized_colors = (chunk_colors - min_colors) / (max_colors - min_colors)
|
| 325 |
+
chunk_opacities = opacities[splat_idxs]
|
| 326 |
+
chunk_opacities = 1 / (1 + torch.exp(-chunk_opacities))
|
| 327 |
+
chunk_opacities = chunk_opacities.unsqueeze(-1)
|
| 328 |
+
normalized_colors_i = torch.cat([normalized_colors, chunk_opacities], dim=-1)
|
| 329 |
+
color_i = pack_8888(
|
| 330 |
+
normalized_colors_i[:, 0],
|
| 331 |
+
normalized_colors_i[:, 1],
|
| 332 |
+
normalized_colors_i[:, 2],
|
| 333 |
+
normalized_colors_i[:, 3],
|
| 334 |
+
)
|
| 335 |
+
splat_data_chunk = torch.stack([means_i, quat_i, scales_i, color_i], dim=1)
|
| 336 |
+
splat_data_chunk = splat_data_chunk.ravel().to(torch.int64)
|
| 337 |
+
splat_data.extend([splat_data_chunk])
|
| 338 |
+
|
| 339 |
+
# Quantized spherical harmonics
|
| 340 |
+
shN_chunk = shN[splat_idxs]
|
| 341 |
+
shN_chunk_quantized = (shN_chunk / 8 + 0.5) * 256
|
| 342 |
+
shN_chunk_quantized = torch.clamp(torch.trunc(shN_chunk_quantized), 0, 255)
|
| 343 |
+
shN_chunk_quantized = shN_chunk_quantized.to(torch.uint8)
|
| 344 |
+
sh_data.extend([shN_chunk_quantized.ravel()])
|
| 345 |
+
|
| 346 |
+
float_dtype = np.dtype(np.float32).newbyteorder("<")
|
| 347 |
+
uint32_dtype = np.dtype(np.uint32).newbyteorder("<")
|
| 348 |
+
uint8_dtype = np.dtype(np.uint8)
|
| 349 |
+
|
| 350 |
+
buffer.write(
|
| 351 |
+
torch.cat(chunk_data).detach().cpu().numpy().astype(float_dtype).tobytes()
|
| 352 |
+
)
|
| 353 |
+
buffer.write(
|
| 354 |
+
torch.cat(splat_data).detach().cpu().numpy().astype(uint32_dtype).tobytes()
|
| 355 |
+
)
|
| 356 |
+
buffer.write(
|
| 357 |
+
torch.cat(sh_data).detach().cpu().numpy().astype(uint8_dtype).tobytes()
|
| 358 |
+
)
|
| 359 |
+
|
| 360 |
+
return buffer.getvalue()
|
| 361 |
+
|
| 362 |
+
|
| 363 |
+
def splat2ply_bytes(
|
| 364 |
+
means: torch.Tensor,
|
| 365 |
+
scales: torch.Tensor,
|
| 366 |
+
quats: torch.Tensor,
|
| 367 |
+
opacities: torch.Tensor,
|
| 368 |
+
sh0: torch.Tensor,
|
| 369 |
+
shN: torch.Tensor,
|
| 370 |
+
) -> bytes:
|
| 371 |
+
"""Return the binary Ply file. Supported by almost all viewers.
|
| 372 |
+
|
| 373 |
+
Args:
|
| 374 |
+
means (torch.Tensor): Splat means. Shape (N, 3)
|
| 375 |
+
scales (torch.Tensor): Splat scales. Shape (N, 3)
|
| 376 |
+
quats (torch.Tensor): Splat quaternions. Shape (N, 4)
|
| 377 |
+
opacities (torch.Tensor): Splat opacities. Shape (N,)
|
| 378 |
+
sh0 (torch.Tensor): Spherical harmonics. Shape (N, 3)
|
| 379 |
+
shN (torch.Tensor): Spherical harmonics. Shape (N, K*3)
|
| 380 |
+
|
| 381 |
+
Returns:
|
| 382 |
+
bytes: Binary Ply file representing the model.
|
| 383 |
+
"""
|
| 384 |
+
|
| 385 |
+
num_splats = means.shape[0]
|
| 386 |
+
buffer = BytesIO()
|
| 387 |
+
|
| 388 |
+
# Write PLY header
|
| 389 |
+
buffer.write(b"ply\n")
|
| 390 |
+
buffer.write(b"format binary_little_endian 1.0\n")
|
| 391 |
+
buffer.write(f"element vertex {num_splats}\n".encode())
|
| 392 |
+
buffer.write(b"property float x\n")
|
| 393 |
+
buffer.write(b"property float y\n")
|
| 394 |
+
buffer.write(b"property float z\n")
|
| 395 |
+
for i, data in enumerate([sh0, shN]):
|
| 396 |
+
prefix = "f_dc" if i == 0 else "f_rest"
|
| 397 |
+
for j in range(data.shape[1]):
|
| 398 |
+
buffer.write(f"property float {prefix}_{j}\n".encode())
|
| 399 |
+
buffer.write(b"property float opacity\n")
|
| 400 |
+
for i in range(scales.shape[1]):
|
| 401 |
+
buffer.write(f"property float scale_{i}\n".encode())
|
| 402 |
+
for i in range(quats.shape[1]):
|
| 403 |
+
buffer.write(f"property float rot_{i}\n".encode())
|
| 404 |
+
buffer.write(b"end_header\n")
|
| 405 |
+
|
| 406 |
+
# Concatenate all tensors in the correct order
|
| 407 |
+
splat_data = torch.cat(
|
| 408 |
+
[means, sh0, shN, opacities.unsqueeze(1), scales, quats], dim=1
|
| 409 |
+
)
|
| 410 |
+
# Ensure correct dtype
|
| 411 |
+
splat_data = splat_data.to(torch.float32)
|
| 412 |
+
|
| 413 |
+
# Write binary data
|
| 414 |
+
float_dtype = np.dtype(np.float32).newbyteorder("<")
|
| 415 |
+
buffer.write(splat_data.detach().cpu().numpy().astype(float_dtype).tobytes())
|
| 416 |
+
|
| 417 |
+
return buffer.getvalue()
|
| 418 |
+
|
| 419 |
+
|
| 420 |
+
def splat2splat_bytes(
|
| 421 |
+
means: torch.Tensor,
|
| 422 |
+
scales: torch.Tensor,
|
| 423 |
+
quats: torch.Tensor,
|
| 424 |
+
opacities: torch.Tensor,
|
| 425 |
+
sh0: torch.Tensor,
|
| 426 |
+
) -> bytes:
|
| 427 |
+
"""Return the binary Splat file. Supported by antimatter15 viewer.
|
| 428 |
+
|
| 429 |
+
Args:
|
| 430 |
+
means (torch.Tensor): Splat means. Shape (N, 3)
|
| 431 |
+
scales (torch.Tensor): Splat scales. Shape (N, 3)
|
| 432 |
+
quats (torch.Tensor): Splat quaternions. Shape (N, 4)
|
| 433 |
+
opacities (torch.Tensor): Splat opacities. Shape (N,)
|
| 434 |
+
sh0 (torch.Tensor): Spherical harmonics. Shape (N, 3)
|
| 435 |
+
|
| 436 |
+
Returns:
|
| 437 |
+
bytes: Binary Splat file representing the model.
|
| 438 |
+
"""
|
| 439 |
+
|
| 440 |
+
# Preprocess
|
| 441 |
+
scales = torch.exp(scales)
|
| 442 |
+
sh0_color = sh2rgb(sh0)
|
| 443 |
+
colors = torch.cat([sh0_color, torch.sigmoid(opacities).unsqueeze(-1)], dim=1)
|
| 444 |
+
colors = (colors * 255).clamp(0, 255).to(torch.uint8)
|
| 445 |
+
|
| 446 |
+
rots = (quats / torch.linalg.norm(quats, dim=1, keepdim=True)) * 128 + 128
|
| 447 |
+
rots = rots.clamp(0, 255).to(torch.uint8)
|
| 448 |
+
|
| 449 |
+
# Sort splats
|
| 450 |
+
num_splats = means.shape[0]
|
| 451 |
+
indices = sort_centers(means, torch.arange(num_splats))
|
| 452 |
+
|
| 453 |
+
# Reorder everything
|
| 454 |
+
means = means[indices]
|
| 455 |
+
scales = scales[indices]
|
| 456 |
+
colors = colors[indices]
|
| 457 |
+
rots = rots[indices]
|
| 458 |
+
|
| 459 |
+
float_dtype = np.dtype(np.float32).newbyteorder("<")
|
| 460 |
+
means_np = means.detach().cpu().numpy().astype(float_dtype)
|
| 461 |
+
scales_np = scales.detach().cpu().numpy().astype(float_dtype)
|
| 462 |
+
colors_np = colors.detach().cpu().numpy().astype(np.uint8)
|
| 463 |
+
rots_np = rots.detach().cpu().numpy().astype(np.uint8)
|
| 464 |
+
|
| 465 |
+
buffer = BytesIO()
|
| 466 |
+
for i in range(num_splats):
|
| 467 |
+
buffer.write(means_np[i].tobytes())
|
| 468 |
+
buffer.write(scales_np[i].tobytes())
|
| 469 |
+
buffer.write(colors_np[i].tobytes())
|
| 470 |
+
buffer.write(rots_np[i].tobytes())
|
| 471 |
+
|
| 472 |
+
return buffer.getvalue()
|
| 473 |
+
|
| 474 |
+
|
| 475 |
+
def export_splats(
|
| 476 |
+
means: torch.Tensor,
|
| 477 |
+
scales: torch.Tensor,
|
| 478 |
+
quats: torch.Tensor,
|
| 479 |
+
opacities: torch.Tensor,
|
| 480 |
+
sh0: torch.Tensor,
|
| 481 |
+
shN: torch.Tensor,
|
| 482 |
+
format: Literal["ply", "splat", "ply_compressed"] = "ply",
|
| 483 |
+
save_to: Optional[str] = None,
|
| 484 |
+
) -> bytes:
|
| 485 |
+
"""Export a Gaussian Splats model to bytes.
|
| 486 |
+
The three supported formats are:
|
| 487 |
+
- ply: A standard PLY file format. Supported by most viewers.
|
| 488 |
+
- splat: A custom Splat file format. Supported by antimatter15 viewer.
|
| 489 |
+
- ply_compressed: A compressed PLY file format. Used by Supersplat viewer.
|
| 490 |
+
|
| 491 |
+
Args:
|
| 492 |
+
means (torch.Tensor): Splat means. Shape (N, 3)
|
| 493 |
+
scales (torch.Tensor): Splat scales. Shape (N, 3)
|
| 494 |
+
quats (torch.Tensor): Splat quaternions. Shape (N, 4)
|
| 495 |
+
opacities (torch.Tensor): Splat opacities. Shape (N,)
|
| 496 |
+
sh0 (torch.Tensor): Spherical harmonics. Shape (N, 1, 3)
|
| 497 |
+
shN (torch.Tensor): Spherical harmonics. Shape (N, K, 3)
|
| 498 |
+
format (str): Export format. Options: "ply", "splat", "ply_compressed". Default: "ply"
|
| 499 |
+
save_to (str): Output file path. If provided, the bytes will be written to file.
|
| 500 |
+
"""
|
| 501 |
+
total_splats = means.shape[0]
|
| 502 |
+
assert means.shape == (total_splats, 3), "Means must be of shape (N, 3)"
|
| 503 |
+
assert scales.shape == (total_splats, 3), "Scales must be of shape (N, 3)"
|
| 504 |
+
assert quats.shape == (total_splats, 4), "Quaternions must be of shape (N, 4)"
|
| 505 |
+
assert opacities.shape == (total_splats,), "Opacities must be of shape (N,)"
|
| 506 |
+
assert sh0.shape == (total_splats, 1, 3), "sh0 must be of shape (N, 1, 3)"
|
| 507 |
+
assert (
|
| 508 |
+
shN.ndim == 3 and shN.shape[0] == total_splats and shN.shape[2] == 3
|
| 509 |
+
), f"shN must be of shape (N, K, 3), got {shN.shape}"
|
| 510 |
+
|
| 511 |
+
# Reshape spherical harmonics
|
| 512 |
+
sh0 = sh0.squeeze(1) # Shape (N, 3)
|
| 513 |
+
shN = shN.permute(0, 2, 1).reshape(means.shape[0], -1) # Shape (N, K * 3)
|
| 514 |
+
|
| 515 |
+
# Check for NaN or Inf values
|
| 516 |
+
invalid_mask = (
|
| 517 |
+
torch.isnan(means).any(dim=1)
|
| 518 |
+
| torch.isinf(means).any(dim=1)
|
| 519 |
+
| torch.isnan(scales).any(dim=1)
|
| 520 |
+
| torch.isinf(scales).any(dim=1)
|
| 521 |
+
| torch.isnan(quats).any(dim=1)
|
| 522 |
+
| torch.isinf(quats).any(dim=1)
|
| 523 |
+
| torch.isnan(opacities).any(dim=0)
|
| 524 |
+
| torch.isinf(opacities).any(dim=0)
|
| 525 |
+
| torch.isnan(sh0).any(dim=1)
|
| 526 |
+
| torch.isinf(sh0).any(dim=1)
|
| 527 |
+
| torch.isnan(shN).any(dim=1)
|
| 528 |
+
| torch.isinf(shN).any(dim=1)
|
| 529 |
+
)
|
| 530 |
+
|
| 531 |
+
# Filter out invalid entries
|
| 532 |
+
valid_mask = ~invalid_mask
|
| 533 |
+
means = means[valid_mask]
|
| 534 |
+
scales = scales[valid_mask]
|
| 535 |
+
quats = quats[valid_mask]
|
| 536 |
+
opacities = opacities[valid_mask]
|
| 537 |
+
sh0 = sh0[valid_mask]
|
| 538 |
+
shN = shN[valid_mask]
|
| 539 |
+
|
| 540 |
+
if format == "ply":
|
| 541 |
+
data = splat2ply_bytes(means, scales, quats, opacities, sh0, shN)
|
| 542 |
+
elif format == "splat":
|
| 543 |
+
data = splat2splat_bytes(means, scales, quats, opacities, sh0)
|
| 544 |
+
elif format == "ply_compressed":
|
| 545 |
+
data = splat2ply_bytes_compressed(means, scales, quats, opacities, sh0, shN)
|
| 546 |
+
else:
|
| 547 |
+
raise ValueError(f"Unsupported format: {format}")
|
| 548 |
+
|
| 549 |
+
if save_to:
|
| 550 |
+
with open(save_to, "wb") as binary_file:
|
| 551 |
+
binary_file.write(data)
|
| 552 |
+
|
| 553 |
+
return data
|
src/post_opt/gsplat_viewer.py
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import viser
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
from typing import Literal
|
| 4 |
+
from typing import Tuple, Callable
|
| 5 |
+
from nerfview import Viewer, RenderTabState
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class GsplatRenderTabState(RenderTabState):
|
| 9 |
+
# non-controlable parameters
|
| 10 |
+
total_gs_count: int = 0
|
| 11 |
+
rendered_gs_count: int = 0
|
| 12 |
+
|
| 13 |
+
# controlable parameters
|
| 14 |
+
max_sh_degree: int = 5
|
| 15 |
+
near_plane: float = 1e-2
|
| 16 |
+
far_plane: float = 1e2
|
| 17 |
+
radius_clip: float = 0.0
|
| 18 |
+
eps2d: float = 0.3
|
| 19 |
+
backgrounds: Tuple[float, float, float] = (0.0, 0.0, 0.0)
|
| 20 |
+
render_mode: Literal[
|
| 21 |
+
"rgb", "depth(accumulated)", "depth(expected)", "alpha"
|
| 22 |
+
] = "rgb"
|
| 23 |
+
normalize_nearfar: bool = False
|
| 24 |
+
inverse: bool = True
|
| 25 |
+
colormap: Literal[
|
| 26 |
+
"turbo", "viridis", "magma", "inferno", "cividis", "gray"
|
| 27 |
+
] = "turbo"
|
| 28 |
+
rasterize_mode: Literal["classic", "antialiased"] = "classic"
|
| 29 |
+
camera_model: Literal["pinhole", "ortho", "fisheye"] = "pinhole"
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class GsplatViewer(Viewer):
|
| 33 |
+
"""
|
| 34 |
+
Viewer for gsplat.
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
def __init__(
|
| 38 |
+
self,
|
| 39 |
+
server: viser.ViserServer,
|
| 40 |
+
render_fn: Callable,
|
| 41 |
+
output_dir: Path,
|
| 42 |
+
mode: Literal["rendering", "training"] = "rendering",
|
| 43 |
+
):
|
| 44 |
+
super().__init__(server, render_fn, output_dir, mode)
|
| 45 |
+
server.gui.set_panel_label("gsplat viewer")
|
| 46 |
+
|
| 47 |
+
def _init_rendering_tab(self):
|
| 48 |
+
self.render_tab_state = GsplatRenderTabState()
|
| 49 |
+
self._rendering_tab_handles = {}
|
| 50 |
+
self._rendering_folder = self.server.gui.add_folder("Rendering")
|
| 51 |
+
|
| 52 |
+
def _populate_rendering_tab(self):
|
| 53 |
+
server = self.server
|
| 54 |
+
with self._rendering_folder:
|
| 55 |
+
with server.gui.add_folder("Gsplat"):
|
| 56 |
+
total_gs_count_number = server.gui.add_number(
|
| 57 |
+
"Total",
|
| 58 |
+
initial_value=self.render_tab_state.total_gs_count,
|
| 59 |
+
disabled=True,
|
| 60 |
+
hint="Total number of splats in the scene.",
|
| 61 |
+
)
|
| 62 |
+
rendered_gs_count_number = server.gui.add_number(
|
| 63 |
+
"Rendered",
|
| 64 |
+
initial_value=self.render_tab_state.rendered_gs_count,
|
| 65 |
+
disabled=True,
|
| 66 |
+
hint="Number of splats rendered.",
|
| 67 |
+
)
|
| 68 |
+
|
| 69 |
+
max_sh_degree_number = server.gui.add_number(
|
| 70 |
+
"Max SH",
|
| 71 |
+
initial_value=self.render_tab_state.max_sh_degree,
|
| 72 |
+
min=0,
|
| 73 |
+
max=5,
|
| 74 |
+
step=1,
|
| 75 |
+
hint="Maximum SH degree used",
|
| 76 |
+
)
|
| 77 |
+
|
| 78 |
+
@max_sh_degree_number.on_update
|
| 79 |
+
def _(_) -> None:
|
| 80 |
+
self.render_tab_state.max_sh_degree = int(
|
| 81 |
+
max_sh_degree_number.value
|
| 82 |
+
)
|
| 83 |
+
self.rerender(_)
|
| 84 |
+
|
| 85 |
+
near_far_plane_vec2 = server.gui.add_vector2(
|
| 86 |
+
"Near/Far",
|
| 87 |
+
initial_value=(
|
| 88 |
+
self.render_tab_state.near_plane,
|
| 89 |
+
self.render_tab_state.far_plane,
|
| 90 |
+
),
|
| 91 |
+
min=(1e-3, 1e1),
|
| 92 |
+
max=(1e1, 1e2),
|
| 93 |
+
step=1e-3,
|
| 94 |
+
hint="Near and far plane for rendering.",
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
@near_far_plane_vec2.on_update
|
| 98 |
+
def _(_) -> None:
|
| 99 |
+
self.render_tab_state.near_plane = near_far_plane_vec2.value[0]
|
| 100 |
+
self.render_tab_state.far_plane = near_far_plane_vec2.value[1]
|
| 101 |
+
self.rerender(_)
|
| 102 |
+
|
| 103 |
+
radius_clip_slider = server.gui.add_number(
|
| 104 |
+
"Radius Clip",
|
| 105 |
+
initial_value=self.render_tab_state.radius_clip,
|
| 106 |
+
min=0.0,
|
| 107 |
+
max=100.0,
|
| 108 |
+
step=1.0,
|
| 109 |
+
hint="2D radius clip for rendering.",
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
@radius_clip_slider.on_update
|
| 113 |
+
def _(_) -> None:
|
| 114 |
+
self.render_tab_state.radius_clip = radius_clip_slider.value
|
| 115 |
+
self.rerender(_)
|
| 116 |
+
|
| 117 |
+
eps2d_slider = server.gui.add_number(
|
| 118 |
+
"2D Epsilon",
|
| 119 |
+
initial_value=self.render_tab_state.eps2d,
|
| 120 |
+
min=0.0,
|
| 121 |
+
max=1.0,
|
| 122 |
+
step=0.01,
|
| 123 |
+
hint="Epsilon added to the egienvalues of projected 2D covariance matrices.",
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
@eps2d_slider.on_update
|
| 127 |
+
def _(_) -> None:
|
| 128 |
+
self.render_tab_state.eps2d = eps2d_slider.value
|
| 129 |
+
self.rerender(_)
|
| 130 |
+
|
| 131 |
+
backgrounds_slider = server.gui.add_rgb(
|
| 132 |
+
"Background",
|
| 133 |
+
initial_value=self.render_tab_state.backgrounds,
|
| 134 |
+
hint="Background color for rendering.",
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
@backgrounds_slider.on_update
|
| 138 |
+
def _(_) -> None:
|
| 139 |
+
self.render_tab_state.backgrounds = backgrounds_slider.value
|
| 140 |
+
self.rerender(_)
|
| 141 |
+
|
| 142 |
+
render_mode_dropdown = server.gui.add_dropdown(
|
| 143 |
+
"Render Mode",
|
| 144 |
+
("rgb", "depth(accumulated)", "depth(expected)", "alpha"),
|
| 145 |
+
initial_value=self.render_tab_state.render_mode,
|
| 146 |
+
hint="Render mode to use.",
|
| 147 |
+
)
|
| 148 |
+
|
| 149 |
+
@render_mode_dropdown.on_update
|
| 150 |
+
def _(_) -> None:
|
| 151 |
+
if "depth" in render_mode_dropdown.value:
|
| 152 |
+
normalize_nearfar_checkbox.disabled = False
|
| 153 |
+
else:
|
| 154 |
+
normalize_nearfar_checkbox.disabled = True
|
| 155 |
+
if render_mode_dropdown.value == "rgb":
|
| 156 |
+
inverse_checkbox.disabled = True
|
| 157 |
+
else:
|
| 158 |
+
inverse_checkbox.disabled = False
|
| 159 |
+
self.render_tab_state.render_mode = render_mode_dropdown.value
|
| 160 |
+
self.rerender(_)
|
| 161 |
+
|
| 162 |
+
normalize_nearfar_checkbox = server.gui.add_checkbox(
|
| 163 |
+
"Normalize Near/Far",
|
| 164 |
+
initial_value=self.render_tab_state.normalize_nearfar,
|
| 165 |
+
disabled=True,
|
| 166 |
+
hint="Normalize depth with near/far plane.",
|
| 167 |
+
)
|
| 168 |
+
|
| 169 |
+
@normalize_nearfar_checkbox.on_update
|
| 170 |
+
def _(_) -> None:
|
| 171 |
+
self.render_tab_state.normalize_nearfar = (
|
| 172 |
+
normalize_nearfar_checkbox.value
|
| 173 |
+
)
|
| 174 |
+
self.rerender(_)
|
| 175 |
+
|
| 176 |
+
inverse_checkbox = server.gui.add_checkbox(
|
| 177 |
+
"Inverse",
|
| 178 |
+
initial_value=self.render_tab_state.inverse,
|
| 179 |
+
disabled=True,
|
| 180 |
+
hint="Inverse the depth.",
|
| 181 |
+
)
|
| 182 |
+
|
| 183 |
+
@inverse_checkbox.on_update
|
| 184 |
+
def _(_) -> None:
|
| 185 |
+
self.render_tab_state.inverse = inverse_checkbox.value
|
| 186 |
+
self.rerender(_)
|
| 187 |
+
|
| 188 |
+
colormap_dropdown = server.gui.add_dropdown(
|
| 189 |
+
"Colormap",
|
| 190 |
+
("turbo", "viridis", "magma", "inferno", "cividis", "gray"),
|
| 191 |
+
initial_value=self.render_tab_state.colormap,
|
| 192 |
+
hint="Colormap used for rendering depth/alpha.",
|
| 193 |
+
)
|
| 194 |
+
|
| 195 |
+
@colormap_dropdown.on_update
|
| 196 |
+
def _(_) -> None:
|
| 197 |
+
self.render_tab_state.colormap = colormap_dropdown.value
|
| 198 |
+
self.rerender(_)
|
| 199 |
+
|
| 200 |
+
rasterize_mode_dropdown = server.gui.add_dropdown(
|
| 201 |
+
"Anti-Aliasing",
|
| 202 |
+
("classic", "antialiased"),
|
| 203 |
+
initial_value=self.render_tab_state.rasterize_mode,
|
| 204 |
+
hint="Whether to use classic or antialiased rasterization.",
|
| 205 |
+
)
|
| 206 |
+
|
| 207 |
+
@rasterize_mode_dropdown.on_update
|
| 208 |
+
def _(_) -> None:
|
| 209 |
+
self.render_tab_state.rasterize_mode = rasterize_mode_dropdown.value
|
| 210 |
+
self.rerender(_)
|
| 211 |
+
|
| 212 |
+
camera_model_dropdown = server.gui.add_dropdown(
|
| 213 |
+
"Camera",
|
| 214 |
+
("pinhole", "ortho", "fisheye"),
|
| 215 |
+
initial_value=self.render_tab_state.camera_model,
|
| 216 |
+
hint="Camera model used for rendering.",
|
| 217 |
+
)
|
| 218 |
+
|
| 219 |
+
@camera_model_dropdown.on_update
|
| 220 |
+
def _(_) -> None:
|
| 221 |
+
self.render_tab_state.camera_model = camera_model_dropdown.value
|
| 222 |
+
self.rerender(_)
|
| 223 |
+
|
| 224 |
+
self._rendering_tab_handles.update(
|
| 225 |
+
{
|
| 226 |
+
"total_gs_count_number": total_gs_count_number,
|
| 227 |
+
"rendered_gs_count_number": rendered_gs_count_number,
|
| 228 |
+
"near_far_plane_vec2": near_far_plane_vec2,
|
| 229 |
+
"radius_clip_slider": radius_clip_slider,
|
| 230 |
+
"eps2d_slider": eps2d_slider,
|
| 231 |
+
"backgrounds_slider": backgrounds_slider,
|
| 232 |
+
"render_mode_dropdown": render_mode_dropdown,
|
| 233 |
+
"normalize_nearfar_checkbox": normalize_nearfar_checkbox,
|
| 234 |
+
"inverse_checkbox": inverse_checkbox,
|
| 235 |
+
"colormap_dropdown": colormap_dropdown,
|
| 236 |
+
"rasterize_mode_dropdown": rasterize_mode_dropdown,
|
| 237 |
+
"camera_model_dropdown": camera_model_dropdown,
|
| 238 |
+
}
|
| 239 |
+
)
|
| 240 |
+
super()._populate_rendering_tab()
|
| 241 |
+
|
| 242 |
+
def _after_render(self):
|
| 243 |
+
# Update the GUI elements with current values
|
| 244 |
+
self._rendering_tab_handles[
|
| 245 |
+
"total_gs_count_number"
|
| 246 |
+
].value = self.render_tab_state.total_gs_count
|
| 247 |
+
self._rendering_tab_handles[
|
| 248 |
+
"rendered_gs_count_number"
|
| 249 |
+
].value = self.render_tab_state.rendered_gs_count
|
src/post_opt/image_fitting.py
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
import os
|
| 3 |
+
import time
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from typing import Literal, Optional
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
import torch
|
| 9 |
+
import tyro
|
| 10 |
+
from PIL import Image
|
| 11 |
+
from torch import Tensor, optim
|
| 12 |
+
|
| 13 |
+
from gsplat import rasterization, rasterization_2dgs
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class SimpleTrainer:
|
| 17 |
+
"""Trains random gaussians to fit an image."""
|
| 18 |
+
|
| 19 |
+
def __init__(
|
| 20 |
+
self,
|
| 21 |
+
gt_image: Tensor,
|
| 22 |
+
num_points: int = 2000,
|
| 23 |
+
):
|
| 24 |
+
self.device = torch.device("cuda:0")
|
| 25 |
+
self.gt_image = gt_image.to(device=self.device)
|
| 26 |
+
self.num_points = num_points
|
| 27 |
+
|
| 28 |
+
fov_x = math.pi / 2.0
|
| 29 |
+
self.H, self.W = gt_image.shape[0], gt_image.shape[1]
|
| 30 |
+
self.focal = 0.5 * float(self.W) / math.tan(0.5 * fov_x)
|
| 31 |
+
self.img_size = torch.tensor([self.W, self.H, 1], device=self.device)
|
| 32 |
+
|
| 33 |
+
self._init_gaussians()
|
| 34 |
+
|
| 35 |
+
def _init_gaussians(self):
|
| 36 |
+
"""Random gaussians"""
|
| 37 |
+
bd = 2
|
| 38 |
+
|
| 39 |
+
self.means = bd * (torch.rand(self.num_points, 3, device=self.device) - 0.5)
|
| 40 |
+
self.scales = torch.rand(self.num_points, 3, device=self.device)
|
| 41 |
+
d = 3
|
| 42 |
+
self.rgbs = torch.rand(self.num_points, d, device=self.device)
|
| 43 |
+
|
| 44 |
+
u = torch.rand(self.num_points, 1, device=self.device)
|
| 45 |
+
v = torch.rand(self.num_points, 1, device=self.device)
|
| 46 |
+
w = torch.rand(self.num_points, 1, device=self.device)
|
| 47 |
+
|
| 48 |
+
self.quats = torch.cat(
|
| 49 |
+
[
|
| 50 |
+
torch.sqrt(1.0 - u) * torch.sin(2.0 * math.pi * v),
|
| 51 |
+
torch.sqrt(1.0 - u) * torch.cos(2.0 * math.pi * v),
|
| 52 |
+
torch.sqrt(u) * torch.sin(2.0 * math.pi * w),
|
| 53 |
+
torch.sqrt(u) * torch.cos(2.0 * math.pi * w),
|
| 54 |
+
],
|
| 55 |
+
-1,
|
| 56 |
+
)
|
| 57 |
+
self.opacities = torch.ones((self.num_points), device=self.device)
|
| 58 |
+
|
| 59 |
+
self.viewmat = torch.tensor(
|
| 60 |
+
[
|
| 61 |
+
[1.0, 0.0, 0.0, 0.0],
|
| 62 |
+
[0.0, 1.0, 0.0, 0.0],
|
| 63 |
+
[0.0, 0.0, 1.0, 8.0],
|
| 64 |
+
[0.0, 0.0, 0.0, 1.0],
|
| 65 |
+
],
|
| 66 |
+
device=self.device,
|
| 67 |
+
)
|
| 68 |
+
self.background = torch.zeros(d, device=self.device)
|
| 69 |
+
|
| 70 |
+
self.means.requires_grad = True
|
| 71 |
+
self.scales.requires_grad = True
|
| 72 |
+
self.quats.requires_grad = True
|
| 73 |
+
self.rgbs.requires_grad = True
|
| 74 |
+
self.opacities.requires_grad = True
|
| 75 |
+
self.viewmat.requires_grad = False
|
| 76 |
+
|
| 77 |
+
def train(
|
| 78 |
+
self,
|
| 79 |
+
iterations: int = 1000,
|
| 80 |
+
lr: float = 0.01,
|
| 81 |
+
save_imgs: bool = False,
|
| 82 |
+
model_type: Literal["3dgs", "2dgs"] = "3dgs",
|
| 83 |
+
):
|
| 84 |
+
optimizer = optim.Adam(
|
| 85 |
+
[self.rgbs, self.means, self.scales, self.opacities, self.quats], lr
|
| 86 |
+
)
|
| 87 |
+
mse_loss = torch.nn.MSELoss()
|
| 88 |
+
frames = []
|
| 89 |
+
times = [0] * 2 # rasterization, backward
|
| 90 |
+
K = torch.tensor(
|
| 91 |
+
[
|
| 92 |
+
[self.focal, 0, self.W / 2],
|
| 93 |
+
[0, self.focal, self.H / 2],
|
| 94 |
+
[0, 0, 1],
|
| 95 |
+
],
|
| 96 |
+
device=self.device,
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
if model_type == "3dgs":
|
| 100 |
+
rasterize_fnc = rasterization
|
| 101 |
+
elif model_type == "2dgs":
|
| 102 |
+
rasterize_fnc = rasterization_2dgs
|
| 103 |
+
|
| 104 |
+
for iter in range(iterations):
|
| 105 |
+
start = time.time()
|
| 106 |
+
|
| 107 |
+
renders = rasterize_fnc(
|
| 108 |
+
self.means,
|
| 109 |
+
self.quats / self.quats.norm(dim=-1, keepdim=True),
|
| 110 |
+
self.scales,
|
| 111 |
+
torch.sigmoid(self.opacities),
|
| 112 |
+
torch.sigmoid(self.rgbs),
|
| 113 |
+
self.viewmat[None],
|
| 114 |
+
K[None],
|
| 115 |
+
self.W,
|
| 116 |
+
self.H,
|
| 117 |
+
packed=False,
|
| 118 |
+
)[0]
|
| 119 |
+
out_img = renders[0]
|
| 120 |
+
torch.cuda.synchronize()
|
| 121 |
+
times[0] += time.time() - start
|
| 122 |
+
loss = mse_loss(out_img, self.gt_image)
|
| 123 |
+
optimizer.zero_grad()
|
| 124 |
+
start = time.time()
|
| 125 |
+
loss.backward()
|
| 126 |
+
torch.cuda.synchronize()
|
| 127 |
+
times[1] += time.time() - start
|
| 128 |
+
optimizer.step()
|
| 129 |
+
print(f"Iteration {iter + 1}/{iterations}, Loss: {loss.item()}")
|
| 130 |
+
|
| 131 |
+
if save_imgs and iter % 5 == 0:
|
| 132 |
+
frames.append((out_img.detach().cpu().numpy() * 255).astype(np.uint8))
|
| 133 |
+
if save_imgs:
|
| 134 |
+
# save them as a gif with PIL
|
| 135 |
+
frames = [Image.fromarray(frame) for frame in frames]
|
| 136 |
+
out_dir = os.path.join(os.getcwd(), "results")
|
| 137 |
+
os.makedirs(out_dir, exist_ok=True)
|
| 138 |
+
frames[0].save(
|
| 139 |
+
f"{out_dir}/training.gif",
|
| 140 |
+
save_all=True,
|
| 141 |
+
append_images=frames[1:],
|
| 142 |
+
optimize=False,
|
| 143 |
+
duration=5,
|
| 144 |
+
loop=0,
|
| 145 |
+
)
|
| 146 |
+
print(f"Total(s):\nRasterization: {times[0]:.3f}, Backward: {times[1]:.3f}")
|
| 147 |
+
print(
|
| 148 |
+
f"Per step(s):\nRasterization: {times[0]/iterations:.5f}, Backward: {times[1]/iterations:.5f}"
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def image_path_to_tensor(image_path: Path):
|
| 153 |
+
import torchvision.transforms as transforms
|
| 154 |
+
|
| 155 |
+
img = Image.open(image_path)
|
| 156 |
+
transform = transforms.ToTensor()
|
| 157 |
+
img_tensor = transform(img).permute(1, 2, 0)[..., :3]
|
| 158 |
+
return img_tensor
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def main(
|
| 162 |
+
height: int = 256,
|
| 163 |
+
width: int = 256,
|
| 164 |
+
num_points: int = 100000,
|
| 165 |
+
save_imgs: bool = True,
|
| 166 |
+
img_path: Optional[Path] = None,
|
| 167 |
+
iterations: int = 1000,
|
| 168 |
+
lr: float = 0.01,
|
| 169 |
+
model_type: Literal["3dgs", "2dgs"] = "3dgs",
|
| 170 |
+
) -> None:
|
| 171 |
+
if img_path:
|
| 172 |
+
gt_image = image_path_to_tensor(img_path)
|
| 173 |
+
else:
|
| 174 |
+
gt_image = torch.ones((height, width, 3)) * 1.0
|
| 175 |
+
# make top left and bottom right red, blue
|
| 176 |
+
gt_image[: height // 2, : width // 2, :] = torch.tensor([1.0, 0.0, 0.0])
|
| 177 |
+
gt_image[height // 2 :, width // 2 :, :] = torch.tensor([0.0, 0.0, 1.0])
|
| 178 |
+
|
| 179 |
+
trainer = SimpleTrainer(gt_image=gt_image, num_points=num_points)
|
| 180 |
+
trainer.train(
|
| 181 |
+
iterations=iterations,
|
| 182 |
+
lr=lr,
|
| 183 |
+
save_imgs=save_imgs,
|
| 184 |
+
model_type=model_type,
|
| 185 |
+
)
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
if __name__ == "__main__":
|
| 189 |
+
tyro.cli(main)
|
src/post_opt/lib_bilagrid.py
ADDED
|
@@ -0,0 +1,573 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# # Copyright 2024 Yuehao Wang (https://github.com/yuehaowang). This part of code is borrowed form ["Bilateral Guided Radiance Field Processing"](https://bilarfpro.github.io/).
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
"""
|
| 16 |
+
This is a standalone PyTorch implementation of 3D bilateral grid and CP-decomposed 4D bilateral grid.
|
| 17 |
+
To use this module, you can download the "lib_bilagrid.py" file and simply put it in your project directory.
|
| 18 |
+
|
| 19 |
+
For the details, please check our research project: ["Bilateral Guided Radiance Field Processing"](https://bilarfpro.github.io/).
|
| 20 |
+
|
| 21 |
+
#### Dependencies
|
| 22 |
+
|
| 23 |
+
In addition to PyTorch and Numpy, please install [tensorly](https://github.com/tensorly/tensorly).
|
| 24 |
+
We have tested this module on Python 3.9.18, PyTorch 2.0.1 (CUDA 11), tensorly 0.8.1, and Numpy 1.25.2.
|
| 25 |
+
|
| 26 |
+
#### Overview
|
| 27 |
+
|
| 28 |
+
- For bilateral guided training, you need to construct a `BilateralGrid` instance, which can hold multiple bilateral grids
|
| 29 |
+
for input views. Then, use `slice` function to obtain transformed RGB output and the corresponding affine transformations.
|
| 30 |
+
|
| 31 |
+
- For bilateral guided finishing, you need to instantiate a `BilateralGridCP4D` object and use `slice4d`.
|
| 32 |
+
|
| 33 |
+
#### Examples
|
| 34 |
+
|
| 35 |
+
- Bilateral grid for approximating ISP:
|
| 36 |
+
<a target="_blank" href="https://colab.research.google.com/drive/1tx2qKtsHH9deDDnParMWrChcsa9i7Prr?usp=sharing">
|
| 37 |
+
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>
|
| 38 |
+
|
| 39 |
+
- Low-rank 4D bilateral grid for MR enhancement:
|
| 40 |
+
<a target="_blank" href="https://colab.research.google.com/drive/17YOjQqgWFT3QI1vysOIH494rMYtt_mHL?usp=sharing">
|
| 41 |
+
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
Below is the API reference.
|
| 45 |
+
|
| 46 |
+
"""
|
| 47 |
+
|
| 48 |
+
import tensorly as tl
|
| 49 |
+
import torch
|
| 50 |
+
import torch.nn.functional as F
|
| 51 |
+
from torch import nn
|
| 52 |
+
|
| 53 |
+
tl.set_backend("pytorch")
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def color_correct(
|
| 57 |
+
img: torch.Tensor, ref: torch.Tensor, num_iters: int = 5, eps: float = 0.5 / 255
|
| 58 |
+
) -> torch.Tensor:
|
| 59 |
+
"""
|
| 60 |
+
Warp `img` to match the colors in `ref_img` using iterative color matching.
|
| 61 |
+
|
| 62 |
+
This function performs color correction by warping the colors of the input image
|
| 63 |
+
to match those of a reference image. It uses a least squares method to find a
|
| 64 |
+
transformation that maps the input image's colors to the reference image's colors.
|
| 65 |
+
|
| 66 |
+
The algorithm iteratively solves a system of linear equations, updating the set of
|
| 67 |
+
unsaturated pixels in each iteration. This approach helps handle non-linear color
|
| 68 |
+
transformations and reduces the impact of clipping.
|
| 69 |
+
|
| 70 |
+
Args:
|
| 71 |
+
img (torch.Tensor): Input image to be color corrected. Shape: [..., num_channels]
|
| 72 |
+
ref (torch.Tensor): Reference image to match colors. Shape: [..., num_channels]
|
| 73 |
+
num_iters (int, optional): Number of iterations for the color matching process.
|
| 74 |
+
Default is 5.
|
| 75 |
+
eps (float, optional): Small value to determine the range of unclipped pixels.
|
| 76 |
+
Default is 0.5 / 255.
|
| 77 |
+
|
| 78 |
+
Returns:
|
| 79 |
+
torch.Tensor: Color corrected image with the same shape as the input image.
|
| 80 |
+
|
| 81 |
+
Note:
|
| 82 |
+
- Both input and reference images should be in the range [0, 1].
|
| 83 |
+
- The function works with any number of channels, but typically used with 3 (RGB).
|
| 84 |
+
"""
|
| 85 |
+
if img.shape[-1] != ref.shape[-1]:
|
| 86 |
+
raise ValueError(
|
| 87 |
+
f"img's {img.shape[-1]} and ref's {ref.shape[-1]} channels must match"
|
| 88 |
+
)
|
| 89 |
+
num_channels = img.shape[-1]
|
| 90 |
+
img_mat = img.reshape([-1, num_channels])
|
| 91 |
+
ref_mat = ref.reshape([-1, num_channels])
|
| 92 |
+
|
| 93 |
+
def is_unclipped(z):
|
| 94 |
+
return (z >= eps) & (z <= 1 - eps) # z \in [eps, 1-eps].
|
| 95 |
+
|
| 96 |
+
mask0 = is_unclipped(img_mat)
|
| 97 |
+
# Because the set of saturated pixels may change after solving for a
|
| 98 |
+
# transformation, we repeatedly solve a system `num_iters` times and update
|
| 99 |
+
# our estimate of which pixels are saturated.
|
| 100 |
+
for _ in range(num_iters):
|
| 101 |
+
# Construct the left hand side of a linear system that contains a quadratic
|
| 102 |
+
# expansion of each pixel of `img`.
|
| 103 |
+
a_mat = []
|
| 104 |
+
for c in range(num_channels):
|
| 105 |
+
a_mat.append(img_mat[:, c : (c + 1)] * img_mat[:, c:]) # Quadratic term.
|
| 106 |
+
a_mat.append(img_mat) # Linear term.
|
| 107 |
+
a_mat.append(torch.ones_like(img_mat[:, :1])) # Bias term.
|
| 108 |
+
a_mat = torch.cat(a_mat, dim=-1)
|
| 109 |
+
warp = []
|
| 110 |
+
for c in range(num_channels):
|
| 111 |
+
# Construct the right hand side of a linear system containing each color
|
| 112 |
+
# of `ref`.
|
| 113 |
+
b = ref_mat[:, c]
|
| 114 |
+
# Ignore rows of the linear system that were saturated in the input or are
|
| 115 |
+
# saturated in the current corrected color estimate.
|
| 116 |
+
mask = mask0[:, c] & is_unclipped(img_mat[:, c]) & is_unclipped(b)
|
| 117 |
+
ma_mat = torch.where(mask[:, None], a_mat, torch.zeros_like(a_mat))
|
| 118 |
+
mb = torch.where(mask, b, torch.zeros_like(b))
|
| 119 |
+
w = torch.linalg.lstsq(ma_mat, mb, rcond=-1)[0]
|
| 120 |
+
assert torch.all(torch.isfinite(w))
|
| 121 |
+
warp.append(w)
|
| 122 |
+
warp = torch.stack(warp, dim=-1)
|
| 123 |
+
# Apply the warp to update img_mat.
|
| 124 |
+
img_mat = torch.clip(torch.matmul(a_mat, warp), 0, 1)
|
| 125 |
+
corrected_img = torch.reshape(img_mat, img.shape)
|
| 126 |
+
return corrected_img
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def bilateral_grid_tv_loss(model, config):
|
| 130 |
+
"""Computes total variations of bilateral grids."""
|
| 131 |
+
total_loss = 0.0
|
| 132 |
+
|
| 133 |
+
for bil_grids in model.bil_grids:
|
| 134 |
+
total_loss += config.bilgrid_tv_loss_mult * total_variation_loss(
|
| 135 |
+
bil_grids.grids
|
| 136 |
+
)
|
| 137 |
+
|
| 138 |
+
return total_loss
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def color_affine_transform(affine_mats, rgb):
|
| 142 |
+
"""Applies color affine transformations.
|
| 143 |
+
|
| 144 |
+
Args:
|
| 145 |
+
affine_mats (torch.Tensor): Affine transformation matrices. Supported shape: $(..., 3, 4)$.
|
| 146 |
+
rgb (torch.Tensor): Input RGB values. Supported shape: $(..., 3)$.
|
| 147 |
+
|
| 148 |
+
Returns:
|
| 149 |
+
Output transformed colors of shape $(..., 3)$.
|
| 150 |
+
"""
|
| 151 |
+
return (
|
| 152 |
+
torch.matmul(affine_mats[..., :3], rgb.unsqueeze(-1)).squeeze(-1)
|
| 153 |
+
+ affine_mats[..., 3]
|
| 154 |
+
)
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def _num_tensor_elems(t):
|
| 158 |
+
return max(torch.prod(torch.tensor(t.size()[1:]).float()).item(), 1.0)
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def total_variation_loss(x): # noqa: F811
|
| 162 |
+
"""Returns total variation on multi-dimensional tensors.
|
| 163 |
+
|
| 164 |
+
Args:
|
| 165 |
+
x (torch.Tensor): The input tensor with shape $(B, C, ...)$, where $B$ is the batch size and $C$ is the channel size.
|
| 166 |
+
"""
|
| 167 |
+
batch_size = x.shape[0]
|
| 168 |
+
tv = 0
|
| 169 |
+
for i in range(2, len(x.shape)):
|
| 170 |
+
n_res = x.shape[i]
|
| 171 |
+
idx1 = torch.arange(1, n_res, device=x.device)
|
| 172 |
+
idx2 = torch.arange(0, n_res - 1, device=x.device)
|
| 173 |
+
x1 = x.index_select(i, idx1)
|
| 174 |
+
x2 = x.index_select(i, idx2)
|
| 175 |
+
count = _num_tensor_elems(x1)
|
| 176 |
+
tv += torch.pow((x1 - x2), 2).sum() / count
|
| 177 |
+
return tv / batch_size
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def slice(bil_grids, xy, rgb, grid_idx):
|
| 181 |
+
"""Slices a batch of 3D bilateral grids by pixel coordinates `xy` and gray-scale guidances of pixel colors `rgb`.
|
| 182 |
+
|
| 183 |
+
Supports 2-D, 3-D, and 4-D input shapes. The first dimension of the input is the batch size
|
| 184 |
+
and the last dimension is 2 for `xy`, 3 for `rgb`, and 1 for `grid_idx`.
|
| 185 |
+
|
| 186 |
+
The return value is a dictionary containing the affine transformations `affine_mats` sliced from bilateral grids and
|
| 187 |
+
the output color `rgb_out` after applying the afffine transformations.
|
| 188 |
+
|
| 189 |
+
In the 2-D input case, `xy` is a $(N, 2)$ tensor, `rgb` is a $(N, 3)$ tensor, and `grid_idx` is a $(N, 1)$ tensor.
|
| 190 |
+
Then `affine_mats[i]` can be obtained via slicing the bilateral grid indexed at `grid_idx[i]` by `xy[i, :]` and `rgb2gray(rgb[i, :])`.
|
| 191 |
+
For 3-D and 4-D input cases, the behavior of indexing bilateral grids and coordinates is the same with the 2-D case.
|
| 192 |
+
|
| 193 |
+
.. note::
|
| 194 |
+
This function can be regarded as a wrapper of `color_affine_transform` and `BilateralGrid` with a slight performance improvement.
|
| 195 |
+
When `grid_idx` contains a unique index, only a single bilateral grid will used during the slicing. In this case, this function will not
|
| 196 |
+
perform tensor indexing to avoid data copy and extra memory
|
| 197 |
+
(see [this](https://discuss.pytorch.org/t/does-indexing-a-tensor-return-a-copy-of-it/164905)).
|
| 198 |
+
|
| 199 |
+
Args:
|
| 200 |
+
bil_grids (`BilateralGrid`): An instance of $N$ bilateral grids.
|
| 201 |
+
xy (torch.Tensor): The x-y coordinates of shape $(..., 2)$ in the range of $[0,1]$.
|
| 202 |
+
rgb (torch.Tensor): The RGB values of shape $(..., 3)$ for computing the guidance coordinates, ranging in $[0,1]$.
|
| 203 |
+
grid_idx (torch.Tensor): The indices of bilateral grids for each slicing. Shape: $(..., 1)$.
|
| 204 |
+
|
| 205 |
+
Returns:
|
| 206 |
+
A dictionary with keys and values as follows:
|
| 207 |
+
```
|
| 208 |
+
{
|
| 209 |
+
"rgb": Transformed RGB colors. Shape: (..., 3),
|
| 210 |
+
"rgb_affine_mats": The sliced affine transformation matrices from bilateral grids. Shape: (..., 3, 4)
|
| 211 |
+
}
|
| 212 |
+
```
|
| 213 |
+
"""
|
| 214 |
+
|
| 215 |
+
sh_ = rgb.shape
|
| 216 |
+
|
| 217 |
+
grid_idx_unique = torch.unique(grid_idx)
|
| 218 |
+
if len(grid_idx_unique) == 1:
|
| 219 |
+
# All pixels are from a single view.
|
| 220 |
+
grid_idx = grid_idx_unique # (1,)
|
| 221 |
+
xy = xy.unsqueeze(0) # (1, ..., 2)
|
| 222 |
+
rgb = rgb.unsqueeze(0) # (1, ..., 3)
|
| 223 |
+
else:
|
| 224 |
+
# Pixels are randomly sampled from different views.
|
| 225 |
+
if len(grid_idx.shape) == 4:
|
| 226 |
+
grid_idx = grid_idx[:, 0, 0, 0] # (chunk_size,)
|
| 227 |
+
elif len(grid_idx.shape) == 3:
|
| 228 |
+
grid_idx = grid_idx[:, 0, 0] # (chunk_size,)
|
| 229 |
+
elif len(grid_idx.shape) == 2:
|
| 230 |
+
grid_idx = grid_idx[:, 0] # (chunk_size,)
|
| 231 |
+
else:
|
| 232 |
+
raise ValueError(
|
| 233 |
+
"The input to bilateral grid slicing is not supported yet."
|
| 234 |
+
)
|
| 235 |
+
|
| 236 |
+
affine_mats = bil_grids(xy, rgb, grid_idx)
|
| 237 |
+
rgb = color_affine_transform(affine_mats, rgb)
|
| 238 |
+
|
| 239 |
+
return {
|
| 240 |
+
"rgb": rgb.reshape(*sh_),
|
| 241 |
+
"rgb_affine_mats": affine_mats.reshape(
|
| 242 |
+
*sh_[:-1], affine_mats.shape[-2], affine_mats.shape[-1]
|
| 243 |
+
),
|
| 244 |
+
}
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
class BilateralGrid(nn.Module):
|
| 248 |
+
"""Class for 3D bilateral grids.
|
| 249 |
+
|
| 250 |
+
Holds one or more than one bilateral grids.
|
| 251 |
+
"""
|
| 252 |
+
|
| 253 |
+
def __init__(self, num, grid_X=16, grid_Y=16, grid_W=8):
|
| 254 |
+
"""
|
| 255 |
+
Args:
|
| 256 |
+
num (int): The number of bilateral grids (i.e., the number of views).
|
| 257 |
+
grid_X (int): Defines grid width $W$.
|
| 258 |
+
grid_Y (int): Defines grid height $H$.
|
| 259 |
+
grid_W (int): Defines grid guidance dimension $L$.
|
| 260 |
+
"""
|
| 261 |
+
super(BilateralGrid, self).__init__()
|
| 262 |
+
|
| 263 |
+
self.grid_width = grid_X
|
| 264 |
+
"""Grid width. Type: int."""
|
| 265 |
+
self.grid_height = grid_Y
|
| 266 |
+
"""Grid height. Type: int."""
|
| 267 |
+
self.grid_guidance = grid_W
|
| 268 |
+
"""Grid guidance dimension. Type: int."""
|
| 269 |
+
|
| 270 |
+
# Initialize grids.
|
| 271 |
+
grid = self._init_identity_grid()
|
| 272 |
+
self.grids = nn.Parameter(grid.tile(num, 1, 1, 1, 1)) # (N, 12, L, H, W)
|
| 273 |
+
""" A 5-D tensor of shape $(N, 12, L, H, W)$."""
|
| 274 |
+
|
| 275 |
+
# Weights of BT601 RGB-to-gray.
|
| 276 |
+
self.register_buffer("rgb2gray_weight", torch.Tensor([[0.299, 0.587, 0.114]]))
|
| 277 |
+
self.rgb2gray = lambda rgb: (rgb @ self.rgb2gray_weight.T) * 2.0 - 1.0
|
| 278 |
+
""" A function that converts RGB to gray-scale guidance in $[-1, 1]$."""
|
| 279 |
+
|
| 280 |
+
def _init_identity_grid(self):
|
| 281 |
+
grid = torch.tensor(
|
| 282 |
+
[
|
| 283 |
+
1.0,
|
| 284 |
+
0,
|
| 285 |
+
0,
|
| 286 |
+
0,
|
| 287 |
+
0,
|
| 288 |
+
1.0,
|
| 289 |
+
0,
|
| 290 |
+
0,
|
| 291 |
+
0,
|
| 292 |
+
0,
|
| 293 |
+
1.0,
|
| 294 |
+
0,
|
| 295 |
+
]
|
| 296 |
+
).float()
|
| 297 |
+
grid = grid.repeat(
|
| 298 |
+
[self.grid_guidance * self.grid_height * self.grid_width, 1]
|
| 299 |
+
) # (L * H * W, 12)
|
| 300 |
+
grid = grid.reshape(
|
| 301 |
+
1, self.grid_guidance, self.grid_height, self.grid_width, -1
|
| 302 |
+
) # (1, L, H, W, 12)
|
| 303 |
+
grid = grid.permute(0, 4, 1, 2, 3) # (1, 12, L, H, W)
|
| 304 |
+
return grid
|
| 305 |
+
|
| 306 |
+
def tv_loss(self):
|
| 307 |
+
"""Computes and returns total variation loss on the bilateral grids."""
|
| 308 |
+
return total_variation_loss(self.grids)
|
| 309 |
+
|
| 310 |
+
def forward(self, grid_xy, rgb, idx=None):
|
| 311 |
+
"""Bilateral grid slicing. Supports 2-D, 3-D, 4-D, and 5-D input.
|
| 312 |
+
For the 2-D, 3-D, and 4-D cases, please refer to `slice`.
|
| 313 |
+
For the 5-D cases, `idx` will be unused and the first dimension of `xy` should be
|
| 314 |
+
equal to the number of bilateral grids. Then this function becomes PyTorch's
|
| 315 |
+
[`F.grid_sample`](https://pytorch.org/docs/stable/generated/torch.nn.functional.grid_sample.html).
|
| 316 |
+
|
| 317 |
+
Args:
|
| 318 |
+
grid_xy (torch.Tensor): The x-y coordinates in the range of $[0,1]$.
|
| 319 |
+
rgb (torch.Tensor): The RGB values in the range of $[0,1]$.
|
| 320 |
+
idx (torch.Tensor): The bilateral grid indices.
|
| 321 |
+
|
| 322 |
+
Returns:
|
| 323 |
+
Sliced affine matrices of shape $(..., 3, 4)$.
|
| 324 |
+
"""
|
| 325 |
+
|
| 326 |
+
grids = self.grids
|
| 327 |
+
input_ndims = len(grid_xy.shape)
|
| 328 |
+
assert len(rgb.shape) == input_ndims
|
| 329 |
+
|
| 330 |
+
if input_ndims > 1 and input_ndims < 5:
|
| 331 |
+
# Convert input into 5D
|
| 332 |
+
for i in range(5 - input_ndims):
|
| 333 |
+
grid_xy = grid_xy.unsqueeze(1)
|
| 334 |
+
rgb = rgb.unsqueeze(1)
|
| 335 |
+
assert idx is not None
|
| 336 |
+
elif input_ndims != 5:
|
| 337 |
+
raise ValueError(
|
| 338 |
+
"Bilateral grid slicing only takes either 2D, 3D, 4D and 5D inputs"
|
| 339 |
+
)
|
| 340 |
+
|
| 341 |
+
grids = self.grids
|
| 342 |
+
if idx is not None:
|
| 343 |
+
grids = grids[idx]
|
| 344 |
+
assert grids.shape[0] == grid_xy.shape[0]
|
| 345 |
+
|
| 346 |
+
# Generate slicing coordinates.
|
| 347 |
+
grid_xy = (grid_xy - 0.5) * 2 # Rescale to [-1, 1].
|
| 348 |
+
grid_z = self.rgb2gray(rgb)
|
| 349 |
+
|
| 350 |
+
# print(grid_xy.shape, grid_z.shape)
|
| 351 |
+
# exit()
|
| 352 |
+
grid_xyz = torch.cat([grid_xy, grid_z], dim=-1) # (N, m, h, w, 3)
|
| 353 |
+
|
| 354 |
+
affine_mats = F.grid_sample(
|
| 355 |
+
grids, grid_xyz, mode="bilinear", align_corners=True, padding_mode="border"
|
| 356 |
+
) # (N, 12, m, h, w)
|
| 357 |
+
affine_mats = affine_mats.permute(0, 2, 3, 4, 1) # (N, m, h, w, 12)
|
| 358 |
+
affine_mats = affine_mats.reshape(
|
| 359 |
+
*affine_mats.shape[:-1], 3, 4
|
| 360 |
+
) # (N, m, h, w, 3, 4)
|
| 361 |
+
|
| 362 |
+
for _ in range(5 - input_ndims):
|
| 363 |
+
affine_mats = affine_mats.squeeze(1)
|
| 364 |
+
|
| 365 |
+
return affine_mats
|
| 366 |
+
|
| 367 |
+
|
| 368 |
+
def slice4d(bil_grid4d, xyz, rgb):
|
| 369 |
+
"""Slices a 4D bilateral grid by point coordinates `xyz` and gray-scale guidances of radiance colors `rgb`.
|
| 370 |
+
|
| 371 |
+
Args:
|
| 372 |
+
bil_grid4d (`BilateralGridCP4D`): The input 4D bilateral grid.
|
| 373 |
+
xyz (torch.Tensor): The xyz coordinates with shape $(..., 3)$.
|
| 374 |
+
rgb (torch.Tensor): The RGB values with shape $(..., 3)$.
|
| 375 |
+
|
| 376 |
+
Returns:
|
| 377 |
+
A dictionary with keys and values as follows:
|
| 378 |
+
```
|
| 379 |
+
{
|
| 380 |
+
"rgb": Transformed radiance RGB colors. Shape: (..., 3),
|
| 381 |
+
"rgb_affine_mats": The sliced affine transformation matrices from the 4D bilateral grid. Shape: (..., 3, 4)
|
| 382 |
+
}
|
| 383 |
+
```
|
| 384 |
+
"""
|
| 385 |
+
|
| 386 |
+
affine_mats = bil_grid4d(xyz, rgb)
|
| 387 |
+
rgb = color_affine_transform(affine_mats, rgb)
|
| 388 |
+
|
| 389 |
+
return {"rgb": rgb, "rgb_affine_mats": affine_mats}
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
class _ScaledTanh(nn.Module):
|
| 393 |
+
def __init__(self, s=2.0):
|
| 394 |
+
super().__init__()
|
| 395 |
+
self.scaler = s
|
| 396 |
+
|
| 397 |
+
def forward(self, x):
|
| 398 |
+
return torch.tanh(self.scaler * x)
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
class BilateralGridCP4D(nn.Module):
|
| 402 |
+
"""Class for low-rank 4D bilateral grids."""
|
| 403 |
+
|
| 404 |
+
def __init__(
|
| 405 |
+
self,
|
| 406 |
+
grid_X=16,
|
| 407 |
+
grid_Y=16,
|
| 408 |
+
grid_Z=16,
|
| 409 |
+
grid_W=8,
|
| 410 |
+
rank=5,
|
| 411 |
+
learn_gray=True,
|
| 412 |
+
gray_mlp_width=8,
|
| 413 |
+
gray_mlp_depth=2,
|
| 414 |
+
init_noise_scale=1e-6,
|
| 415 |
+
bound=2.0,
|
| 416 |
+
):
|
| 417 |
+
"""
|
| 418 |
+
Args:
|
| 419 |
+
grid_X (int): Defines grid width.
|
| 420 |
+
grid_Y (int): Defines grid height.
|
| 421 |
+
grid_Z (int): Defines grid depth.
|
| 422 |
+
grid_W (int): Defines grid guidance dimension.
|
| 423 |
+
rank (int): Rank of the 4D bilateral grid.
|
| 424 |
+
learn_gray (bool): If True, an MLP will be learned to convert RGB colors to gray-scale guidances.
|
| 425 |
+
gray_mlp_width (int): The MLP width for learnable guidance.
|
| 426 |
+
gray_mlp_depth (int): The number of MLP layers for learnable guidance.
|
| 427 |
+
init_noise_scale (float): The noise scale of the initialized factors.
|
| 428 |
+
bound (float): The bound of the xyz coordinates.
|
| 429 |
+
"""
|
| 430 |
+
super(BilateralGridCP4D, self).__init__()
|
| 431 |
+
|
| 432 |
+
self.grid_X = grid_X
|
| 433 |
+
"""Grid width. Type: int."""
|
| 434 |
+
self.grid_Y = grid_Y
|
| 435 |
+
"""Grid height. Type: int."""
|
| 436 |
+
self.grid_Z = grid_Z
|
| 437 |
+
"""Grid depth. Type: int."""
|
| 438 |
+
self.grid_W = grid_W
|
| 439 |
+
"""Grid guidance dimension. Type: int."""
|
| 440 |
+
self.rank = rank
|
| 441 |
+
"""Rank of the 4D bilateral grid. Type: int."""
|
| 442 |
+
self.learn_gray = learn_gray
|
| 443 |
+
"""Flags of learnable guidance is used. Type: bool."""
|
| 444 |
+
self.gray_mlp_width = gray_mlp_width
|
| 445 |
+
"""The MLP width for learnable guidance. Type: int."""
|
| 446 |
+
self.gray_mlp_depth = gray_mlp_depth
|
| 447 |
+
"""The MLP depth for learnable guidance. Type: int."""
|
| 448 |
+
self.init_noise_scale = init_noise_scale
|
| 449 |
+
"""The noise scale of the initialized factors. Type: float."""
|
| 450 |
+
self.bound = bound
|
| 451 |
+
"""The bound of the xyz coordinates. Type: float."""
|
| 452 |
+
|
| 453 |
+
self._init_cp_factors_parafac()
|
| 454 |
+
|
| 455 |
+
self.rgb2gray = None
|
| 456 |
+
""" A function that converts RGB to gray-scale guidances in $[-1, 1]$.
|
| 457 |
+
If `learn_gray` is True, this will be an MLP network."""
|
| 458 |
+
|
| 459 |
+
if self.learn_gray:
|
| 460 |
+
|
| 461 |
+
def rgb2gray_mlp_linear(layer):
|
| 462 |
+
return nn.Linear(
|
| 463 |
+
self.gray_mlp_width,
|
| 464 |
+
self.gray_mlp_width if layer < self.gray_mlp_depth - 1 else 1,
|
| 465 |
+
)
|
| 466 |
+
|
| 467 |
+
def rgb2gray_mlp_actfn(_):
|
| 468 |
+
return nn.ReLU(inplace=True)
|
| 469 |
+
|
| 470 |
+
self.rgb2gray = nn.Sequential(
|
| 471 |
+
*(
|
| 472 |
+
[nn.Linear(3, self.gray_mlp_width)]
|
| 473 |
+
+ [
|
| 474 |
+
nn_module(layer)
|
| 475 |
+
for layer in range(1, self.gray_mlp_depth)
|
| 476 |
+
for nn_module in [rgb2gray_mlp_actfn, rgb2gray_mlp_linear]
|
| 477 |
+
]
|
| 478 |
+
+ [_ScaledTanh(2.0)]
|
| 479 |
+
)
|
| 480 |
+
)
|
| 481 |
+
else:
|
| 482 |
+
# Weights of BT601/BT470 RGB-to-gray.
|
| 483 |
+
self.register_buffer(
|
| 484 |
+
"rgb2gray_weight", torch.Tensor([[0.299, 0.587, 0.114]])
|
| 485 |
+
)
|
| 486 |
+
self.rgb2gray = lambda rgb: (rgb @ self.rgb2gray_weight.T) * 2.0 - 1.0
|
| 487 |
+
|
| 488 |
+
def _init_identity_grid(self):
|
| 489 |
+
grid = torch.tensor(
|
| 490 |
+
[
|
| 491 |
+
1.0,
|
| 492 |
+
0,
|
| 493 |
+
0,
|
| 494 |
+
0,
|
| 495 |
+
0,
|
| 496 |
+
1.0,
|
| 497 |
+
0,
|
| 498 |
+
0,
|
| 499 |
+
0,
|
| 500 |
+
0,
|
| 501 |
+
1.0,
|
| 502 |
+
0,
|
| 503 |
+
]
|
| 504 |
+
).float()
|
| 505 |
+
grid = grid.repeat([self.grid_W * self.grid_Z * self.grid_Y * self.grid_X, 1])
|
| 506 |
+
grid = grid.reshape(self.grid_W, self.grid_Z, self.grid_Y, self.grid_X, -1)
|
| 507 |
+
grid = grid.permute(4, 0, 1, 2, 3) # (12, grid_W, grid_Z, grid_Y, grid_X)
|
| 508 |
+
return grid
|
| 509 |
+
|
| 510 |
+
def _init_cp_factors_parafac(self):
|
| 511 |
+
# Initialize identity grids.
|
| 512 |
+
init_grids = self._init_identity_grid()
|
| 513 |
+
# Random noises are added to avoid singularity.
|
| 514 |
+
init_grids = torch.randn_like(init_grids) * self.init_noise_scale + init_grids
|
| 515 |
+
from tensorly.decomposition import parafac
|
| 516 |
+
|
| 517 |
+
# Initialize grid CP factors
|
| 518 |
+
_, facs = parafac(init_grids.clone().detach(), rank=self.rank)
|
| 519 |
+
|
| 520 |
+
self.num_facs = len(facs)
|
| 521 |
+
|
| 522 |
+
self.fac_0 = nn.Linear(facs[0].shape[0], facs[0].shape[1], bias=False)
|
| 523 |
+
self.fac_0.weight = nn.Parameter(facs[0]) # (12, rank)
|
| 524 |
+
|
| 525 |
+
for i in range(1, self.num_facs):
|
| 526 |
+
fac = facs[i].T # (rank, grid_size)
|
| 527 |
+
fac = fac.view(1, fac.shape[0], fac.shape[1], 1) # (1, rank, grid_size, 1)
|
| 528 |
+
self.register_buffer(f"fac_{i}_init", fac)
|
| 529 |
+
|
| 530 |
+
fac_resid = torch.zeros_like(fac)
|
| 531 |
+
self.register_parameter(f"fac_{i}", nn.Parameter(fac_resid))
|
| 532 |
+
|
| 533 |
+
def tv_loss(self):
|
| 534 |
+
"""Computes and returns total variation loss on the factors of the low-rank 4D bilateral grids."""
|
| 535 |
+
|
| 536 |
+
total_loss = 0
|
| 537 |
+
for i in range(1, self.num_facs):
|
| 538 |
+
fac = self.get_parameter(f"fac_{i}")
|
| 539 |
+
total_loss += total_variation_loss(fac)
|
| 540 |
+
|
| 541 |
+
return total_loss
|
| 542 |
+
|
| 543 |
+
def forward(self, xyz, rgb):
|
| 544 |
+
"""Low-rank 4D bilateral grid slicing.
|
| 545 |
+
|
| 546 |
+
Args:
|
| 547 |
+
xyz (torch.Tensor): The xyz coordinates with shape $(..., 3)$.
|
| 548 |
+
rgb (torch.Tensor): The corresponding RGB values with shape $(..., 3)$.
|
| 549 |
+
|
| 550 |
+
Returns:
|
| 551 |
+
Sliced affine matrices with shape $(..., 3, 4)$.
|
| 552 |
+
"""
|
| 553 |
+
sh_ = xyz.shape
|
| 554 |
+
xyz = xyz.reshape(-1, 3) # flatten (N, 3)
|
| 555 |
+
rgb = rgb.reshape(-1, 3) # flatten (N, 3)
|
| 556 |
+
|
| 557 |
+
xyz = xyz / self.bound
|
| 558 |
+
assert self.rgb2gray is not None
|
| 559 |
+
gray = self.rgb2gray(rgb)
|
| 560 |
+
xyzw = torch.cat([xyz, gray], dim=-1) # (N, 4)
|
| 561 |
+
xyzw = xyzw.transpose(0, 1) # (4, N)
|
| 562 |
+
coords = torch.stack([torch.zeros_like(xyzw), xyzw], dim=-1) # (4, N, 2)
|
| 563 |
+
coords = coords.unsqueeze(1) # (4, 1, N, 2)
|
| 564 |
+
|
| 565 |
+
coef = 1.0
|
| 566 |
+
for i in range(1, self.num_facs):
|
| 567 |
+
fac = self.get_parameter(f"fac_{i}") + self.get_buffer(f"fac_{i}_init")
|
| 568 |
+
coef = coef * F.grid_sample(
|
| 569 |
+
fac, coords[[i - 1]], align_corners=True, padding_mode="border"
|
| 570 |
+
) # [1, rank, 1, N]
|
| 571 |
+
coef = coef.squeeze([0, 2]).transpose(0, 1) # (N, rank) #type: ignore
|
| 572 |
+
mat = self.fac_0(coef)
|
| 573 |
+
return mat.reshape(*sh_[:-1], 3, 4)
|
src/post_opt/simple_trainer.py
ADDED
|
@@ -0,0 +1,1433 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import math
|
| 3 |
+
import os
|
| 4 |
+
import time
|
| 5 |
+
from collections import defaultdict
|
| 6 |
+
from dataclasses import dataclass, field
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from typing import Dict, List, Optional, Tuple, Union
|
| 9 |
+
|
| 10 |
+
import imageio
|
| 11 |
+
import matplotlib
|
| 12 |
+
import torchvision
|
| 13 |
+
import numpy as np
|
| 14 |
+
import torch
|
| 15 |
+
import torch.nn.functional as F
|
| 16 |
+
import tqdm
|
| 17 |
+
import tyro
|
| 18 |
+
import viser
|
| 19 |
+
import yaml
|
| 20 |
+
import torchvision
|
| 21 |
+
import sys
|
| 22 |
+
from plyfile import PlyData, PlyElement
|
| 23 |
+
|
| 24 |
+
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
| 25 |
+
|
| 26 |
+
from src.model.encoder.vggt.utils.pose_enc import pose_encoding_to_extri_intri
|
| 27 |
+
from src.model.types import Gaussians
|
| 28 |
+
from src.post_opt.datasets.colmap import Dataset, Parser
|
| 29 |
+
from src.post_opt.datasets.traj import (
|
| 30 |
+
generate_ellipse_path_z,
|
| 31 |
+
generate_interpolated_path,
|
| 32 |
+
generate_spiral_path,
|
| 33 |
+
)
|
| 34 |
+
from fused_ssim import fused_ssim
|
| 35 |
+
|
| 36 |
+
from src.utils.image import process_image
|
| 37 |
+
from src.post_opt.exporter import export_splats
|
| 38 |
+
from src.post_opt.lib_bilagrid import BilateralGrid, color_correct, slice, total_variation_loss
|
| 39 |
+
from torch import Tensor
|
| 40 |
+
from torch.nn.parallel import DistributedDataParallel as DDP
|
| 41 |
+
from torch.utils.tensorboard import SummaryWriter
|
| 42 |
+
from torchmetrics.image import PeakSignalNoiseRatio, StructuralSimilarityIndexMeasure
|
| 43 |
+
from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
|
| 44 |
+
from typing_extensions import Literal, assert_never
|
| 45 |
+
from src.post_opt.utils import AppearanceOptModule, CameraOptModule, knn, rgb_to_sh, set_random_seed
|
| 46 |
+
|
| 47 |
+
# from gsplat import export_splats
|
| 48 |
+
from gsplat.compression import PngCompression
|
| 49 |
+
from gsplat.distributed import cli
|
| 50 |
+
# from gsplat.optimizers import SelectiveAdam
|
| 51 |
+
# from gsplat.rendering import rasterization
|
| 52 |
+
from gsplat import rasterization
|
| 53 |
+
from gsplat.strategy import DefaultStrategy, MCMCStrategy
|
| 54 |
+
from src.post_opt.gsplat_viewer import GsplatViewer, GsplatRenderTabState
|
| 55 |
+
from nerfview import CameraState, RenderTabState, apply_float_colormap
|
| 56 |
+
|
| 57 |
+
import torch
|
| 58 |
+
from einops import rearrange
|
| 59 |
+
from jaxtyping import Float
|
| 60 |
+
from torch import Tensor
|
| 61 |
+
from scipy.spatial.transform import Rotation as R
|
| 62 |
+
|
| 63 |
+
from src.model.model.anysplat import AnySplat
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
# pytorch3d/pytorch3d/transforms/rotation_conversions.py at main · facebookresearch/pytorch3d
|
| 67 |
+
def quaternion_to_matrix(
|
| 68 |
+
quaternions: Float[Tensor, "*batch 4"],
|
| 69 |
+
eps: float = 1e-8,
|
| 70 |
+
) -> Float[Tensor, "*batch 3 3"]:
|
| 71 |
+
# Order changed to match scipy format!
|
| 72 |
+
i, j, k, r = torch.unbind(quaternions, dim=-1)
|
| 73 |
+
two_s = 2 / ((quaternions * quaternions).sum(dim=-1) + eps)
|
| 74 |
+
|
| 75 |
+
o = torch.stack(
|
| 76 |
+
(
|
| 77 |
+
1 - two_s * (j * j + k * k),
|
| 78 |
+
two_s * (i * j - k * r),
|
| 79 |
+
two_s * (i * k + j * r),
|
| 80 |
+
two_s * (i * j + k * r),
|
| 81 |
+
1 - two_s * (i * i + k * k),
|
| 82 |
+
two_s * (j * k - i * r),
|
| 83 |
+
two_s * (i * k - j * r),
|
| 84 |
+
two_s * (j * k + i * r),
|
| 85 |
+
1 - two_s * (i * i + j * j),
|
| 86 |
+
),
|
| 87 |
+
-1,
|
| 88 |
+
)
|
| 89 |
+
return rearrange(o, "... (i j) -> ... i j", i=3, j=3)
|
| 90 |
+
|
| 91 |
+
def construct_list_of_attributes(num_rest: int) -> list[str]:
|
| 92 |
+
attributes = ["x", "y", "z", "nx", "ny", "nz"]
|
| 93 |
+
for i in range(3):
|
| 94 |
+
attributes.append(f"f_dc_{i}")
|
| 95 |
+
for i in range(num_rest):
|
| 96 |
+
attributes.append(f"f_rest_{i}")
|
| 97 |
+
attributes.append("opacity")
|
| 98 |
+
for i in range(3):
|
| 99 |
+
attributes.append(f"scale_{i}")
|
| 100 |
+
for i in range(4):
|
| 101 |
+
attributes.append(f"rot_{i}")
|
| 102 |
+
return attributes
|
| 103 |
+
|
| 104 |
+
def export_ply(
|
| 105 |
+
means: Float[Tensor, "gaussian 3"],
|
| 106 |
+
scales: Float[Tensor, "gaussian 3"],
|
| 107 |
+
rotations: Float[Tensor, "gaussian 4"],
|
| 108 |
+
harmonics: Float[Tensor, "gaussian 3 d_sh"],
|
| 109 |
+
opacities: Float[Tensor, " gaussian"],
|
| 110 |
+
path: Path,
|
| 111 |
+
shift_and_scale: bool = False,
|
| 112 |
+
save_sh_dc_only: bool = True,
|
| 113 |
+
):
|
| 114 |
+
if shift_and_scale:
|
| 115 |
+
# Shift the scene so that the median Gaussian is at the origin.
|
| 116 |
+
means = means - means.median(dim=0).values
|
| 117 |
+
|
| 118 |
+
# Rescale the scene so that most Gaussians are within range [-1, 1].
|
| 119 |
+
scale_factor = means.abs().quantile(0.95, dim=0).max()
|
| 120 |
+
means = means / scale_factor
|
| 121 |
+
scales = scales / scale_factor
|
| 122 |
+
|
| 123 |
+
# Apply the rotation to the Gaussian rotations.
|
| 124 |
+
rotations = R.from_quat(rotations.detach().cpu().numpy()).as_matrix()
|
| 125 |
+
rotations = R.from_matrix(rotations).as_quat()
|
| 126 |
+
x, y, z, w = rearrange(rotations, "g xyzw -> xyzw g")
|
| 127 |
+
rotations = np.stack((w, x, y, z), axis=-1)
|
| 128 |
+
|
| 129 |
+
# Since current model use SH_degree = 4,
|
| 130 |
+
# which require large memory to store, we can only save the DC band to save memory.
|
| 131 |
+
f_dc = harmonics[..., 0]
|
| 132 |
+
f_rest = harmonics[..., 1:].flatten(start_dim=1)
|
| 133 |
+
|
| 134 |
+
dtype_full = [(attribute, "f4") for attribute in construct_list_of_attributes(0 if save_sh_dc_only else f_rest.shape[1])]
|
| 135 |
+
elements = np.empty(means.shape[0], dtype=dtype_full)
|
| 136 |
+
attributes = [
|
| 137 |
+
means.detach().cpu().numpy(),
|
| 138 |
+
torch.zeros_like(means).detach().cpu().numpy(),
|
| 139 |
+
f_dc.detach().cpu().contiguous().numpy(),
|
| 140 |
+
f_rest.detach().cpu().contiguous().numpy(),
|
| 141 |
+
opacities[..., None].detach().cpu().numpy(),
|
| 142 |
+
scales.detach().cpu().numpy(),
|
| 143 |
+
rotations,
|
| 144 |
+
]
|
| 145 |
+
if save_sh_dc_only:
|
| 146 |
+
# remove f_rest from attributes
|
| 147 |
+
attributes.pop(3)
|
| 148 |
+
|
| 149 |
+
attributes = np.concatenate(attributes, axis=1)
|
| 150 |
+
elements[:] = list(map(tuple, attributes))
|
| 151 |
+
path.parent.mkdir(exist_ok=True, parents=True)
|
| 152 |
+
PlyData([PlyElement.describe(elements, "vertex")]).write(path)
|
| 153 |
+
|
| 154 |
+
def colorize_depth_maps(depth_map, min_depth=0.0, max_depth=1.0, cmap="Spectral", valid_mask=None):
|
| 155 |
+
"""
|
| 156 |
+
Colorize depth maps.
|
| 157 |
+
"""
|
| 158 |
+
assert len(depth_map.shape) >= 2, "Invalid dimension"
|
| 159 |
+
|
| 160 |
+
if isinstance(depth_map, torch.Tensor):
|
| 161 |
+
depth = depth_map.detach().clone().squeeze().numpy()
|
| 162 |
+
elif isinstance(depth_map, np.ndarray):
|
| 163 |
+
depth = depth_map.copy().squeeze()
|
| 164 |
+
# reshape to [ (B,) H, W ]
|
| 165 |
+
if depth.ndim < 3:
|
| 166 |
+
depth = depth[np.newaxis, :, :]
|
| 167 |
+
|
| 168 |
+
# colorize
|
| 169 |
+
cm = matplotlib.colormaps[cmap]
|
| 170 |
+
# depth = ((depth - min_depth) / (max_depth - min_depth)).clip(0, 1)
|
| 171 |
+
depth = ((depth - depth.min()) / (depth.max() - depth.min())).clip(0, 1)
|
| 172 |
+
img_colored_np = cm(depth, bytes=False)[:, :, :, 0:3] # value from 0 to 1
|
| 173 |
+
img_colored_np = np.rollaxis(img_colored_np, 3, 1)
|
| 174 |
+
|
| 175 |
+
if valid_mask is not None:
|
| 176 |
+
if isinstance(depth_map, torch.Tensor):
|
| 177 |
+
valid_mask = valid_mask.detach().numpy()
|
| 178 |
+
valid_mask = valid_mask.squeeze() # [H, W] or [B, H, W]
|
| 179 |
+
if valid_mask.ndim < 3:
|
| 180 |
+
valid_mask = valid_mask[np.newaxis, np.newaxis, :, :]
|
| 181 |
+
else:
|
| 182 |
+
valid_mask = valid_mask[:, np.newaxis, :, :]
|
| 183 |
+
valid_mask = np.repeat(valid_mask, 3, axis=1)
|
| 184 |
+
img_colored_np[~valid_mask] = 0
|
| 185 |
+
|
| 186 |
+
if isinstance(depth_map, torch.Tensor):
|
| 187 |
+
img_colored = torch.from_numpy(img_colored_np).float()
|
| 188 |
+
elif isinstance(depth_map, np.ndarray):
|
| 189 |
+
img_colored = img_colored_np
|
| 190 |
+
|
| 191 |
+
return img_colored
|
| 192 |
+
|
| 193 |
+
def build_covariance(
|
| 194 |
+
scale: Float[Tensor, "*#batch 3"],
|
| 195 |
+
rotation_xyzw: Float[Tensor, "*#batch 4"],
|
| 196 |
+
) -> Float[Tensor, "*batch 3 3"]:
|
| 197 |
+
scale = scale.diag_embed()
|
| 198 |
+
rotation = quaternion_to_matrix(rotation_xyzw)
|
| 199 |
+
return (
|
| 200 |
+
rotation
|
| 201 |
+
@ scale
|
| 202 |
+
@ rearrange(scale, "... i j -> ... j i")
|
| 203 |
+
@ rearrange(rotation, "... i j -> ... j i")
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
@dataclass
|
| 208 |
+
class Config:
|
| 209 |
+
# Disable viewer
|
| 210 |
+
disable_viewer: bool = True
|
| 211 |
+
# Path to the .pt files. If provide, it will skip training and run evaluation only.
|
| 212 |
+
ckpt: Optional[List[str]] = None
|
| 213 |
+
# Name of compression strategy to use
|
| 214 |
+
compression: Optional[Literal["png"]] = None
|
| 215 |
+
# Render trajectory path
|
| 216 |
+
render_traj_path: str = "interp"
|
| 217 |
+
|
| 218 |
+
data_dir: str = "data/360_v2/garden"
|
| 219 |
+
# Downsample factor for the dataset
|
| 220 |
+
data_factor: int = 4
|
| 221 |
+
# Directory to save results
|
| 222 |
+
result_dir: str = "results/garden"
|
| 223 |
+
# Every N images there is a test image
|
| 224 |
+
test_every: int = 8
|
| 225 |
+
# Random crop size for training (experimental)
|
| 226 |
+
patch_size: Optional[int] = None
|
| 227 |
+
# A global scaler that applies to the scene size related parameters
|
| 228 |
+
global_scale: float = 1.0
|
| 229 |
+
# Normalize the world space
|
| 230 |
+
normalize_world_space: bool = True
|
| 231 |
+
# Camera model
|
| 232 |
+
camera_model: Literal["pinhole", "ortho", "fisheye"] = "pinhole"
|
| 233 |
+
|
| 234 |
+
# Port for the viewer server
|
| 235 |
+
port: int = 8080
|
| 236 |
+
|
| 237 |
+
# Batch size for training. Learning rates are scaled automatically
|
| 238 |
+
batch_size: int = 1
|
| 239 |
+
# A global factor to scale the number of training steps
|
| 240 |
+
steps_scaler: float = 1.0
|
| 241 |
+
|
| 242 |
+
# Number of training steps
|
| 243 |
+
max_steps: int = 3_000
|
| 244 |
+
# Steps to evaluate the model
|
| 245 |
+
eval_steps: List[int] = field(default_factory=lambda: [1, 1_000, 3_000, 7_000, 10_000])
|
| 246 |
+
# Steps to save the model
|
| 247 |
+
save_steps: List[int] = field(default_factory=lambda: [1, 1_000, 3_000, 7_000, 10_000])
|
| 248 |
+
# Whether to save ply file (storage size can be large)
|
| 249 |
+
save_ply: bool = False
|
| 250 |
+
# Steps to save the model as ply
|
| 251 |
+
ply_steps: List[int] = field(default_factory=lambda: [1, 1_000, 3_000, 7_000, 10_000])
|
| 252 |
+
# Whether to disable video generation during training and evaluation
|
| 253 |
+
disable_video: bool = False
|
| 254 |
+
|
| 255 |
+
# Initialization strategy
|
| 256 |
+
init_type: str = "sfm"
|
| 257 |
+
# Initial number of GSs. Ignored if using sfm
|
| 258 |
+
init_num_pts: int = 100_000
|
| 259 |
+
# Initial extent of GSs as a multiple of the camera extent. Ignored if using sfm
|
| 260 |
+
init_extent: float = 3.0
|
| 261 |
+
# Degree of spherical harmonics
|
| 262 |
+
sh_degree: int = 4
|
| 263 |
+
# Turn on another SH degree every this steps
|
| 264 |
+
sh_degree_interval: int = 1000
|
| 265 |
+
# Initial opacity of GS
|
| 266 |
+
init_opa: float = 0.1
|
| 267 |
+
# Initial scale of GS
|
| 268 |
+
init_scale: float = 1.0
|
| 269 |
+
# Weight for SSIM loss
|
| 270 |
+
ssim_lambda: float = 0.2
|
| 271 |
+
|
| 272 |
+
# Near plane clipping distance
|
| 273 |
+
near_plane: float = 1e-10
|
| 274 |
+
# Far plane clipping distance
|
| 275 |
+
far_plane: float = 1e10
|
| 276 |
+
|
| 277 |
+
# Strategy for GS densification
|
| 278 |
+
strategy: Union[DefaultStrategy, MCMCStrategy] = field(
|
| 279 |
+
default_factory=DefaultStrategy
|
| 280 |
+
)
|
| 281 |
+
# Use packed mode for rasterization, this leads to less memory usage but slightly slower.
|
| 282 |
+
packed: bool = False
|
| 283 |
+
# Use sparse gradients for optimization. (experimental)
|
| 284 |
+
sparse_grad: bool = False
|
| 285 |
+
# Use visible adam from Taming 3DGS. (experimental)
|
| 286 |
+
visible_adam: bool = False
|
| 287 |
+
# Anti-aliasing in rasterization. Might slightly hurt quantitative metrics.
|
| 288 |
+
antialiased: bool = False
|
| 289 |
+
|
| 290 |
+
# Use random background for training to discourage transparency
|
| 291 |
+
random_bkgd: bool = False
|
| 292 |
+
|
| 293 |
+
# Opacity regularization
|
| 294 |
+
opacity_reg: float = 0.0
|
| 295 |
+
# Scale regularization
|
| 296 |
+
scale_reg: float = 0.0
|
| 297 |
+
|
| 298 |
+
# Enable camera optimization.
|
| 299 |
+
pose_opt: bool = True
|
| 300 |
+
# Learning rate for camera optimization
|
| 301 |
+
pose_opt_lr: float = 1e-5
|
| 302 |
+
# Regularization for camera optimization as weight decay
|
| 303 |
+
pose_opt_reg: float = 1e-6
|
| 304 |
+
# Add noise to camera extrinsics. This is only to test the camera pose optimization.
|
| 305 |
+
pose_noise: float = 0.0
|
| 306 |
+
|
| 307 |
+
# Enable appearance optimization. (experimental)
|
| 308 |
+
app_opt: bool = False
|
| 309 |
+
# Appearance embedding dimension
|
| 310 |
+
app_embed_dim: int = 16
|
| 311 |
+
# Learning rate for appearance optimization
|
| 312 |
+
app_opt_lr: float = 1e-3
|
| 313 |
+
# Regularization for appearance optimization as weight decay
|
| 314 |
+
app_opt_reg: float = 1e-6
|
| 315 |
+
|
| 316 |
+
# Enable bilateral grid. (experimental)
|
| 317 |
+
use_bilateral_grid: bool = False
|
| 318 |
+
# Shape of the bilateral grid (X, Y, W)
|
| 319 |
+
bilateral_grid_shape: Tuple[int, int, int] = (16, 16, 8)
|
| 320 |
+
|
| 321 |
+
# Enable depth loss. (experimental)
|
| 322 |
+
depth_loss: bool = False
|
| 323 |
+
# Weight for depth loss
|
| 324 |
+
depth_lambda: float = 1e-2
|
| 325 |
+
|
| 326 |
+
# Dump information to tensorboard every this steps
|
| 327 |
+
tb_every: int = 100
|
| 328 |
+
# Save training images to tensorboard
|
| 329 |
+
tb_save_image: bool = False
|
| 330 |
+
|
| 331 |
+
lpips_net: Literal["vgg", "alex"] = "vgg"
|
| 332 |
+
|
| 333 |
+
lr_means: float = 1.6e-4
|
| 334 |
+
lr_scales: float = 5e-3
|
| 335 |
+
lr_quats: float = 1e-3
|
| 336 |
+
lr_opacities: float = 5e-2
|
| 337 |
+
lr_sh: float = 2.5e-3
|
| 338 |
+
|
| 339 |
+
def adjust_steps(self, factor: float):
|
| 340 |
+
self.eval_steps = [int(i * factor) for i in self.eval_steps]
|
| 341 |
+
self.save_steps = [int(i * factor) for i in self.save_steps]
|
| 342 |
+
self.ply_steps = [int(i * factor) for i in self.ply_steps]
|
| 343 |
+
self.max_steps = int(self.max_steps * factor)
|
| 344 |
+
self.sh_degree_interval = int(self.sh_degree_interval * factor)
|
| 345 |
+
|
| 346 |
+
strategy = self.strategy
|
| 347 |
+
if isinstance(strategy, DefaultStrategy):
|
| 348 |
+
# strategy.refine_start_iter = int(strategy.refine_start_iter * factor)
|
| 349 |
+
# strategy.refine_stop_iter = int(strategy.refine_stop_iter * factor)
|
| 350 |
+
# strategy.reset_every = int(strategy.reset_every * factor)
|
| 351 |
+
# strategy.refine_every = int(strategy.refine_every * factor)
|
| 352 |
+
|
| 353 |
+
strategy.refine_start_iter = 30000
|
| 354 |
+
strategy.refine_stop_iter = 0
|
| 355 |
+
strategy.reset_every = 30000
|
| 356 |
+
strategy.refine_every = 30000
|
| 357 |
+
|
| 358 |
+
elif isinstance(strategy, MCMCStrategy):
|
| 359 |
+
strategy.refine_start_iter = int(strategy.refine_start_iter * factor)
|
| 360 |
+
strategy.refine_stop_iter = int(strategy.refine_stop_iter * factor)
|
| 361 |
+
strategy.refine_every = int(strategy.refine_every * factor)
|
| 362 |
+
else:
|
| 363 |
+
assert_never(strategy)
|
| 364 |
+
|
| 365 |
+
|
| 366 |
+
def create_splats_with_optimizers(
|
| 367 |
+
gaussians: Gaussians,
|
| 368 |
+
init_num_pts: int = 100_000,
|
| 369 |
+
init_extent: float = 3.0,
|
| 370 |
+
init_opacity: float = 0.1,
|
| 371 |
+
init_scale: float = 1.0,
|
| 372 |
+
sh_degree: int = 3,
|
| 373 |
+
sparse_grad: bool = False,
|
| 374 |
+
visible_adam: bool = False,
|
| 375 |
+
batch_size: int = 1,
|
| 376 |
+
feature_dim: Optional[int] = None,
|
| 377 |
+
device: str = "cuda",
|
| 378 |
+
world_rank: int = 0,
|
| 379 |
+
world_size: int = 1,
|
| 380 |
+
cfg: Config = None,
|
| 381 |
+
) -> Tuple[torch.nn.ParameterDict, Dict[str, torch.optim.Optimizer]]:
|
| 382 |
+
|
| 383 |
+
points = gaussians.means[0].detach().float()
|
| 384 |
+
scales = torch.log(gaussians.scales[0].detach().float())
|
| 385 |
+
quats = gaussians.rotations[0].detach().float()
|
| 386 |
+
opacities = torch.logit(gaussians.opacities[0].detach().float())
|
| 387 |
+
harmonics = gaussians.harmonics[0].detach().float().permute(0, 2, 1).contiguous()
|
| 388 |
+
|
| 389 |
+
N = points.shape[0]
|
| 390 |
+
|
| 391 |
+
scene_scale = 1.0
|
| 392 |
+
masks = opacities.sigmoid() > 0.01
|
| 393 |
+
harmonics = harmonics[masks]
|
| 394 |
+
params = [
|
| 395 |
+
# name, value, lr
|
| 396 |
+
("means", torch.nn.Parameter(points[masks]), cfg.lr_means * scene_scale),
|
| 397 |
+
("scales", torch.nn.Parameter(scales[masks]), cfg.lr_scales),
|
| 398 |
+
("quats", torch.nn.Parameter(quats[masks]), cfg.lr_quats),
|
| 399 |
+
("opacities", torch.nn.Parameter(opacities[masks]), cfg.lr_opacities),
|
| 400 |
+
]
|
| 401 |
+
|
| 402 |
+
params.append(("sh0", torch.nn.Parameter(harmonics[:, :1, :]), cfg.lr_sh))
|
| 403 |
+
params.append(("shN", torch.nn.Parameter(harmonics[:, 1:, :]), cfg.lr_sh/20))
|
| 404 |
+
|
| 405 |
+
splats = torch.nn.ParameterDict({n: v for n, v, _ in params}).to(device)
|
| 406 |
+
# Scale learning rate based on batch size, reference:
|
| 407 |
+
# https://www.cs.princeton.edu/~smalladi/blog/2024/01/22/SDEs-ScalingRules/
|
| 408 |
+
# Note that this would not make the training exactly equivalent, see
|
| 409 |
+
# https://arxiv.org/pdf/2402.18824v1
|
| 410 |
+
BS = batch_size * world_size
|
| 411 |
+
optimizer_class = None
|
| 412 |
+
if sparse_grad:
|
| 413 |
+
optimizer_class = torch.optim.SparseAdam
|
| 414 |
+
elif visible_adam:
|
| 415 |
+
optimizer_class = SelectiveAdam
|
| 416 |
+
else:
|
| 417 |
+
optimizer_class = torch.optim.Adam
|
| 418 |
+
optimizers = {
|
| 419 |
+
name: optimizer_class(
|
| 420 |
+
[{"params": splats[name], "lr": lr * math.sqrt(BS), "name": name}],
|
| 421 |
+
eps=1e-15 / math.sqrt(BS),
|
| 422 |
+
# TODO: check betas logic when BS is larger than 10 betas[0] will be zero.
|
| 423 |
+
betas=(1 - BS * (1 - 0.9), 1 - BS * (1 - 0.999)),
|
| 424 |
+
)
|
| 425 |
+
for name, _, lr in params
|
| 426 |
+
}
|
| 427 |
+
return splats, optimizers
|
| 428 |
+
|
| 429 |
+
|
| 430 |
+
class Runner:
|
| 431 |
+
"""Engine for training and testing."""
|
| 432 |
+
|
| 433 |
+
def __init__(
|
| 434 |
+
self, local_rank: int, world_rank, world_size: int, cfg: Config
|
| 435 |
+
) -> None:
|
| 436 |
+
set_random_seed(42 + local_rank)
|
| 437 |
+
|
| 438 |
+
self.cfg = cfg
|
| 439 |
+
self.world_rank = world_rank
|
| 440 |
+
self.local_rank = local_rank
|
| 441 |
+
self.world_size = world_size
|
| 442 |
+
self.device = f"cuda:{local_rank}"
|
| 443 |
+
|
| 444 |
+
# Where to dump results.
|
| 445 |
+
os.makedirs(cfg.result_dir, exist_ok=True)
|
| 446 |
+
|
| 447 |
+
# Setup output directories.
|
| 448 |
+
self.ckpt_dir = f"{cfg.result_dir}/ckpts"
|
| 449 |
+
os.makedirs(self.ckpt_dir, exist_ok=True)
|
| 450 |
+
self.stats_dir = f"{cfg.result_dir}/stats"
|
| 451 |
+
os.makedirs(self.stats_dir, exist_ok=True)
|
| 452 |
+
self.render_dir = f"{cfg.result_dir}/renders"
|
| 453 |
+
os.makedirs(self.render_dir, exist_ok=True)
|
| 454 |
+
self.ply_dir = f"{cfg.result_dir}/ply"
|
| 455 |
+
os.makedirs(self.ply_dir, exist_ok=True)
|
| 456 |
+
|
| 457 |
+
# Tensorboard
|
| 458 |
+
self.writer = SummaryWriter(log_dir=f"{cfg.result_dir}/tb")
|
| 459 |
+
|
| 460 |
+
# first get the initial 3DGS and camera poses
|
| 461 |
+
model = AnySplat.from_pretrained("lhjiang/anysplat")
|
| 462 |
+
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 463 |
+
model.to(device)
|
| 464 |
+
model.eval()
|
| 465 |
+
for param in model.parameters():
|
| 466 |
+
param.requires_grad = False
|
| 467 |
+
|
| 468 |
+
image_folder = cfg.data_dir
|
| 469 |
+
image_names = sorted([os.path.join(image_folder, f) for f in os.listdir(image_folder) if f.lower().endswith(('.png', '.jpg', '.jpeg'))])
|
| 470 |
+
images = [process_image(img_path) for img_path in image_names]
|
| 471 |
+
ctx_indices = [idx for idx, name in enumerate(image_names) if idx % cfg.test_every != 0]
|
| 472 |
+
tgt_indices = [idx for idx, name in enumerate(image_names) if idx % cfg.test_every == 0]
|
| 473 |
+
|
| 474 |
+
ctx_images = torch.stack([images[i] for i in ctx_indices], dim=0).unsqueeze(0).to(device)
|
| 475 |
+
tgt_images = torch.stack([images[i] for i in tgt_indices], dim=0).unsqueeze(0).to(device)
|
| 476 |
+
ctx_images = (ctx_images+1)*0.5
|
| 477 |
+
tgt_images = (tgt_images+1)*0.5
|
| 478 |
+
b, v, _, h, w = tgt_images.shape
|
| 479 |
+
|
| 480 |
+
# run inference
|
| 481 |
+
encoder_output = model.encoder(
|
| 482 |
+
ctx_images,
|
| 483 |
+
global_step=0,
|
| 484 |
+
visualization_dump={},
|
| 485 |
+
)
|
| 486 |
+
gaussians, pred_context_pose = encoder_output.gaussians, encoder_output.pred_context_pose
|
| 487 |
+
|
| 488 |
+
num_context_view = ctx_images.shape[1]
|
| 489 |
+
vggt_input_image = torch.cat((ctx_images, tgt_images), dim=1).to(torch.bfloat16)
|
| 490 |
+
with torch.no_grad(), torch.cuda.amp.autocast(enabled=False, dtype=torch.bfloat16):
|
| 491 |
+
aggregated_tokens_list, patch_start_idx = model.encoder.aggregator(vggt_input_image, intermediate_layer_idx=model.encoder.cfg.intermediate_layer_idx)
|
| 492 |
+
with torch.cuda.amp.autocast(enabled=False):
|
| 493 |
+
fp32_tokens = [token.float() for token in aggregated_tokens_list]
|
| 494 |
+
pred_all_pose_enc = model.encoder.camera_head(fp32_tokens)[-1]
|
| 495 |
+
pred_all_extrinsic, pred_all_intrinsic = pose_encoding_to_extri_intri(pred_all_pose_enc, vggt_input_image.shape[-2:])
|
| 496 |
+
|
| 497 |
+
extrinsic_padding = torch.tensor([0, 0, 0, 1], device=pred_all_extrinsic.device, dtype=pred_all_extrinsic.dtype).view(1, 1, 1, 4).repeat(b, vggt_input_image.shape[1], 1, 1)
|
| 498 |
+
pred_all_extrinsic = torch.cat([pred_all_extrinsic, extrinsic_padding], dim=2).inverse()
|
| 499 |
+
|
| 500 |
+
pred_all_intrinsic[:, :, 0] = pred_all_intrinsic[:, :, 0] / w
|
| 501 |
+
pred_all_intrinsic[:, :, 1] = pred_all_intrinsic[:, :, 1] / h
|
| 502 |
+
pred_all_context_extrinsic, pred_all_target_extrinsic = pred_all_extrinsic[:, :num_context_view], pred_all_extrinsic[:, num_context_view:]
|
| 503 |
+
pred_all_context_intrinsic, pred_all_target_intrinsic = pred_all_intrinsic[:, :num_context_view], pred_all_intrinsic[:, num_context_view:]
|
| 504 |
+
|
| 505 |
+
scale_factor = pred_context_pose['extrinsic'][:, :, :3, 3].mean() / pred_all_context_extrinsic[:, :, :3, 3].mean()
|
| 506 |
+
pred_all_target_extrinsic[..., :3, 3] = pred_all_target_extrinsic[..., :3, 3] * scale_factor
|
| 507 |
+
pred_all_context_extrinsic[..., :3, 3] = pred_all_context_extrinsic[..., :3, 3] * scale_factor
|
| 508 |
+
print("scale_factor:", scale_factor)
|
| 509 |
+
|
| 510 |
+
# Load data: Training data should contain initial points and colors.
|
| 511 |
+
# self.parser = Parser(
|
| 512 |
+
# data_dir=cfg.data_dir,
|
| 513 |
+
# factor=cfg.data_factor,
|
| 514 |
+
# normalize=cfg.normalize_world_space,
|
| 515 |
+
# test_every=cfg.test_every,
|
| 516 |
+
# )
|
| 517 |
+
self.trainset = Dataset(
|
| 518 |
+
split="train",
|
| 519 |
+
images=ctx_images[0].detach().cpu().numpy(),
|
| 520 |
+
camtoworlds=pred_all_context_extrinsic[0].detach().cpu().numpy(),
|
| 521 |
+
Ks=pred_all_context_intrinsic[0].detach().cpu().numpy(),
|
| 522 |
+
patch_size=cfg.patch_size,
|
| 523 |
+
load_depths=cfg.depth_loss,
|
| 524 |
+
)
|
| 525 |
+
self.valset = Dataset(
|
| 526 |
+
images=tgt_images[0].detach().cpu().numpy(),
|
| 527 |
+
camtoworlds=pred_all_target_extrinsic[0].detach().cpu().numpy(),
|
| 528 |
+
Ks=pred_all_target_intrinsic[0].detach().cpu().numpy(),
|
| 529 |
+
split="val"
|
| 530 |
+
)
|
| 531 |
+
|
| 532 |
+
# Model
|
| 533 |
+
feature_dim = 32 if cfg.app_opt else None
|
| 534 |
+
self.splats, self.optimizers = create_splats_with_optimizers(
|
| 535 |
+
gaussians=gaussians,
|
| 536 |
+
init_num_pts=cfg.init_num_pts,
|
| 537 |
+
init_extent=cfg.init_extent,
|
| 538 |
+
init_opacity=cfg.init_opa,
|
| 539 |
+
init_scale=cfg.init_scale,
|
| 540 |
+
sh_degree=cfg.sh_degree,
|
| 541 |
+
sparse_grad=cfg.sparse_grad,
|
| 542 |
+
visible_adam=cfg.visible_adam,
|
| 543 |
+
batch_size=cfg.batch_size,
|
| 544 |
+
feature_dim=feature_dim,
|
| 545 |
+
device=self.device,
|
| 546 |
+
world_rank=world_rank,
|
| 547 |
+
world_size=world_size,
|
| 548 |
+
cfg=cfg,
|
| 549 |
+
)
|
| 550 |
+
print("Model initialized. Number of GS:", len(self.splats["means"]))
|
| 551 |
+
|
| 552 |
+
# Densification Strategy
|
| 553 |
+
self.cfg.strategy.check_sanity(self.splats, self.optimizers)
|
| 554 |
+
|
| 555 |
+
if isinstance(self.cfg.strategy, DefaultStrategy):
|
| 556 |
+
self.strategy_state = self.cfg.strategy.initialize_state(
|
| 557 |
+
scene_scale=1.0
|
| 558 |
+
)
|
| 559 |
+
elif isinstance(self.cfg.strategy, MCMCStrategy):
|
| 560 |
+
self.strategy_state = self.cfg.strategy.initialize_state()
|
| 561 |
+
else:
|
| 562 |
+
assert_never(self.cfg.strategy)
|
| 563 |
+
|
| 564 |
+
# Compression Strategy
|
| 565 |
+
self.compression_method = None
|
| 566 |
+
if cfg.compression is not None:
|
| 567 |
+
if cfg.compression == "png":
|
| 568 |
+
self.compression_method = PngCompression()
|
| 569 |
+
else:
|
| 570 |
+
raise ValueError(f"Unknown compression strategy: {cfg.compression}")
|
| 571 |
+
|
| 572 |
+
self.pose_optimizers = []
|
| 573 |
+
if cfg.pose_opt:
|
| 574 |
+
self.pose_adjust = CameraOptModule(len(self.trainset)).to(self.device)
|
| 575 |
+
self.pose_adjust.zero_init()
|
| 576 |
+
self.pose_optimizers = [
|
| 577 |
+
torch.optim.Adam(
|
| 578 |
+
self.pose_adjust.parameters(),
|
| 579 |
+
lr=cfg.pose_opt_lr * math.sqrt(cfg.batch_size),
|
| 580 |
+
weight_decay=cfg.pose_opt_reg,
|
| 581 |
+
)
|
| 582 |
+
]
|
| 583 |
+
if world_size > 1:
|
| 584 |
+
self.pose_adjust = DDP(self.pose_adjust)
|
| 585 |
+
|
| 586 |
+
if cfg.pose_noise > 0.0:
|
| 587 |
+
self.pose_perturb = CameraOptModule(len(self.trainset)).to(self.device)
|
| 588 |
+
self.pose_perturb.random_init(cfg.pose_noise)
|
| 589 |
+
if world_size > 1:
|
| 590 |
+
self.pose_perturb = DDP(self.pose_perturb)
|
| 591 |
+
|
| 592 |
+
self.app_optimizers = []
|
| 593 |
+
if cfg.app_opt:
|
| 594 |
+
assert feature_dim is not None
|
| 595 |
+
self.app_module = AppearanceOptModule(
|
| 596 |
+
len(self.trainset), feature_dim, cfg.app_embed_dim, cfg.sh_degree
|
| 597 |
+
).to(self.device)
|
| 598 |
+
# initialize the last layer to be zero so that the initial output is zero.
|
| 599 |
+
torch.nn.init.zeros_(self.app_module.color_head[-1].weight)
|
| 600 |
+
torch.nn.init.zeros_(self.app_module.color_head[-1].bias)
|
| 601 |
+
self.app_optimizers = [
|
| 602 |
+
torch.optim.Adam(
|
| 603 |
+
self.app_module.embeds.parameters(),
|
| 604 |
+
lr=cfg.app_opt_lr * math.sqrt(cfg.batch_size) * 10.0,
|
| 605 |
+
weight_decay=cfg.app_opt_reg,
|
| 606 |
+
),
|
| 607 |
+
torch.optim.Adam(
|
| 608 |
+
self.app_module.color_head.parameters(),
|
| 609 |
+
lr=cfg.app_opt_lr * math.sqrt(cfg.batch_size),
|
| 610 |
+
),
|
| 611 |
+
]
|
| 612 |
+
if world_size > 1:
|
| 613 |
+
self.app_module = DDP(self.app_module)
|
| 614 |
+
|
| 615 |
+
self.bil_grid_optimizers = []
|
| 616 |
+
if cfg.use_bilateral_grid:
|
| 617 |
+
self.bil_grids = BilateralGrid(
|
| 618 |
+
len(self.trainset),
|
| 619 |
+
grid_X=cfg.bilateral_grid_shape[0],
|
| 620 |
+
grid_Y=cfg.bilateral_grid_shape[1],
|
| 621 |
+
grid_W=cfg.bilateral_grid_shape[2],
|
| 622 |
+
).to(self.device)
|
| 623 |
+
self.bil_grid_optimizers = [
|
| 624 |
+
torch.optim.Adam(
|
| 625 |
+
self.bil_grids.parameters(),
|
| 626 |
+
lr=2e-3 * math.sqrt(cfg.batch_size),
|
| 627 |
+
eps=1e-15,
|
| 628 |
+
),
|
| 629 |
+
]
|
| 630 |
+
|
| 631 |
+
# Losses & Metrics.
|
| 632 |
+
self.ssim = StructuralSimilarityIndexMeasure(data_range=1.0).to(self.device)
|
| 633 |
+
self.psnr = PeakSignalNoiseRatio(data_range=1.0).to(self.device)
|
| 634 |
+
|
| 635 |
+
if cfg.lpips_net == "alex":
|
| 636 |
+
self.lpips = LearnedPerceptualImagePatchSimilarity(
|
| 637 |
+
net_type="alex", normalize=True
|
| 638 |
+
).to(self.device)
|
| 639 |
+
elif cfg.lpips_net == "vgg":
|
| 640 |
+
# The 3DGS official repo uses lpips vgg, which is equivalent with the following:
|
| 641 |
+
self.lpips = LearnedPerceptualImagePatchSimilarity(
|
| 642 |
+
net_type="vgg", normalize=False
|
| 643 |
+
).to(self.device)
|
| 644 |
+
else:
|
| 645 |
+
raise ValueError(f"Unknown LPIPS network: {cfg.lpips_net}")
|
| 646 |
+
|
| 647 |
+
# Viewer
|
| 648 |
+
if not self.cfg.disable_viewer:
|
| 649 |
+
self.server = viser.ViserServer(port=cfg.port, verbose=False)
|
| 650 |
+
self.viewer = GsplatViewer(
|
| 651 |
+
server=self.server,
|
| 652 |
+
render_fn=self._viewer_render_fn,
|
| 653 |
+
output_dir=Path(cfg.result_dir),
|
| 654 |
+
mode="training",
|
| 655 |
+
)
|
| 656 |
+
|
| 657 |
+
def rasterize_splats(
|
| 658 |
+
self,
|
| 659 |
+
camtoworlds: Tensor,
|
| 660 |
+
Ks: Tensor,
|
| 661 |
+
width: int,
|
| 662 |
+
height: int,
|
| 663 |
+
masks: Optional[Tensor] = None,
|
| 664 |
+
rasterize_mode: Optional[Literal["classic", "antialiased"]] = None,
|
| 665 |
+
camera_model: Optional[Literal["pinhole", "ortho", "fisheye"]] = None,
|
| 666 |
+
**kwargs,
|
| 667 |
+
) -> Tuple[Tensor, Tensor, Dict]:
|
| 668 |
+
means = self.splats["means"] # [N, 3]
|
| 669 |
+
# quats = F.normalize(self.splats["quats"], dim=-1) # [N, 4]
|
| 670 |
+
# rasterization does normalization internally
|
| 671 |
+
quats = self.splats["quats"] # [N, 4]
|
| 672 |
+
scales = torch.exp(self.splats["scales"]) # [N, 3]
|
| 673 |
+
opacities = torch.sigmoid(self.splats["opacities"]) # [N,]
|
| 674 |
+
|
| 675 |
+
image_ids = kwargs.pop("image_ids", None)
|
| 676 |
+
if self.cfg.app_opt:
|
| 677 |
+
colors = self.app_module(
|
| 678 |
+
features=self.splats["features"],
|
| 679 |
+
embed_ids=image_ids,
|
| 680 |
+
dirs=means[None, :, :] - camtoworlds[:, None, :3, 3],
|
| 681 |
+
sh_degree=kwargs.pop("sh_degree", self.cfg.sh_degree),
|
| 682 |
+
)
|
| 683 |
+
colors = colors + self.splats["colors"]
|
| 684 |
+
colors = torch.sigmoid(colors)
|
| 685 |
+
else:
|
| 686 |
+
colors = torch.cat([self.splats["sh0"], self.splats["shN"]], 1) # [N, K, 3]
|
| 687 |
+
|
| 688 |
+
if rasterize_mode is None:
|
| 689 |
+
rasterize_mode = "antialiased" if self.cfg.antialiased else "classic"
|
| 690 |
+
if camera_model is None:
|
| 691 |
+
camera_model = self.cfg.camera_model
|
| 692 |
+
|
| 693 |
+
# covariance = build_covariance(scales[None], quats[None]).squeeze(0)
|
| 694 |
+
render_colors, render_alphas, info = rasterization(
|
| 695 |
+
means=means,
|
| 696 |
+
quats=quats,
|
| 697 |
+
scales=scales,
|
| 698 |
+
opacities=opacities,
|
| 699 |
+
colors=colors,
|
| 700 |
+
# covars=covariance,
|
| 701 |
+
viewmats=torch.linalg.inv(camtoworlds), # [C, 4, 4]
|
| 702 |
+
Ks=Ks, # [C, 3, 3]
|
| 703 |
+
width=width,
|
| 704 |
+
height=height,
|
| 705 |
+
packed=self.cfg.packed,
|
| 706 |
+
absgrad=(
|
| 707 |
+
self.cfg.strategy.absgrad
|
| 708 |
+
if isinstance(self.cfg.strategy, DefaultStrategy)
|
| 709 |
+
else False
|
| 710 |
+
),
|
| 711 |
+
sparse_grad=self.cfg.sparse_grad,
|
| 712 |
+
rasterize_mode=rasterize_mode,
|
| 713 |
+
distributed=self.world_size > 1,
|
| 714 |
+
camera_model=self.cfg.camera_model,
|
| 715 |
+
radius_clip=0.1,
|
| 716 |
+
backgrounds=torch.tensor([0.0, 0.0, 0.0]).cuda().unsqueeze(0).repeat(1, 1),
|
| 717 |
+
**kwargs,
|
| 718 |
+
)
|
| 719 |
+
if masks is not None:
|
| 720 |
+
render_colors[~masks] = 0
|
| 721 |
+
return render_colors, render_alphas, info
|
| 722 |
+
|
| 723 |
+
def train(self):
|
| 724 |
+
cfg = self.cfg
|
| 725 |
+
device = self.device
|
| 726 |
+
world_rank = self.world_rank
|
| 727 |
+
world_size = self.world_size
|
| 728 |
+
|
| 729 |
+
# Dump cfg.
|
| 730 |
+
if world_rank == 0:
|
| 731 |
+
with open(f"{cfg.result_dir}/cfg.yml", "w") as f:
|
| 732 |
+
yaml.dump(vars(cfg), f)
|
| 733 |
+
|
| 734 |
+
max_steps = cfg.max_steps
|
| 735 |
+
init_step = 0
|
| 736 |
+
|
| 737 |
+
schedulers = [
|
| 738 |
+
# means has a learning rate schedule, that end at 0.01 of the initial value
|
| 739 |
+
torch.optim.lr_scheduler.ExponentialLR(
|
| 740 |
+
self.optimizers["means"], gamma=0.01 ** (1.0 / max_steps)
|
| 741 |
+
),
|
| 742 |
+
]
|
| 743 |
+
if cfg.pose_opt:
|
| 744 |
+
# pose optimization has a learning rate schedule
|
| 745 |
+
schedulers.append(
|
| 746 |
+
torch.optim.lr_scheduler.ExponentialLR(
|
| 747 |
+
self.pose_optimizers[0], gamma=0.01 ** (1.0 / max_steps)
|
| 748 |
+
)
|
| 749 |
+
)
|
| 750 |
+
if cfg.use_bilateral_grid:
|
| 751 |
+
# bilateral grid has a learning rate schedule. Linear warmup for 1000 steps.
|
| 752 |
+
schedulers.append(
|
| 753 |
+
torch.optim.lr_scheduler.ChainedScheduler(
|
| 754 |
+
[
|
| 755 |
+
torch.optim.lr_scheduler.LinearLR(
|
| 756 |
+
self.bil_grid_optimizers[0],
|
| 757 |
+
start_factor=0.01,
|
| 758 |
+
total_iters=1000,
|
| 759 |
+
),
|
| 760 |
+
torch.optim.lr_scheduler.ExponentialLR(
|
| 761 |
+
self.bil_grid_optimizers[0], gamma=0.01 ** (1.0 / max_steps)
|
| 762 |
+
),
|
| 763 |
+
]
|
| 764 |
+
)
|
| 765 |
+
)
|
| 766 |
+
|
| 767 |
+
trainloader = torch.utils.data.DataLoader(
|
| 768 |
+
self.trainset,
|
| 769 |
+
batch_size=cfg.batch_size,
|
| 770 |
+
shuffle=True,
|
| 771 |
+
num_workers=4,
|
| 772 |
+
persistent_workers=True,
|
| 773 |
+
pin_memory=True,
|
| 774 |
+
)
|
| 775 |
+
trainloader_iter = iter(trainloader)
|
| 776 |
+
|
| 777 |
+
# Training loop.
|
| 778 |
+
global_tic = time.time()
|
| 779 |
+
pbar = tqdm.tqdm(range(init_step, max_steps))
|
| 780 |
+
for step in pbar:
|
| 781 |
+
if not cfg.disable_viewer:
|
| 782 |
+
while self.viewer.state == "paused":
|
| 783 |
+
time.sleep(0.01)
|
| 784 |
+
self.viewer.lock.acquire()
|
| 785 |
+
tic = time.time()
|
| 786 |
+
|
| 787 |
+
try:
|
| 788 |
+
data = next(trainloader_iter)
|
| 789 |
+
except StopIteration:
|
| 790 |
+
trainloader_iter = iter(trainloader)
|
| 791 |
+
data = next(trainloader_iter)
|
| 792 |
+
|
| 793 |
+
camtoworlds = camtoworlds_gt = data["camtoworld"].to(device) # [1, 4, 4]
|
| 794 |
+
Ks = data["K"].to(device) # [1, 3, 3]
|
| 795 |
+
pixels = data["image"].to(device) / 255.0 # [1, H, W, 3]
|
| 796 |
+
num_train_rays_per_step = (
|
| 797 |
+
pixels.shape[0] * pixels.shape[1] * pixels.shape[2]
|
| 798 |
+
)
|
| 799 |
+
image_ids = data["image_id"].to(device)
|
| 800 |
+
masks = data["mask"].to(device) if "mask" in data else None # [1, H, W]
|
| 801 |
+
if cfg.depth_loss:
|
| 802 |
+
points = data["points"].to(device) # [1, M, 2]
|
| 803 |
+
depths_gt = data["depths"].to(device) # [1, M]
|
| 804 |
+
|
| 805 |
+
height, width = pixels.shape[1:3]
|
| 806 |
+
|
| 807 |
+
if cfg.pose_noise:
|
| 808 |
+
camtoworlds = self.pose_perturb(camtoworlds, image_ids)
|
| 809 |
+
|
| 810 |
+
if cfg.pose_opt:
|
| 811 |
+
camtoworlds = self.pose_adjust(camtoworlds, image_ids)
|
| 812 |
+
|
| 813 |
+
# sh schedule
|
| 814 |
+
# sh_degree_to_use = min(step // cfg.sh_degree_interval, cfg.sh_degree)
|
| 815 |
+
sh_degree_to_use = cfg.sh_degree
|
| 816 |
+
|
| 817 |
+
# forward
|
| 818 |
+
renders, alphas, info = self.rasterize_splats(
|
| 819 |
+
camtoworlds=camtoworlds,
|
| 820 |
+
Ks=Ks,
|
| 821 |
+
width=width,
|
| 822 |
+
height=height,
|
| 823 |
+
sh_degree=sh_degree_to_use,
|
| 824 |
+
near_plane=cfg.near_plane,
|
| 825 |
+
far_plane=cfg.far_plane,
|
| 826 |
+
image_ids=image_ids,
|
| 827 |
+
render_mode="RGB+ED" if cfg.depth_loss else "RGB",
|
| 828 |
+
masks=masks,
|
| 829 |
+
)
|
| 830 |
+
if renders.shape[-1] == 4:
|
| 831 |
+
colors, depths = renders[..., 0:3], renders[..., 3:4]
|
| 832 |
+
else:
|
| 833 |
+
colors, depths = renders, None
|
| 834 |
+
|
| 835 |
+
if cfg.use_bilateral_grid:
|
| 836 |
+
grid_y, grid_x = torch.meshgrid(
|
| 837 |
+
(torch.arange(height, device=self.device) + 0.5) / height,
|
| 838 |
+
(torch.arange(width, device=self.device) + 0.5) / width,
|
| 839 |
+
indexing="ij",
|
| 840 |
+
)
|
| 841 |
+
grid_xy = torch.stack([grid_x, grid_y], dim=-1).unsqueeze(0)
|
| 842 |
+
colors = slice(self.bil_grids, grid_xy, colors, image_ids)["rgb"]
|
| 843 |
+
|
| 844 |
+
if cfg.random_bkgd:
|
| 845 |
+
bkgd = torch.rand(1, 3, device=device)
|
| 846 |
+
colors = colors + bkgd * (1.0 - alphas)
|
| 847 |
+
|
| 848 |
+
self.cfg.strategy.step_pre_backward(
|
| 849 |
+
params=self.splats,
|
| 850 |
+
optimizers=self.optimizers,
|
| 851 |
+
state=self.strategy_state,
|
| 852 |
+
step=step,
|
| 853 |
+
info=info,
|
| 854 |
+
)
|
| 855 |
+
|
| 856 |
+
# loss
|
| 857 |
+
l1loss = F.l1_loss(colors, pixels)
|
| 858 |
+
ssimloss = 1.0 - fused_ssim(
|
| 859 |
+
colors.permute(0, 3, 1, 2), pixels.permute(0, 3, 1, 2), padding="valid"
|
| 860 |
+
)
|
| 861 |
+
loss = l1loss * (1.0 - cfg.ssim_lambda) + ssimloss * cfg.ssim_lambda
|
| 862 |
+
if cfg.depth_loss:
|
| 863 |
+
# query depths from depth map
|
| 864 |
+
points = torch.stack(
|
| 865 |
+
[
|
| 866 |
+
points[:, :, 0] / (width - 1) * 2 - 1,
|
| 867 |
+
points[:, :, 1] / (height - 1) * 2 - 1,
|
| 868 |
+
],
|
| 869 |
+
dim=-1,
|
| 870 |
+
) # normalize to [-1, 1]
|
| 871 |
+
grid = points.unsqueeze(2) # [1, M, 1, 2]
|
| 872 |
+
depths = F.grid_sample(
|
| 873 |
+
depths.permute(0, 3, 1, 2), grid, align_corners=True
|
| 874 |
+
) # [1, 1, M, 1]
|
| 875 |
+
depths = depths.squeeze(3).squeeze(1) # [1, M]
|
| 876 |
+
# calculate loss in disparity space
|
| 877 |
+
disp = torch.where(depths > 0.0, 1.0 / depths, torch.zeros_like(depths))
|
| 878 |
+
disp_gt = 1.0 / depths_gt # [1, M]
|
| 879 |
+
depthloss = F.l1_loss(disp, disp_gt) * self.scene_scale
|
| 880 |
+
loss += depthloss * cfg.depth_lambda
|
| 881 |
+
if cfg.use_bilateral_grid:
|
| 882 |
+
tvloss = 10 * total_variation_loss(self.bil_grids.grids)
|
| 883 |
+
loss += tvloss
|
| 884 |
+
|
| 885 |
+
# regularizations
|
| 886 |
+
if cfg.opacity_reg > 0.0:
|
| 887 |
+
loss = (
|
| 888 |
+
loss
|
| 889 |
+
+ cfg.opacity_reg
|
| 890 |
+
* torch.abs(torch.sigmoid(self.splats["opacities"])).mean()
|
| 891 |
+
)
|
| 892 |
+
if cfg.scale_reg > 0.0:
|
| 893 |
+
loss = (
|
| 894 |
+
loss
|
| 895 |
+
+ cfg.scale_reg * torch.abs(torch.exp(self.splats["scales"])).mean()
|
| 896 |
+
)
|
| 897 |
+
|
| 898 |
+
loss.backward()
|
| 899 |
+
|
| 900 |
+
desc = f"loss={loss.item():.3f}| " f"sh degree={sh_degree_to_use}| "
|
| 901 |
+
if cfg.depth_loss:
|
| 902 |
+
desc += f"depth loss={depthloss.item():.6f}| "
|
| 903 |
+
if cfg.pose_opt and cfg.pose_noise:
|
| 904 |
+
# monitor the pose error if we inject noise
|
| 905 |
+
pose_err = F.l1_loss(camtoworlds_gt, camtoworlds)
|
| 906 |
+
desc += f"pose err={pose_err.item():.6f}| "
|
| 907 |
+
pbar.set_description(desc)
|
| 908 |
+
|
| 909 |
+
# write images (gt and render)
|
| 910 |
+
# if world_rank == 0 and step % 800 == 0:
|
| 911 |
+
# canvas = torch.cat([pixels, colors], dim=2).detach().cpu().numpy()
|
| 912 |
+
# canvas = canvas.reshape(-1, *canvas.shape[2:])
|
| 913 |
+
# imageio.imwrite(
|
| 914 |
+
# f"{self.render_dir}/train_rank{self.world_rank}.png",
|
| 915 |
+
# (canvas * 255).astype(np.uint8),
|
| 916 |
+
# )
|
| 917 |
+
|
| 918 |
+
if world_rank == 0 and cfg.tb_every > 0 and step % cfg.tb_every == 0:
|
| 919 |
+
mem = torch.cuda.max_memory_allocated() / 1024**3
|
| 920 |
+
self.writer.add_scalar("train/loss", loss.item(), step)
|
| 921 |
+
self.writer.add_scalar("train/l1loss", l1loss.item(), step)
|
| 922 |
+
self.writer.add_scalar("train/ssimloss", ssimloss.item(), step)
|
| 923 |
+
self.writer.add_scalar("train/num_GS", len(self.splats["means"]), step)
|
| 924 |
+
self.writer.add_scalar("train/mem", mem, step)
|
| 925 |
+
if cfg.depth_loss:
|
| 926 |
+
self.writer.add_scalar("train/depthloss", depthloss.item(), step)
|
| 927 |
+
if cfg.use_bilateral_grid:
|
| 928 |
+
self.writer.add_scalar("train/tvloss", tvloss.item(), step)
|
| 929 |
+
if cfg.tb_save_image:
|
| 930 |
+
canvas = torch.cat([pixels, colors], dim=2).detach().cpu().numpy()
|
| 931 |
+
canvas = canvas.reshape(-1, *canvas.shape[2:])
|
| 932 |
+
self.writer.add_image("train/render", canvas, step)
|
| 933 |
+
self.writer.flush()
|
| 934 |
+
|
| 935 |
+
# save checkpoint before updating the model
|
| 936 |
+
if step in [i - 1 for i in cfg.save_steps] or step == max_steps - 1:
|
| 937 |
+
mem = torch.cuda.max_memory_allocated() / 1024**3
|
| 938 |
+
stats = {
|
| 939 |
+
"mem": mem,
|
| 940 |
+
"ellipse_time": time.time() - global_tic,
|
| 941 |
+
"num_GS": len(self.splats["means"]),
|
| 942 |
+
}
|
| 943 |
+
print("Step: ", step, stats)
|
| 944 |
+
with open(
|
| 945 |
+
f"{self.stats_dir}/train_step{step:04d}_rank{self.world_rank}.json",
|
| 946 |
+
"w",
|
| 947 |
+
) as f:
|
| 948 |
+
json.dump(stats, f)
|
| 949 |
+
data = {"step": step, "splats": self.splats.state_dict()}
|
| 950 |
+
if cfg.pose_opt:
|
| 951 |
+
if world_size > 1:
|
| 952 |
+
data["pose_adjust"] = self.pose_adjust.module.state_dict()
|
| 953 |
+
else:
|
| 954 |
+
data["pose_adjust"] = self.pose_adjust.state_dict()
|
| 955 |
+
if cfg.app_opt:
|
| 956 |
+
if world_size > 1:
|
| 957 |
+
data["app_module"] = self.app_module.module.state_dict()
|
| 958 |
+
else:
|
| 959 |
+
data["app_module"] = self.app_module.state_dict()
|
| 960 |
+
torch.save(
|
| 961 |
+
data, f"{self.ckpt_dir}/ckpt_{step}_rank{self.world_rank}.pt"
|
| 962 |
+
)
|
| 963 |
+
if (
|
| 964 |
+
step in [i - 1 for i in cfg.ply_steps] or step == max_steps - 1
|
| 965 |
+
) and cfg.save_ply:
|
| 966 |
+
|
| 967 |
+
if self.cfg.app_opt:
|
| 968 |
+
# eval at origin to bake the appeareance into the colors
|
| 969 |
+
rgb = self.app_module(
|
| 970 |
+
features=self.splats["features"],
|
| 971 |
+
embed_ids=None,
|
| 972 |
+
dirs=torch.zeros_like(self.splats["means"][None, :, :]),
|
| 973 |
+
sh_degree=sh_degree_to_use,
|
| 974 |
+
)
|
| 975 |
+
rgb = rgb + self.splats["colors"]
|
| 976 |
+
rgb = torch.sigmoid(rgb).squeeze(0).unsqueeze(1)
|
| 977 |
+
sh0 = rgb_to_sh(rgb)
|
| 978 |
+
shN = torch.empty([sh0.shape[0], 0, 3], device=sh0.device)
|
| 979 |
+
else:
|
| 980 |
+
sh0 = self.splats["sh0"]
|
| 981 |
+
shN = self.splats["shN"]
|
| 982 |
+
# shN = torch.empty([sh0.shape[0], 0, 3], device=sh0.device)
|
| 983 |
+
|
| 984 |
+
means = self.splats["means"]
|
| 985 |
+
scales = self.splats["scales"]
|
| 986 |
+
quats = self.splats["quats"]
|
| 987 |
+
opacities = self.splats["opacities"]
|
| 988 |
+
|
| 989 |
+
# export_splats(
|
| 990 |
+
# means=means,
|
| 991 |
+
# scales=scales,
|
| 992 |
+
# quats=quats,
|
| 993 |
+
# opacities=opacities,
|
| 994 |
+
# sh0=sh0,
|
| 995 |
+
# shN=shN,
|
| 996 |
+
# format="ply",
|
| 997 |
+
# save_to=f"{self.ply_dir}/point_cloud_{step}.ply",
|
| 998 |
+
# )
|
| 999 |
+
export_ply(
|
| 1000 |
+
means=means,
|
| 1001 |
+
scales=scales,
|
| 1002 |
+
rotations=quats,
|
| 1003 |
+
harmonics=torch.cat([sh0, shN], dim=1).permute(0, 2, 1),
|
| 1004 |
+
opacities=opacities.sigmoid(),
|
| 1005 |
+
path=Path(f"{self.ply_dir}/point_cloud_{step}.ply"),
|
| 1006 |
+
)
|
| 1007 |
+
|
| 1008 |
+
# Turn Gradients into Sparse Tensor before running optimizer
|
| 1009 |
+
if cfg.sparse_grad:
|
| 1010 |
+
assert cfg.packed, "Sparse gradients only work with packed mode."
|
| 1011 |
+
gaussian_ids = info["gaussian_ids"]
|
| 1012 |
+
for k in self.splats.keys():
|
| 1013 |
+
grad = self.splats[k].grad
|
| 1014 |
+
if grad is None or grad.is_sparse:
|
| 1015 |
+
continue
|
| 1016 |
+
self.splats[k].grad = torch.sparse_coo_tensor(
|
| 1017 |
+
indices=gaussian_ids[None], # [1, nnz]
|
| 1018 |
+
values=grad[gaussian_ids], # [nnz, ...]
|
| 1019 |
+
size=self.splats[k].size(), # [N, ...]
|
| 1020 |
+
is_coalesced=len(Ks) == 1,
|
| 1021 |
+
)
|
| 1022 |
+
|
| 1023 |
+
if cfg.visible_adam:
|
| 1024 |
+
gaussian_cnt = self.splats.means.shape[0]
|
| 1025 |
+
if cfg.packed:
|
| 1026 |
+
visibility_mask = torch.zeros_like(
|
| 1027 |
+
self.splats["opacities"], dtype=bool
|
| 1028 |
+
)
|
| 1029 |
+
visibility_mask.scatter_(0, info["gaussian_ids"], 1)
|
| 1030 |
+
else:
|
| 1031 |
+
visibility_mask = (info["radii"] > 0).all(-1).any(0)
|
| 1032 |
+
|
| 1033 |
+
# optimize
|
| 1034 |
+
for optimizer in self.optimizers.values():
|
| 1035 |
+
if cfg.visible_adam:
|
| 1036 |
+
optimizer.step(visibility_mask)
|
| 1037 |
+
else:
|
| 1038 |
+
optimizer.step()
|
| 1039 |
+
optimizer.zero_grad(set_to_none=True)
|
| 1040 |
+
for optimizer in self.pose_optimizers:
|
| 1041 |
+
optimizer.step()
|
| 1042 |
+
optimizer.zero_grad(set_to_none=True)
|
| 1043 |
+
for optimizer in self.app_optimizers:
|
| 1044 |
+
optimizer.step()
|
| 1045 |
+
optimizer.zero_grad(set_to_none=True)
|
| 1046 |
+
for optimizer in self.bil_grid_optimizers:
|
| 1047 |
+
optimizer.step()
|
| 1048 |
+
optimizer.zero_grad(set_to_none=True)
|
| 1049 |
+
for scheduler in schedulers:
|
| 1050 |
+
scheduler.step()
|
| 1051 |
+
|
| 1052 |
+
# Run post-backward steps after backward and optimizer
|
| 1053 |
+
if isinstance(self.cfg.strategy, DefaultStrategy):
|
| 1054 |
+
self.cfg.strategy.step_post_backward(
|
| 1055 |
+
params=self.splats,
|
| 1056 |
+
optimizers=self.optimizers,
|
| 1057 |
+
state=self.strategy_state,
|
| 1058 |
+
step=step,
|
| 1059 |
+
info=info,
|
| 1060 |
+
packed=cfg.packed,
|
| 1061 |
+
)
|
| 1062 |
+
elif isinstance(self.cfg.strategy, MCMCStrategy):
|
| 1063 |
+
self.cfg.strategy.step_post_backward(
|
| 1064 |
+
params=self.splats,
|
| 1065 |
+
optimizers=self.optimizers,
|
| 1066 |
+
state=self.strategy_state,
|
| 1067 |
+
step=step,
|
| 1068 |
+
info=info,
|
| 1069 |
+
lr=schedulers[0].get_last_lr()[0],
|
| 1070 |
+
)
|
| 1071 |
+
else:
|
| 1072 |
+
assert_never(self.cfg.strategy)
|
| 1073 |
+
|
| 1074 |
+
# eval the full set
|
| 1075 |
+
if step in [i - 1 for i in cfg.eval_steps]:
|
| 1076 |
+
self.eval(step)
|
| 1077 |
+
# self.render_traj(step)
|
| 1078 |
+
|
| 1079 |
+
# run compression
|
| 1080 |
+
if cfg.compression is not None and step in [i - 1 for i in cfg.eval_steps]:
|
| 1081 |
+
self.run_compression(step=step)
|
| 1082 |
+
|
| 1083 |
+
if not cfg.disable_viewer:
|
| 1084 |
+
self.viewer.lock.release()
|
| 1085 |
+
num_train_steps_per_sec = 1.0 / (time.time() - tic)
|
| 1086 |
+
num_train_rays_per_sec = (
|
| 1087 |
+
num_train_rays_per_step * num_train_steps_per_sec
|
| 1088 |
+
)
|
| 1089 |
+
# Update the viewer state.
|
| 1090 |
+
self.viewer.render_tab_state.num_train_rays_per_sec = (
|
| 1091 |
+
num_train_rays_per_sec
|
| 1092 |
+
)
|
| 1093 |
+
# Update the scene.
|
| 1094 |
+
self.viewer.update(step, num_train_rays_per_step)
|
| 1095 |
+
|
| 1096 |
+
@torch.no_grad()
|
| 1097 |
+
def eval(self, step: int, stage: str = "val"):
|
| 1098 |
+
"""Entry for evaluation."""
|
| 1099 |
+
print("Running evaluation...")
|
| 1100 |
+
cfg = self.cfg
|
| 1101 |
+
device = self.device
|
| 1102 |
+
world_rank = self.world_rank
|
| 1103 |
+
world_size = self.world_size
|
| 1104 |
+
|
| 1105 |
+
valloader = torch.utils.data.DataLoader(
|
| 1106 |
+
self.valset, batch_size=1, shuffle=False, num_workers=1
|
| 1107 |
+
)
|
| 1108 |
+
ellipse_time = 0
|
| 1109 |
+
metrics = defaultdict(list)
|
| 1110 |
+
for i, data in enumerate(valloader):
|
| 1111 |
+
camtoworlds = data["camtoworld"].to(device)
|
| 1112 |
+
Ks = data["K"].to(device)
|
| 1113 |
+
pixels = data["image"].to(device) / 255.0
|
| 1114 |
+
masks = data["mask"].to(device) if "mask" in data else None
|
| 1115 |
+
height, width = pixels.shape[1:3]
|
| 1116 |
+
|
| 1117 |
+
torch.cuda.synchronize()
|
| 1118 |
+
tic = time.time()
|
| 1119 |
+
render_colors, _, _ = self.rasterize_splats(
|
| 1120 |
+
camtoworlds=camtoworlds,
|
| 1121 |
+
Ks=Ks,
|
| 1122 |
+
width=width,
|
| 1123 |
+
height=height,
|
| 1124 |
+
sh_degree=cfg.sh_degree,
|
| 1125 |
+
near_plane=cfg.near_plane,
|
| 1126 |
+
far_plane=cfg.far_plane,
|
| 1127 |
+
# radius_clip=0.1,
|
| 1128 |
+
render_mode="RGB+ED",
|
| 1129 |
+
masks=masks,
|
| 1130 |
+
) # [1, H, W, 3]
|
| 1131 |
+
torch.cuda.synchronize()
|
| 1132 |
+
ellipse_time += time.time() - tic
|
| 1133 |
+
|
| 1134 |
+
colors = render_colors[..., :3]
|
| 1135 |
+
depths = render_colors[..., 3]
|
| 1136 |
+
|
| 1137 |
+
colors = torch.clamp(colors, 0.0, 1.0)
|
| 1138 |
+
canvas_list = [pixels, colors]
|
| 1139 |
+
|
| 1140 |
+
if world_rank == 0:
|
| 1141 |
+
# write images
|
| 1142 |
+
canvas = torch.cat(canvas_list, dim=2).squeeze(0).cpu().numpy()
|
| 1143 |
+
canvas = (canvas * 255).astype(np.uint8)
|
| 1144 |
+
imageio.imwrite(
|
| 1145 |
+
f"{self.render_dir}/{stage}_step{step}_{i:04d}.png",
|
| 1146 |
+
canvas,
|
| 1147 |
+
)
|
| 1148 |
+
torchvision.utils.save_image(pixels.permute(0, 3, 1, 2), f"{self.render_dir}/gt_rgb_{stage}_step{step}_{i:04d}.png")
|
| 1149 |
+
torchvision.utils.save_image(colors.permute(0, 3, 1, 2), f"{self.render_dir}/render_rgb_{stage}_step{step}_{i:04d}.png")
|
| 1150 |
+
# save depth & normal map
|
| 1151 |
+
|
| 1152 |
+
|
| 1153 |
+
pixels_p = pixels.permute(0, 3, 1, 2) # [1, 3, H, W]
|
| 1154 |
+
colors_p = colors.permute(0, 3, 1, 2) # [1, 3, H, W]
|
| 1155 |
+
|
| 1156 |
+
metrics["psnr"].append(self.psnr(colors_p, pixels_p))
|
| 1157 |
+
metrics["ssim"].append(self.ssim(colors_p, pixels_p))
|
| 1158 |
+
metrics["lpips"].append(self.lpips(colors_p, pixels_p))
|
| 1159 |
+
if cfg.use_bilateral_grid:
|
| 1160 |
+
cc_colors = color_correct(colors, pixels)
|
| 1161 |
+
cc_colors_p = cc_colors.permute(0, 3, 1, 2) # [1, 3, H, W]
|
| 1162 |
+
metrics["cc_psnr"].append(self.psnr(cc_colors_p, pixels_p))
|
| 1163 |
+
|
| 1164 |
+
if world_rank == 0:
|
| 1165 |
+
ellipse_time /= len(valloader)
|
| 1166 |
+
|
| 1167 |
+
stats = {k: torch.stack(v).mean().item() for k, v in metrics.items()}
|
| 1168 |
+
stats.update(
|
| 1169 |
+
{
|
| 1170 |
+
"ellipse_time": ellipse_time,
|
| 1171 |
+
"num_GS": len(self.splats["means"]),
|
| 1172 |
+
}
|
| 1173 |
+
)
|
| 1174 |
+
print(
|
| 1175 |
+
f"PSNR: {stats['psnr']:.3f}, SSIM: {stats['ssim']:.4f}, LPIPS: {stats['lpips']:.3f} "
|
| 1176 |
+
f"Time: {stats['ellipse_time']:.3f}s/image "
|
| 1177 |
+
f"Number of GS: {stats['num_GS']}"
|
| 1178 |
+
)
|
| 1179 |
+
# save stats as json
|
| 1180 |
+
with open(f"{self.stats_dir}/{stage}_step{step:04d}.json", "w") as f:
|
| 1181 |
+
json.dump(stats, f)
|
| 1182 |
+
# save stats to tensorboard
|
| 1183 |
+
for k, v in stats.items():
|
| 1184 |
+
self.writer.add_scalar(f"{stage}/{k}", v, step)
|
| 1185 |
+
self.writer.flush()
|
| 1186 |
+
|
| 1187 |
+
@torch.no_grad()
|
| 1188 |
+
def render_traj(self, step: int):
|
| 1189 |
+
"""Entry for trajectory rendering."""
|
| 1190 |
+
if self.cfg.disable_video:
|
| 1191 |
+
return
|
| 1192 |
+
print("Running trajectory rendering...")
|
| 1193 |
+
cfg = self.cfg
|
| 1194 |
+
device = self.device
|
| 1195 |
+
|
| 1196 |
+
camtoworlds_all = self.parser.camtoworlds[5:-5]
|
| 1197 |
+
if cfg.render_traj_path == "interp":
|
| 1198 |
+
camtoworlds_all = generate_interpolated_path(
|
| 1199 |
+
camtoworlds_all, 1
|
| 1200 |
+
) # [N, 3, 4]
|
| 1201 |
+
elif cfg.render_traj_path == "ellipse":
|
| 1202 |
+
height = camtoworlds_all[:, 2, 3].mean()
|
| 1203 |
+
camtoworlds_all = generate_ellipse_path_z(
|
| 1204 |
+
camtoworlds_all, height=height
|
| 1205 |
+
) # [N, 3, 4]
|
| 1206 |
+
elif cfg.render_traj_path == "spiral":
|
| 1207 |
+
camtoworlds_all = generate_spiral_path(
|
| 1208 |
+
camtoworlds_all,
|
| 1209 |
+
bounds=self.parser.bounds * self.scene_scale,
|
| 1210 |
+
spiral_scale_r=self.parser.extconf["spiral_radius_scale"],
|
| 1211 |
+
)
|
| 1212 |
+
else:
|
| 1213 |
+
raise ValueError(
|
| 1214 |
+
f"Render trajectory type not supported: {cfg.render_traj_path}"
|
| 1215 |
+
)
|
| 1216 |
+
|
| 1217 |
+
camtoworlds_all = np.concatenate(
|
| 1218 |
+
[
|
| 1219 |
+
camtoworlds_all,
|
| 1220 |
+
np.repeat(
|
| 1221 |
+
np.array([[[0.0, 0.0, 0.0, 1.0]]]), len(camtoworlds_all), axis=0
|
| 1222 |
+
),
|
| 1223 |
+
],
|
| 1224 |
+
axis=1,
|
| 1225 |
+
) # [N, 4, 4]
|
| 1226 |
+
|
| 1227 |
+
camtoworlds_all = torch.from_numpy(camtoworlds_all).float().to(device)
|
| 1228 |
+
K = torch.from_numpy(list(self.parser.Ks_dict.values())[0]).float().to(device)
|
| 1229 |
+
width, height = list(self.parser.imsize_dict.values())[0]
|
| 1230 |
+
|
| 1231 |
+
# save to video
|
| 1232 |
+
video_dir = f"{cfg.result_dir}/videos"
|
| 1233 |
+
os.makedirs(video_dir, exist_ok=True)
|
| 1234 |
+
writer = imageio.get_writer(f"{video_dir}/traj_{step}.mp4", fps=30)
|
| 1235 |
+
for i in tqdm.trange(len(camtoworlds_all), desc="Rendering trajectory"):
|
| 1236 |
+
camtoworlds = camtoworlds_all[i : i + 1]
|
| 1237 |
+
Ks = K[None]
|
| 1238 |
+
|
| 1239 |
+
renders, _, _ = self.rasterize_splats(
|
| 1240 |
+
camtoworlds=camtoworlds,
|
| 1241 |
+
Ks=Ks,
|
| 1242 |
+
width=width,
|
| 1243 |
+
height=height,
|
| 1244 |
+
sh_degree=cfg.sh_degree,
|
| 1245 |
+
near_plane=cfg.near_plane,
|
| 1246 |
+
far_plane=cfg.far_plane,
|
| 1247 |
+
render_mode="RGB+ED",
|
| 1248 |
+
) # [1, H, W, 4]
|
| 1249 |
+
colors = torch.clamp(renders[..., 0:3], 0.0, 1.0) # [1, H, W, 3]
|
| 1250 |
+
depths = renders[..., 3:4] # [1, H, W, 1]
|
| 1251 |
+
depths = (depths - depths.min()) / (depths.max() - depths.min())
|
| 1252 |
+
canvas_list = [colors, depths.repeat(1, 1, 1, 3)]
|
| 1253 |
+
|
| 1254 |
+
# write images
|
| 1255 |
+
canvas = torch.cat(canvas_list, dim=2).squeeze(0).cpu().numpy()
|
| 1256 |
+
canvas = (canvas * 255).astype(np.uint8)
|
| 1257 |
+
writer.append_data(canvas)
|
| 1258 |
+
writer.close()
|
| 1259 |
+
print(f"Video saved to {video_dir}/traj_{step}.mp4")
|
| 1260 |
+
|
| 1261 |
+
@torch.no_grad()
|
| 1262 |
+
def run_compression(self, step: int):
|
| 1263 |
+
"""Entry for running compression."""
|
| 1264 |
+
print("Running compression...")
|
| 1265 |
+
world_rank = self.world_rank
|
| 1266 |
+
|
| 1267 |
+
compress_dir = f"{cfg.result_dir}/compression/rank{world_rank}"
|
| 1268 |
+
os.makedirs(compress_dir, exist_ok=True)
|
| 1269 |
+
|
| 1270 |
+
self.compression_method.compress(compress_dir, self.splats)
|
| 1271 |
+
|
| 1272 |
+
# evaluate compression
|
| 1273 |
+
splats_c = self.compression_method.decompress(compress_dir)
|
| 1274 |
+
for k in splats_c.keys():
|
| 1275 |
+
self.splats[k].data = splats_c[k].to(self.device)
|
| 1276 |
+
self.eval(step=step, stage="compress")
|
| 1277 |
+
|
| 1278 |
+
@torch.no_grad()
|
| 1279 |
+
def _viewer_render_fn(
|
| 1280 |
+
self, camera_state: CameraState, render_tab_state: RenderTabState
|
| 1281 |
+
):
|
| 1282 |
+
assert isinstance(render_tab_state, GsplatRenderTabState)
|
| 1283 |
+
if render_tab_state.preview_render:
|
| 1284 |
+
width = render_tab_state.render_width
|
| 1285 |
+
height = render_tab_state.render_height
|
| 1286 |
+
else:
|
| 1287 |
+
width = render_tab_state.viewer_width
|
| 1288 |
+
height = render_tab_state.viewer_height
|
| 1289 |
+
c2w = camera_state.c2w
|
| 1290 |
+
K = camera_state.get_K((width, height))
|
| 1291 |
+
c2w = torch.from_numpy(c2w).float().to(self.device)
|
| 1292 |
+
K = torch.from_numpy(K).float().to(self.device)
|
| 1293 |
+
|
| 1294 |
+
RENDER_MODE_MAP = {
|
| 1295 |
+
"rgb": "RGB",
|
| 1296 |
+
"depth(accumulated)": "D",
|
| 1297 |
+
"depth(expected)": "ED",
|
| 1298 |
+
"alpha": "RGB",
|
| 1299 |
+
}
|
| 1300 |
+
|
| 1301 |
+
render_colors, render_alphas, info = self.rasterize_splats(
|
| 1302 |
+
camtoworlds=c2w[None],
|
| 1303 |
+
Ks=K[None],
|
| 1304 |
+
width=width,
|
| 1305 |
+
height=height,
|
| 1306 |
+
sh_degree=min(render_tab_state.max_sh_degree, self.cfg.sh_degree),
|
| 1307 |
+
near_plane=render_tab_state.near_plane,
|
| 1308 |
+
far_plane=render_tab_state.far_plane,
|
| 1309 |
+
radius_clip=render_tab_state.radius_clip,
|
| 1310 |
+
# radius_clip=0.1,
|
| 1311 |
+
eps2d=render_tab_state.eps2d,
|
| 1312 |
+
backgrounds=torch.tensor([render_tab_state.backgrounds], device=self.device)
|
| 1313 |
+
/ 255.0,
|
| 1314 |
+
render_mode=RENDER_MODE_MAP[render_tab_state.render_mode],
|
| 1315 |
+
rasterize_mode=render_tab_state.rasterize_mode,
|
| 1316 |
+
camera_model=render_tab_state.camera_model,
|
| 1317 |
+
) # [1, H, W, 3]
|
| 1318 |
+
render_tab_state.total_gs_count = len(self.splats["means"])
|
| 1319 |
+
render_tab_state.rendered_gs_count = (info["radii"] > 0).all(-1).sum().item()
|
| 1320 |
+
|
| 1321 |
+
if render_tab_state.render_mode == "rgb":
|
| 1322 |
+
# colors represented with sh are not guranteed to be in [0, 1]
|
| 1323 |
+
render_colors = render_colors[0, ..., 0:3].clamp(0, 1)
|
| 1324 |
+
renders = render_colors.cpu().numpy()
|
| 1325 |
+
elif render_tab_state.render_mode in ["depth(accumulated)", "depth(expected)"]:
|
| 1326 |
+
# normalize depth to [0, 1]
|
| 1327 |
+
depth = render_colors[0, ..., 0:1]
|
| 1328 |
+
if render_tab_state.normalize_nearfar:
|
| 1329 |
+
near_plane = render_tab_state.near_plane
|
| 1330 |
+
far_plane = render_tab_state.far_plane
|
| 1331 |
+
else:
|
| 1332 |
+
near_plane = depth.min()
|
| 1333 |
+
far_plane = depth.max()
|
| 1334 |
+
depth_norm = (depth - near_plane) / (far_plane - near_plane + 1e-10)
|
| 1335 |
+
depth_norm = torch.clip(depth_norm, 0, 1)
|
| 1336 |
+
if render_tab_state.inverse:
|
| 1337 |
+
depth_norm = 1 - depth_norm
|
| 1338 |
+
renders = (
|
| 1339 |
+
apply_float_colormap(depth_norm, render_tab_state.colormap)
|
| 1340 |
+
.cpu()
|
| 1341 |
+
.numpy()
|
| 1342 |
+
)
|
| 1343 |
+
elif render_tab_state.render_mode == "alpha":
|
| 1344 |
+
alpha = render_alphas[0, ..., 0:1]
|
| 1345 |
+
if render_tab_state.inverse:
|
| 1346 |
+
alpha = 1 - alpha
|
| 1347 |
+
renders = (
|
| 1348 |
+
apply_float_colormap(alpha, render_tab_state.colormap).cpu().numpy()
|
| 1349 |
+
)
|
| 1350 |
+
return renders
|
| 1351 |
+
|
| 1352 |
+
|
| 1353 |
+
def main(local_rank: int, world_rank, world_size: int, cfg: Config):
|
| 1354 |
+
if world_size > 1 and not cfg.disable_viewer:
|
| 1355 |
+
cfg.disable_viewer = True
|
| 1356 |
+
if world_rank == 0:
|
| 1357 |
+
print("Viewer is disabled in distributed training.")
|
| 1358 |
+
|
| 1359 |
+
runner = Runner(local_rank, world_rank, world_size, cfg)
|
| 1360 |
+
|
| 1361 |
+
if cfg.ckpt is not None:
|
| 1362 |
+
# run eval only
|
| 1363 |
+
ckpts = [
|
| 1364 |
+
torch.load(file, map_location=runner.device, weights_only=True)
|
| 1365 |
+
for file in cfg.ckpt
|
| 1366 |
+
]
|
| 1367 |
+
for k in runner.splats.keys():
|
| 1368 |
+
runner.splats[k].data = torch.cat([ckpt["splats"][k] for ckpt in ckpts])
|
| 1369 |
+
step = ckpts[0]["step"]
|
| 1370 |
+
runner.eval(step=step)
|
| 1371 |
+
# runner.render_traj(step=step)
|
| 1372 |
+
if cfg.compression is not None:
|
| 1373 |
+
runner.run_compression(step=step)
|
| 1374 |
+
else:
|
| 1375 |
+
runner.train()
|
| 1376 |
+
runner.eval(step=runner.cfg.max_steps)
|
| 1377 |
+
# runner.render_traj(step=runner.cfg.max_steps)
|
| 1378 |
+
print("Training complete.")
|
| 1379 |
+
# runner.viewer.complete()
|
| 1380 |
+
# if not cfg.disable_viewer:
|
| 1381 |
+
# print("Viewer running... Ctrl+C to exit.")
|
| 1382 |
+
# time.sleep(1000000)
|
| 1383 |
+
|
| 1384 |
+
|
| 1385 |
+
if __name__ == "__main__":
|
| 1386 |
+
"""
|
| 1387 |
+
Usage:
|
| 1388 |
+
|
| 1389 |
+
```bash
|
| 1390 |
+
# Single GPU training
|
| 1391 |
+
CUDA_VISIBLE_DEVICES=9 python -m examples.simple_trainer default
|
| 1392 |
+
|
| 1393 |
+
# Distributed training on 4 GPUs: Effectively 4x batch size so run 4x less steps.
|
| 1394 |
+
CUDA_VISIBLE_DEVICES=0,1,2,3 python simple_trainer.py default --steps_scaler 0.25
|
| 1395 |
+
|
| 1396 |
+
"""
|
| 1397 |
+
|
| 1398 |
+
# Config objects we can choose between.
|
| 1399 |
+
# Each is a tuple of (CLI description, config object).
|
| 1400 |
+
configs = {
|
| 1401 |
+
"default": (
|
| 1402 |
+
"Gaussian splatting training using densification heuristics from the original paper.",
|
| 1403 |
+
Config(
|
| 1404 |
+
strategy=DefaultStrategy(verbose=True),
|
| 1405 |
+
),
|
| 1406 |
+
),
|
| 1407 |
+
"mcmc": (
|
| 1408 |
+
"Gaussian splatting training using densification from the paper '3D Gaussian Splatting as Markov Chain Monte Carlo'.",
|
| 1409 |
+
Config(
|
| 1410 |
+
init_opa=0.5,
|
| 1411 |
+
init_scale=0.1,
|
| 1412 |
+
opacity_reg=0.01,
|
| 1413 |
+
scale_reg=0.01,
|
| 1414 |
+
strategy=MCMCStrategy(verbose=True),
|
| 1415 |
+
),
|
| 1416 |
+
),
|
| 1417 |
+
}
|
| 1418 |
+
cfg = tyro.extras.overridable_config_cli(configs)
|
| 1419 |
+
cfg.adjust_steps(cfg.steps_scaler)
|
| 1420 |
+
|
| 1421 |
+
# try import extra dependencies
|
| 1422 |
+
if cfg.compression == "png":
|
| 1423 |
+
try:
|
| 1424 |
+
import plas
|
| 1425 |
+
import torchpq
|
| 1426 |
+
except:
|
| 1427 |
+
raise ImportError(
|
| 1428 |
+
"To use PNG compression, you need to install "
|
| 1429 |
+
"torchpq (instruction at https://github.com/DeMoriarty/TorchPQ?tab=readme-ov-file#install) "
|
| 1430 |
+
"and plas (via 'pip install git+https://github.com/fraunhoferhhi/PLAS.git') "
|
| 1431 |
+
)
|
| 1432 |
+
|
| 1433 |
+
cli(main, cfg, verbose=True)
|
src/post_opt/simple_viewer.py
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
import math
|
| 3 |
+
import os
|
| 4 |
+
import time
|
| 5 |
+
|
| 6 |
+
import imageio
|
| 7 |
+
import numpy as np
|
| 8 |
+
import torch
|
| 9 |
+
import torch.nn.functional as F
|
| 10 |
+
import tqdm
|
| 11 |
+
import viser
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
from gsplat._helper import load_test_data
|
| 14 |
+
from gsplat.distributed import cli
|
| 15 |
+
from gsplat.rendering import rasterization
|
| 16 |
+
|
| 17 |
+
from nerfview import CameraState, RenderTabState, apply_float_colormap
|
| 18 |
+
from examples.gsplat_viewer import GsplatViewer, GsplatRenderTabState
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def main(local_rank: int, world_rank, world_size: int, args):
|
| 22 |
+
torch.manual_seed(42)
|
| 23 |
+
device = torch.device("cuda", local_rank)
|
| 24 |
+
|
| 25 |
+
if args.ckpt is None:
|
| 26 |
+
(
|
| 27 |
+
means,
|
| 28 |
+
quats,
|
| 29 |
+
scales,
|
| 30 |
+
opacities,
|
| 31 |
+
colors,
|
| 32 |
+
viewmats,
|
| 33 |
+
Ks,
|
| 34 |
+
width,
|
| 35 |
+
height,
|
| 36 |
+
) = load_test_data(device=device, scene_grid=args.scene_grid)
|
| 37 |
+
|
| 38 |
+
assert world_size <= 2
|
| 39 |
+
means = means[world_rank::world_size].contiguous()
|
| 40 |
+
means.requires_grad = True
|
| 41 |
+
quats = quats[world_rank::world_size].contiguous()
|
| 42 |
+
quats.requires_grad = True
|
| 43 |
+
scales = scales[world_rank::world_size].contiguous()
|
| 44 |
+
scales.requires_grad = True
|
| 45 |
+
opacities = opacities[world_rank::world_size].contiguous()
|
| 46 |
+
opacities.requires_grad = True
|
| 47 |
+
colors = colors[world_rank::world_size].contiguous()
|
| 48 |
+
colors.requires_grad = True
|
| 49 |
+
|
| 50 |
+
viewmats = viewmats[world_rank::world_size][:1].contiguous()
|
| 51 |
+
Ks = Ks[world_rank::world_size][:1].contiguous()
|
| 52 |
+
|
| 53 |
+
sh_degree = None
|
| 54 |
+
C = len(viewmats)
|
| 55 |
+
N = len(means)
|
| 56 |
+
print("rank", world_rank, "Number of Gaussians:", N, "Number of Cameras:", C)
|
| 57 |
+
|
| 58 |
+
# batched render
|
| 59 |
+
for _ in tqdm.trange(1):
|
| 60 |
+
render_colors, render_alphas, meta = rasterization(
|
| 61 |
+
means, # [N, 3]
|
| 62 |
+
quats, # [N, 4]
|
| 63 |
+
scales, # [N, 3]
|
| 64 |
+
opacities, # [N]
|
| 65 |
+
colors, # [N, S, 3]
|
| 66 |
+
viewmats, # [C, 4, 4]
|
| 67 |
+
Ks, # [C, 3, 3]
|
| 68 |
+
width,
|
| 69 |
+
height,
|
| 70 |
+
render_mode="RGB+D",
|
| 71 |
+
packed=False,
|
| 72 |
+
distributed=world_size > 1,
|
| 73 |
+
)
|
| 74 |
+
C = render_colors.shape[0]
|
| 75 |
+
assert render_colors.shape == (C, height, width, 4)
|
| 76 |
+
assert render_alphas.shape == (C, height, width, 1)
|
| 77 |
+
render_colors.sum().backward()
|
| 78 |
+
|
| 79 |
+
render_rgbs = render_colors[..., 0:3]
|
| 80 |
+
render_depths = render_colors[..., 3:4]
|
| 81 |
+
render_depths = render_depths / render_depths.max()
|
| 82 |
+
|
| 83 |
+
# dump batch images
|
| 84 |
+
os.makedirs(args.output_dir, exist_ok=True)
|
| 85 |
+
canvas = (
|
| 86 |
+
torch.cat(
|
| 87 |
+
[
|
| 88 |
+
render_rgbs.reshape(C * height, width, 3),
|
| 89 |
+
render_depths.reshape(C * height, width, 1).expand(-1, -1, 3),
|
| 90 |
+
render_alphas.reshape(C * height, width, 1).expand(-1, -1, 3),
|
| 91 |
+
],
|
| 92 |
+
dim=1,
|
| 93 |
+
)
|
| 94 |
+
.detach()
|
| 95 |
+
.cpu()
|
| 96 |
+
.numpy()
|
| 97 |
+
)
|
| 98 |
+
imageio.imsave(
|
| 99 |
+
f"{args.output_dir}/render_rank{world_rank}.png",
|
| 100 |
+
(canvas * 255).astype(np.uint8),
|
| 101 |
+
)
|
| 102 |
+
else:
|
| 103 |
+
means, quats, scales, opacities, sh0, shN = [], [], [], [], [], []
|
| 104 |
+
for ckpt_path in args.ckpt:
|
| 105 |
+
ckpt = torch.load(ckpt_path, map_location=device)["splats"]
|
| 106 |
+
means.append(ckpt["means"])
|
| 107 |
+
quats.append(F.normalize(ckpt["quats"], p=2, dim=-1))
|
| 108 |
+
scales.append(torch.exp(ckpt["scales"]))
|
| 109 |
+
opacities.append(torch.sigmoid(ckpt["opacities"]))
|
| 110 |
+
sh0.append(ckpt["sh0"])
|
| 111 |
+
shN.append(ckpt["shN"])
|
| 112 |
+
means = torch.cat(means, dim=0)
|
| 113 |
+
quats = torch.cat(quats, dim=0)
|
| 114 |
+
scales = torch.cat(scales, dim=0)
|
| 115 |
+
opacities = torch.cat(opacities, dim=0)
|
| 116 |
+
sh0 = torch.cat(sh0, dim=0)
|
| 117 |
+
shN = torch.cat(shN, dim=0)
|
| 118 |
+
colors = torch.cat([sh0, shN], dim=-2)
|
| 119 |
+
sh_degree = int(math.sqrt(colors.shape[-2]) - 1)
|
| 120 |
+
|
| 121 |
+
# # crop
|
| 122 |
+
# aabb = torch.tensor((-1.0, -1.0, -1.0, 1.0, 1.0, 0.7), device=device)
|
| 123 |
+
# edges = aabb[3:] - aabb[:3]
|
| 124 |
+
# sel = ((means >= aabb[:3]) & (means <= aabb[3:])).all(dim=-1)
|
| 125 |
+
# sel = torch.where(sel)[0]
|
| 126 |
+
# means, quats, scales, colors, opacities = (
|
| 127 |
+
# means[sel],
|
| 128 |
+
# quats[sel],
|
| 129 |
+
# scales[sel],
|
| 130 |
+
# colors[sel],
|
| 131 |
+
# opacities[sel],
|
| 132 |
+
# )
|
| 133 |
+
|
| 134 |
+
# # repeat the scene into a grid (to mimic a large-scale setting)
|
| 135 |
+
# repeats = args.scene_grid
|
| 136 |
+
# gridx, gridy = torch.meshgrid(
|
| 137 |
+
# [
|
| 138 |
+
# torch.arange(-(repeats // 2), repeats // 2 + 1, device=device),
|
| 139 |
+
# torch.arange(-(repeats // 2), repeats // 2 + 1, device=device),
|
| 140 |
+
# ],
|
| 141 |
+
# indexing="ij",
|
| 142 |
+
# )
|
| 143 |
+
# grid = torch.stack([gridx, gridy, torch.zeros_like(gridx)], dim=-1).reshape(
|
| 144 |
+
# -1, 3
|
| 145 |
+
# )
|
| 146 |
+
# means = means[None, :, :] + grid[:, None, :] * edges[None, None, :]
|
| 147 |
+
# means = means.reshape(-1, 3)
|
| 148 |
+
# quats = quats.repeat(repeats**2, 1)
|
| 149 |
+
# scales = scales.repeat(repeats**2, 1)
|
| 150 |
+
# colors = colors.repeat(repeats**2, 1, 1)
|
| 151 |
+
# opacities = opacities.repeat(repeats**2)
|
| 152 |
+
print("Number of Gaussians:", len(means))
|
| 153 |
+
|
| 154 |
+
# register and open viewer
|
| 155 |
+
@torch.no_grad()
|
| 156 |
+
def viewer_render_fn(camera_state: CameraState, render_tab_state: RenderTabState):
|
| 157 |
+
assert isinstance(render_tab_state, GsplatRenderTabState)
|
| 158 |
+
if render_tab_state.preview_render:
|
| 159 |
+
width = render_tab_state.render_width
|
| 160 |
+
height = render_tab_state.render_height
|
| 161 |
+
else:
|
| 162 |
+
width = render_tab_state.viewer_width
|
| 163 |
+
height = render_tab_state.viewer_height
|
| 164 |
+
c2w = camera_state.c2w
|
| 165 |
+
K = camera_state.get_K((width, height))
|
| 166 |
+
c2w = torch.from_numpy(c2w).float().to(device)
|
| 167 |
+
K = torch.from_numpy(K).float().to(device)
|
| 168 |
+
viewmat = c2w.inverse()
|
| 169 |
+
|
| 170 |
+
RENDER_MODE_MAP = {
|
| 171 |
+
"rgb": "RGB",
|
| 172 |
+
"depth(accumulated)": "D",
|
| 173 |
+
"depth(expected)": "ED",
|
| 174 |
+
"alpha": "RGB",
|
| 175 |
+
}
|
| 176 |
+
|
| 177 |
+
render_colors, render_alphas, info = rasterization(
|
| 178 |
+
means, # [N, 3]
|
| 179 |
+
quats, # [N, 4]
|
| 180 |
+
scales, # [N, 3]
|
| 181 |
+
opacities, # [N]
|
| 182 |
+
colors, # [N, S, 3]
|
| 183 |
+
viewmat[None], # [1, 4, 4]
|
| 184 |
+
K[None], # [1, 3, 3]
|
| 185 |
+
width,
|
| 186 |
+
height,
|
| 187 |
+
sh_degree=(
|
| 188 |
+
min(render_tab_state.max_sh_degree, sh_degree)
|
| 189 |
+
if sh_degree is not None
|
| 190 |
+
else None
|
| 191 |
+
),
|
| 192 |
+
near_plane=render_tab_state.near_plane,
|
| 193 |
+
far_plane=render_tab_state.far_plane,
|
| 194 |
+
radius_clip=render_tab_state.radius_clip,
|
| 195 |
+
eps2d=render_tab_state.eps2d,
|
| 196 |
+
backgrounds=torch.tensor([render_tab_state.backgrounds], device=device)
|
| 197 |
+
/ 255.0,
|
| 198 |
+
render_mode=RENDER_MODE_MAP[render_tab_state.render_mode],
|
| 199 |
+
rasterize_mode=render_tab_state.rasterize_mode,
|
| 200 |
+
camera_model=render_tab_state.camera_model,
|
| 201 |
+
)
|
| 202 |
+
render_tab_state.total_gs_count = len(means)
|
| 203 |
+
render_tab_state.rendered_gs_count = (info["radii"] > 0).all(-1).sum().item()
|
| 204 |
+
|
| 205 |
+
if render_tab_state.render_mode == "rgb":
|
| 206 |
+
# colors represented with sh are not guranteed to be in [0, 1]
|
| 207 |
+
render_colors = render_colors[0, ..., 0:3].clamp(0, 1)
|
| 208 |
+
renders = render_colors.cpu().numpy()
|
| 209 |
+
elif render_tab_state.render_mode in ["depth(accumulated)", "depth(expected)"]:
|
| 210 |
+
# normalize depth to [0, 1]
|
| 211 |
+
depth = render_colors[0, ..., 0:1]
|
| 212 |
+
if render_tab_state.normalize_nearfar:
|
| 213 |
+
near_plane = render_tab_state.near_plane
|
| 214 |
+
far_plane = render_tab_state.far_plane
|
| 215 |
+
else:
|
| 216 |
+
near_plane = depth.min()
|
| 217 |
+
far_plane = depth.max()
|
| 218 |
+
depth_norm = (depth - near_plane) / (far_plane - near_plane + 1e-10)
|
| 219 |
+
depth_norm = torch.clip(depth_norm, 0, 1)
|
| 220 |
+
if render_tab_state.inverse:
|
| 221 |
+
depth_norm = 1 - depth_norm
|
| 222 |
+
renders = (
|
| 223 |
+
apply_float_colormap(depth_norm, render_tab_state.colormap)
|
| 224 |
+
.cpu()
|
| 225 |
+
.numpy()
|
| 226 |
+
)
|
| 227 |
+
elif render_tab_state.render_mode == "alpha":
|
| 228 |
+
alpha = render_alphas[0, ..., 0:1]
|
| 229 |
+
if render_tab_state.inverse:
|
| 230 |
+
alpha = 1 - alpha
|
| 231 |
+
renders = (
|
| 232 |
+
apply_float_colormap(alpha, render_tab_state.colormap).cpu().numpy()
|
| 233 |
+
)
|
| 234 |
+
return renders
|
| 235 |
+
|
| 236 |
+
server = viser.ViserServer(port=args.port, verbose=False)
|
| 237 |
+
_ = GsplatViewer(
|
| 238 |
+
server=server,
|
| 239 |
+
render_fn=viewer_render_fn,
|
| 240 |
+
output_dir=Path(args.output_dir),
|
| 241 |
+
mode="rendering",
|
| 242 |
+
)
|
| 243 |
+
print("Viewer running... Ctrl+C to exit.")
|
| 244 |
+
time.sleep(100000)
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
if __name__ == "__main__":
|
| 248 |
+
"""
|
| 249 |
+
# Use single GPU to view the scene
|
| 250 |
+
CUDA_VISIBLE_DEVICES=9 python -m simple_viewer \
|
| 251 |
+
--ckpt results/garden/ckpts/ckpt_6999_rank0.pt \
|
| 252 |
+
--output_dir results/garden/ \
|
| 253 |
+
--port 8082
|
| 254 |
+
|
| 255 |
+
CUDA_VISIBLE_DEVICES=9 python -m simple_viewer \
|
| 256 |
+
--output_dir results/garden/ \
|
| 257 |
+
--port 8082
|
| 258 |
+
"""
|
| 259 |
+
parser = argparse.ArgumentParser()
|
| 260 |
+
parser.add_argument(
|
| 261 |
+
"--output_dir", type=str, default="results/", help="where to dump outputs"
|
| 262 |
+
)
|
| 263 |
+
parser.add_argument(
|
| 264 |
+
"--scene_grid", type=int, default=1, help="repeat the scene into a grid of NxN"
|
| 265 |
+
)
|
| 266 |
+
parser.add_argument(
|
| 267 |
+
"--ckpt", type=str, nargs="+", default=None, help="path to the .pt file"
|
| 268 |
+
)
|
| 269 |
+
parser.add_argument(
|
| 270 |
+
"--port", type=int, default=8080, help="port for the viewer server"
|
| 271 |
+
)
|
| 272 |
+
args = parser.parse_args()
|
| 273 |
+
assert args.scene_grid % 2 == 1, "scene_grid must be odd"
|
| 274 |
+
|
| 275 |
+
cli(main, args, verbose=True)
|
src/post_opt/utils.py
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import random
|
| 2 |
+
|
| 3 |
+
import numpy as np
|
| 4 |
+
import torch
|
| 5 |
+
from sklearn.neighbors import NearestNeighbors
|
| 6 |
+
from torch import Tensor
|
| 7 |
+
import torch.nn.functional as F
|
| 8 |
+
import matplotlib.pyplot as plt
|
| 9 |
+
from matplotlib import colormaps
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class CameraOptModule(torch.nn.Module):
|
| 13 |
+
"""Camera pose optimization module."""
|
| 14 |
+
|
| 15 |
+
def __init__(self, n: int):
|
| 16 |
+
super().__init__()
|
| 17 |
+
# Delta positions (3D) + Delta rotations (6D)
|
| 18 |
+
self.embeds = torch.nn.Embedding(n, 9)
|
| 19 |
+
# Identity rotation in 6D representation
|
| 20 |
+
self.register_buffer("identity", torch.tensor([1.0, 0.0, 0.0, 0.0, 1.0, 0.0]))
|
| 21 |
+
|
| 22 |
+
def zero_init(self):
|
| 23 |
+
torch.nn.init.zeros_(self.embeds.weight)
|
| 24 |
+
|
| 25 |
+
def random_init(self, std: float):
|
| 26 |
+
torch.nn.init.normal_(self.embeds.weight, std=std)
|
| 27 |
+
|
| 28 |
+
def forward(self, camtoworlds: Tensor, embed_ids: Tensor) -> Tensor:
|
| 29 |
+
"""Adjust camera pose based on deltas.
|
| 30 |
+
|
| 31 |
+
Args:
|
| 32 |
+
camtoworlds: (..., 4, 4)
|
| 33 |
+
embed_ids: (...,)
|
| 34 |
+
|
| 35 |
+
Returns:
|
| 36 |
+
updated camtoworlds: (..., 4, 4)
|
| 37 |
+
"""
|
| 38 |
+
assert camtoworlds.shape[:-2] == embed_ids.shape
|
| 39 |
+
batch_shape = camtoworlds.shape[:-2]
|
| 40 |
+
pose_deltas = self.embeds(embed_ids) # (..., 9)
|
| 41 |
+
dx, drot = pose_deltas[..., :3], pose_deltas[..., 3:]
|
| 42 |
+
rot = rotation_6d_to_matrix(
|
| 43 |
+
drot + self.identity.expand(*batch_shape, -1)
|
| 44 |
+
) # (..., 3, 3)
|
| 45 |
+
transform = torch.eye(4, device=pose_deltas.device).repeat((*batch_shape, 1, 1))
|
| 46 |
+
transform[..., :3, :3] = rot
|
| 47 |
+
transform[..., :3, 3] = dx
|
| 48 |
+
return torch.matmul(camtoworlds, transform)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class AppearanceOptModule(torch.nn.Module):
|
| 52 |
+
"""Appearance optimization module."""
|
| 53 |
+
|
| 54 |
+
def __init__(
|
| 55 |
+
self,
|
| 56 |
+
n: int,
|
| 57 |
+
feature_dim: int,
|
| 58 |
+
embed_dim: int = 16,
|
| 59 |
+
sh_degree: int = 3,
|
| 60 |
+
mlp_width: int = 64,
|
| 61 |
+
mlp_depth: int = 2,
|
| 62 |
+
):
|
| 63 |
+
super().__init__()
|
| 64 |
+
self.embed_dim = embed_dim
|
| 65 |
+
self.sh_degree = sh_degree
|
| 66 |
+
self.embeds = torch.nn.Embedding(n, embed_dim)
|
| 67 |
+
layers = []
|
| 68 |
+
layers.append(
|
| 69 |
+
torch.nn.Linear(embed_dim + feature_dim + (sh_degree + 1) ** 2, mlp_width)
|
| 70 |
+
)
|
| 71 |
+
layers.append(torch.nn.ReLU(inplace=True))
|
| 72 |
+
for _ in range(mlp_depth - 1):
|
| 73 |
+
layers.append(torch.nn.Linear(mlp_width, mlp_width))
|
| 74 |
+
layers.append(torch.nn.ReLU(inplace=True))
|
| 75 |
+
layers.append(torch.nn.Linear(mlp_width, 3))
|
| 76 |
+
self.color_head = torch.nn.Sequential(*layers)
|
| 77 |
+
|
| 78 |
+
def forward(
|
| 79 |
+
self, features: Tensor, embed_ids: Tensor, dirs: Tensor, sh_degree: int
|
| 80 |
+
) -> Tensor:
|
| 81 |
+
"""Adjust appearance based on embeddings.
|
| 82 |
+
|
| 83 |
+
Args:
|
| 84 |
+
features: (N, feature_dim)
|
| 85 |
+
embed_ids: (C,)
|
| 86 |
+
dirs: (C, N, 3)
|
| 87 |
+
|
| 88 |
+
Returns:
|
| 89 |
+
colors: (C, N, 3)
|
| 90 |
+
"""
|
| 91 |
+
from gsplat.cuda._torch_impl import _eval_sh_bases_fast
|
| 92 |
+
|
| 93 |
+
C, N = dirs.shape[:2]
|
| 94 |
+
# Camera embeddings
|
| 95 |
+
if embed_ids is None:
|
| 96 |
+
embeds = torch.zeros(C, self.embed_dim, device=features.device)
|
| 97 |
+
else:
|
| 98 |
+
embeds = self.embeds(embed_ids) # [C, D2]
|
| 99 |
+
embeds = embeds[:, None, :].expand(-1, N, -1) # [C, N, D2]
|
| 100 |
+
# GS features
|
| 101 |
+
features = features[None, :, :].expand(C, -1, -1) # [C, N, D1]
|
| 102 |
+
# View directions
|
| 103 |
+
dirs = F.normalize(dirs, dim=-1) # [C, N, 3]
|
| 104 |
+
num_bases_to_use = (sh_degree + 1) ** 2
|
| 105 |
+
num_bases = (self.sh_degree + 1) ** 2
|
| 106 |
+
sh_bases = torch.zeros(C, N, num_bases, device=features.device) # [C, N, K]
|
| 107 |
+
sh_bases[:, :, :num_bases_to_use] = _eval_sh_bases_fast(num_bases_to_use, dirs)
|
| 108 |
+
# Get colors
|
| 109 |
+
if self.embed_dim > 0:
|
| 110 |
+
h = torch.cat([embeds, features, sh_bases], dim=-1) # [C, N, D1 + D2 + K]
|
| 111 |
+
else:
|
| 112 |
+
h = torch.cat([features, sh_bases], dim=-1)
|
| 113 |
+
colors = self.color_head(h)
|
| 114 |
+
return colors
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def rotation_6d_to_matrix(d6: Tensor) -> Tensor:
|
| 118 |
+
"""
|
| 119 |
+
Converts 6D rotation representation by Zhou et al. [1] to rotation matrix
|
| 120 |
+
using Gram--Schmidt orthogonalization per Section B of [1]. Adapted from pytorch3d.
|
| 121 |
+
Args:
|
| 122 |
+
d6: 6D rotation representation, of size (*, 6)
|
| 123 |
+
|
| 124 |
+
Returns:
|
| 125 |
+
batch of rotation matrices of size (*, 3, 3)
|
| 126 |
+
|
| 127 |
+
[1] Zhou, Y., Barnes, C., Lu, J., Yang, J., & Li, H.
|
| 128 |
+
On the Continuity of Rotation Representations in Neural Networks.
|
| 129 |
+
IEEE Conference on Computer Vision and Pattern Recognition, 2019.
|
| 130 |
+
Retrieved from http://arxiv.org/abs/1812.07035
|
| 131 |
+
"""
|
| 132 |
+
|
| 133 |
+
a1, a2 = d6[..., :3], d6[..., 3:]
|
| 134 |
+
b1 = F.normalize(a1, dim=-1)
|
| 135 |
+
b2 = a2 - (b1 * a2).sum(-1, keepdim=True) * b1
|
| 136 |
+
b2 = F.normalize(b2, dim=-1)
|
| 137 |
+
b3 = torch.cross(b1, b2, dim=-1)
|
| 138 |
+
return torch.stack((b1, b2, b3), dim=-2)
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def knn(x: Tensor, K: int = 4) -> Tensor:
|
| 142 |
+
x_np = x.cpu().numpy()
|
| 143 |
+
model = NearestNeighbors(n_neighbors=K, metric="euclidean").fit(x_np)
|
| 144 |
+
distances, _ = model.kneighbors(x_np)
|
| 145 |
+
return torch.from_numpy(distances).to(x)
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def rgb_to_sh(rgb: Tensor) -> Tensor:
|
| 149 |
+
C0 = 0.28209479177387814
|
| 150 |
+
return (rgb - 0.5) / C0
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def set_random_seed(seed: int):
|
| 154 |
+
random.seed(seed)
|
| 155 |
+
np.random.seed(seed)
|
| 156 |
+
torch.manual_seed(seed)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
# ref: https://github.com/hbb1/2d-gaussian-splatting/blob/main/utils/general_utils.py#L163
|
| 160 |
+
def colormap(img, cmap="jet"):
|
| 161 |
+
W, H = img.shape[:2]
|
| 162 |
+
dpi = 300
|
| 163 |
+
fig, ax = plt.subplots(1, figsize=(H / dpi, W / dpi), dpi=dpi)
|
| 164 |
+
im = ax.imshow(img, cmap=cmap)
|
| 165 |
+
ax.set_axis_off()
|
| 166 |
+
fig.colorbar(im, ax=ax)
|
| 167 |
+
fig.tight_layout()
|
| 168 |
+
fig.canvas.draw()
|
| 169 |
+
data = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)
|
| 170 |
+
data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
|
| 171 |
+
img = torch.from_numpy(data).float().permute(2, 0, 1)
|
| 172 |
+
plt.close()
|
| 173 |
+
return img
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def apply_float_colormap(img: torch.Tensor, colormap: str = "turbo") -> torch.Tensor:
|
| 177 |
+
"""Convert single channel to a color img.
|
| 178 |
+
|
| 179 |
+
Args:
|
| 180 |
+
img (torch.Tensor): (..., 1) float32 single channel image.
|
| 181 |
+
colormap (str): Colormap for img.
|
| 182 |
+
|
| 183 |
+
Returns:
|
| 184 |
+
(..., 3) colored img with colors in [0, 1].
|
| 185 |
+
"""
|
| 186 |
+
img = torch.nan_to_num(img, 0)
|
| 187 |
+
if colormap == "gray":
|
| 188 |
+
return img.repeat(1, 1, 3)
|
| 189 |
+
img_long = (img * 255).long()
|
| 190 |
+
img_long_min = torch.min(img_long)
|
| 191 |
+
img_long_max = torch.max(img_long)
|
| 192 |
+
assert img_long_min >= 0, f"the min value is {img_long_min}"
|
| 193 |
+
assert img_long_max <= 255, f"the max value is {img_long_max}"
|
| 194 |
+
return torch.tensor(
|
| 195 |
+
colormaps[colormap].colors, # type: ignore
|
| 196 |
+
device=img.device,
|
| 197 |
+
)[img_long[..., 0]]
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def apply_depth_colormap(
|
| 201 |
+
depth: torch.Tensor,
|
| 202 |
+
acc: torch.Tensor = None,
|
| 203 |
+
near_plane: float = None,
|
| 204 |
+
far_plane: float = None,
|
| 205 |
+
) -> torch.Tensor:
|
| 206 |
+
"""Converts a depth image to color for easier analysis.
|
| 207 |
+
|
| 208 |
+
Args:
|
| 209 |
+
depth (torch.Tensor): (..., 1) float32 depth.
|
| 210 |
+
acc (torch.Tensor | None): (..., 1) optional accumulation mask.
|
| 211 |
+
near_plane: Closest depth to consider. If None, use min image value.
|
| 212 |
+
far_plane: Furthest depth to consider. If None, use max image value.
|
| 213 |
+
|
| 214 |
+
Returns:
|
| 215 |
+
(..., 3) colored depth image with colors in [0, 1].
|
| 216 |
+
"""
|
| 217 |
+
near_plane = near_plane or float(torch.min(depth))
|
| 218 |
+
far_plane = far_plane or float(torch.max(depth))
|
| 219 |
+
depth = (depth - near_plane) / (far_plane - near_plane + 1e-10)
|
| 220 |
+
depth = torch.clip(depth, 0.0, 1.0)
|
| 221 |
+
img = apply_float_colormap(depth, colormap="turbo")
|
| 222 |
+
if acc is not None:
|
| 223 |
+
img = img * acc + (1.0 - acc)
|
| 224 |
+
return img
|
src/utils/__init.py
ADDED
|
File without changes
|
src/utils/ba.py
ADDED
|
@@ -0,0 +1,407 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import pycolmap
|
| 2 |
+
import numpy as np
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
import torch.nn.functional as F
|
| 6 |
+
from src.model.encoder.vggt.utils.pose_enc import pose_encoding_to_extri_intri
|
| 7 |
+
from src.model.encoder.vggt.utils.geometry import closed_form_inverse_se3, unproject_depth_map_to_point_map
|
| 8 |
+
from lightglue import ALIKED, SuperPoint, SIFT
|
| 9 |
+
from src.utils.tensor_to_pycolmap import batch_matrix_to_pycolmap, pycolmap_to_batch_matrix
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
_RESNET_MEAN = [0.485, 0.456, 0.406]
|
| 13 |
+
_RESNET_STD = [0.229, 0.224, 0.225]
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def generate_rank_by_dino(
|
| 17 |
+
images, query_frame_num, image_size=518, model_name="dinov2_vitb14_reg", device="cuda", spatial_similarity=True
|
| 18 |
+
):
|
| 19 |
+
"""
|
| 20 |
+
Generate a ranking of frames using DINO ViT features.
|
| 21 |
+
|
| 22 |
+
Args:
|
| 23 |
+
images: Tensor of shape (S, 3, H, W) with values in range [0, 1]
|
| 24 |
+
query_frame_num: Number of frames to select
|
| 25 |
+
image_size: Size to resize images to before processing
|
| 26 |
+
model_name: Name of the DINO model to use
|
| 27 |
+
device: Device to run the model on
|
| 28 |
+
spatial_similarity: Whether to use spatial token similarity or CLS token similarity
|
| 29 |
+
|
| 30 |
+
Returns:
|
| 31 |
+
List of frame indices ranked by their representativeness
|
| 32 |
+
"""
|
| 33 |
+
dino_v2_model = torch.hub.load('facebookresearch/dinov2', model_name)
|
| 34 |
+
dino_v2_model.eval()
|
| 35 |
+
dino_v2_model = dino_v2_model.to(device)
|
| 36 |
+
|
| 37 |
+
resnet_mean = torch.tensor(_RESNET_MEAN, device=device).view(1, 3, 1, 1)
|
| 38 |
+
resnet_std = torch.tensor(_RESNET_STD, device=device).view(1, 3, 1, 1)
|
| 39 |
+
images_resnet_norm = (images - resnet_mean) / resnet_std
|
| 40 |
+
|
| 41 |
+
with torch.no_grad():
|
| 42 |
+
frame_feat = dino_v2_model(images_resnet_norm, is_training=True)
|
| 43 |
+
|
| 44 |
+
if spatial_similarity:
|
| 45 |
+
frame_feat = frame_feat["x_norm_patchtokens"]
|
| 46 |
+
frame_feat_norm = F.normalize(frame_feat, p=2, dim=1)
|
| 47 |
+
|
| 48 |
+
# Compute the similarity matrix
|
| 49 |
+
frame_feat_norm = frame_feat_norm.permute(1, 0, 2)
|
| 50 |
+
similarity_matrix = torch.bmm(
|
| 51 |
+
frame_feat_norm, frame_feat_norm.transpose(-1, -2)
|
| 52 |
+
)
|
| 53 |
+
similarity_matrix = similarity_matrix.mean(dim=0)
|
| 54 |
+
else:
|
| 55 |
+
frame_feat = frame_feat["x_norm_clstoken"]
|
| 56 |
+
frame_feat_norm = F.normalize(frame_feat, p=2, dim=1)
|
| 57 |
+
similarity_matrix = torch.mm(
|
| 58 |
+
frame_feat_norm, frame_feat_norm.transpose(-1, -2)
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
distance_matrix = 100 - similarity_matrix.clone()
|
| 62 |
+
|
| 63 |
+
# Ignore self-pairing
|
| 64 |
+
similarity_matrix.fill_diagonal_(-100)
|
| 65 |
+
similarity_sum = similarity_matrix.sum(dim=1)
|
| 66 |
+
|
| 67 |
+
# Find the most common frame
|
| 68 |
+
most_common_frame_index = torch.argmax(similarity_sum).item()
|
| 69 |
+
|
| 70 |
+
# Conduct FPS sampling starting from the most common frame
|
| 71 |
+
fps_idx = farthest_point_sampling(
|
| 72 |
+
distance_matrix, query_frame_num, most_common_frame_index
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
return fps_idx
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def farthest_point_sampling(
|
| 79 |
+
distance_matrix, num_samples, most_common_frame_index=0
|
| 80 |
+
):
|
| 81 |
+
"""
|
| 82 |
+
Farthest point sampling algorithm to select diverse frames.
|
| 83 |
+
|
| 84 |
+
Args:
|
| 85 |
+
distance_matrix: Matrix of distances between frames
|
| 86 |
+
num_samples: Number of frames to select
|
| 87 |
+
most_common_frame_index: Index of the first frame to select
|
| 88 |
+
|
| 89 |
+
Returns:
|
| 90 |
+
List of selected frame indices
|
| 91 |
+
"""
|
| 92 |
+
distance_matrix = distance_matrix.clamp(min=0)
|
| 93 |
+
N = distance_matrix.size(0)
|
| 94 |
+
|
| 95 |
+
# Initialize with the most common frame
|
| 96 |
+
selected_indices = [most_common_frame_index]
|
| 97 |
+
check_distances = distance_matrix[selected_indices]
|
| 98 |
+
|
| 99 |
+
while len(selected_indices) < num_samples:
|
| 100 |
+
# Find the farthest point from the current set of selected points
|
| 101 |
+
farthest_point = torch.argmax(check_distances)
|
| 102 |
+
selected_indices.append(farthest_point.item())
|
| 103 |
+
|
| 104 |
+
check_distances = distance_matrix[farthest_point]
|
| 105 |
+
# Mark already selected points to avoid selecting them again
|
| 106 |
+
check_distances[selected_indices] = 0
|
| 107 |
+
|
| 108 |
+
# Break if all points have been selected
|
| 109 |
+
if len(selected_indices) == N:
|
| 110 |
+
break
|
| 111 |
+
|
| 112 |
+
return selected_indices
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def calculate_index_mappings(query_index, S, device=None):
|
| 116 |
+
"""
|
| 117 |
+
Construct an order that switches [query_index] and [0]
|
| 118 |
+
so that the content of query_index would be placed at [0].
|
| 119 |
+
|
| 120 |
+
Args:
|
| 121 |
+
query_index: Index to swap with 0
|
| 122 |
+
S: Total number of elements
|
| 123 |
+
device: Device to place the tensor on
|
| 124 |
+
|
| 125 |
+
Returns:
|
| 126 |
+
Tensor of indices with the swapped order
|
| 127 |
+
"""
|
| 128 |
+
new_order = torch.arange(S)
|
| 129 |
+
new_order[0] = query_index
|
| 130 |
+
new_order[query_index] = 0
|
| 131 |
+
if device is not None:
|
| 132 |
+
new_order = new_order.to(device)
|
| 133 |
+
return new_order
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def switch_tensor_order(tensors, order, dim=1):
|
| 137 |
+
"""
|
| 138 |
+
Reorder tensors along a specific dimension according to the given order.
|
| 139 |
+
|
| 140 |
+
Args:
|
| 141 |
+
tensors: List of tensors to reorder
|
| 142 |
+
order: Tensor of indices specifying the new order
|
| 143 |
+
dim: Dimension along which to reorder
|
| 144 |
+
|
| 145 |
+
Returns:
|
| 146 |
+
List of reordered tensors
|
| 147 |
+
"""
|
| 148 |
+
return [
|
| 149 |
+
torch.index_select(tensor, dim, order) if tensor is not None else None
|
| 150 |
+
for tensor in tensors
|
| 151 |
+
]
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def predict_track(model, images, query_points, dtype=torch.bfloat16, use_tf32_for_track=True, iters=4):
|
| 155 |
+
"""
|
| 156 |
+
Predict tracks for query points across frames.
|
| 157 |
+
|
| 158 |
+
Args:
|
| 159 |
+
model: VGGT model
|
| 160 |
+
images: Tensor of images of shape (S, 3, H, W)
|
| 161 |
+
query_points: Query points to track
|
| 162 |
+
dtype: Data type for computation
|
| 163 |
+
use_tf32_for_track: Whether to use TF32 precision for tracking
|
| 164 |
+
iters: Number of iterations for tracking
|
| 165 |
+
|
| 166 |
+
Returns:
|
| 167 |
+
Predicted tracks, visibility scores, and confidence scores
|
| 168 |
+
"""
|
| 169 |
+
with torch.no_grad():
|
| 170 |
+
with torch.cuda.amp.autocast(dtype=dtype):
|
| 171 |
+
images = images[None] # add batch dimension
|
| 172 |
+
aggregated_tokens_list, ps_idx = model.aggregator(images)
|
| 173 |
+
|
| 174 |
+
if not use_tf32_for_track:
|
| 175 |
+
track_list, vis_score, conf_score = model.track_head(
|
| 176 |
+
aggregated_tokens_list, images, ps_idx, query_points=query_points, iters=iters
|
| 177 |
+
)
|
| 178 |
+
|
| 179 |
+
if use_tf32_for_track:
|
| 180 |
+
with torch.cuda.amp.autocast(enabled=False):
|
| 181 |
+
track_list, vis_score, conf_score = model.track_head(
|
| 182 |
+
aggregated_tokens_list, images, ps_idx, query_points=query_points, iters=iters
|
| 183 |
+
)
|
| 184 |
+
|
| 185 |
+
pred_track = track_list[-1]
|
| 186 |
+
|
| 187 |
+
return pred_track.squeeze(0), vis_score.squeeze(0), conf_score.squeeze(0)
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def initialize_feature_extractors(max_query_num, det_thres, extractor_method="aliked", device="cuda"):
|
| 191 |
+
"""
|
| 192 |
+
Initialize feature extractors that can be reused based on a method string.
|
| 193 |
+
|
| 194 |
+
Args:
|
| 195 |
+
max_query_num: Maximum number of keypoints to extract
|
| 196 |
+
det_thres: Detection threshold for keypoint extraction
|
| 197 |
+
extractor_method: String specifying which extractors to use (e.g., "aliked", "sp+sift", "aliked+sp+sift")
|
| 198 |
+
device: Device to run extraction on
|
| 199 |
+
|
| 200 |
+
Returns:
|
| 201 |
+
Dictionary of initialized extractors
|
| 202 |
+
"""
|
| 203 |
+
extractors = {}
|
| 204 |
+
methods = extractor_method.lower().split('+')
|
| 205 |
+
active_extractors = len(methods)
|
| 206 |
+
|
| 207 |
+
for method in methods:
|
| 208 |
+
method = method.strip()
|
| 209 |
+
if method == "aliked":
|
| 210 |
+
aliked_max_points = max_query_num // active_extractors
|
| 211 |
+
aliked_extractor = ALIKED(max_num_keypoints=aliked_max_points, detection_threshold=det_thres)
|
| 212 |
+
extractors['aliked'] = aliked_extractor.to(device).eval()
|
| 213 |
+
elif method == "sp":
|
| 214 |
+
sp_max_points = max_query_num // active_extractors
|
| 215 |
+
sp_extractor = SuperPoint(max_num_keypoints=sp_max_points, detection_threshold=det_thres)
|
| 216 |
+
extractors['sp'] = sp_extractor.to(device).eval()
|
| 217 |
+
elif method == "sift":
|
| 218 |
+
sift_max_points = max_query_num // active_extractors
|
| 219 |
+
sift_extractor = SIFT(max_num_keypoints=sift_max_points)
|
| 220 |
+
extractors['sift'] = sift_extractor.to(device).eval()
|
| 221 |
+
else:
|
| 222 |
+
print(f"Warning: Unknown feature extractor '{method}', ignoring.")
|
| 223 |
+
|
| 224 |
+
if not extractors:
|
| 225 |
+
print(f"Warning: No valid extractors found in '{extractor_method}'. Using ALIKED by default.")
|
| 226 |
+
aliked_extractor = ALIKED(max_num_keypoints=max_query_num, detection_threshold=det_thres)
|
| 227 |
+
extractors['aliked'] = aliked_extractor.to(device).eval()
|
| 228 |
+
|
| 229 |
+
return extractors
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def extract_keypoints(query_image, extractors):
|
| 233 |
+
"""
|
| 234 |
+
Extract keypoints using pre-initialized feature extractors.
|
| 235 |
+
|
| 236 |
+
Args:
|
| 237 |
+
query_image: Input image tensor (3xHxW, range [0, 1])
|
| 238 |
+
extractors: Dictionary of initialized extractors
|
| 239 |
+
|
| 240 |
+
Returns:
|
| 241 |
+
Tensor of keypoint coordinates (1xNx2)
|
| 242 |
+
"""
|
| 243 |
+
query_points_round = None
|
| 244 |
+
|
| 245 |
+
with torch.no_grad():
|
| 246 |
+
for extractor_name, extractor in extractors.items():
|
| 247 |
+
query_points_data = extractor.extract(query_image)
|
| 248 |
+
extractor_points = query_points_data["keypoints"].round()
|
| 249 |
+
|
| 250 |
+
if query_points_round is not None:
|
| 251 |
+
query_points_round = torch.cat([query_points_round, extractor_points], dim=1)
|
| 252 |
+
else:
|
| 253 |
+
query_points_round = extractor_points
|
| 254 |
+
|
| 255 |
+
return query_points_round
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
def run_vggt_with_ba(model, images, image_names=None, dtype=torch.bfloat16,
|
| 259 |
+
max_query_num=2048, det_thres=0.005, query_frame_num=3,
|
| 260 |
+
extractor_method="aliked+sp+sift",
|
| 261 |
+
max_reproj_error=4,
|
| 262 |
+
shared_camera=True,
|
| 263 |
+
):
|
| 264 |
+
"""
|
| 265 |
+
Run VGGT with bundle adjustment for pose estimation.
|
| 266 |
+
|
| 267 |
+
Args:
|
| 268 |
+
model: VGGT model
|
| 269 |
+
images: Tensor of images of shape (S, 3, H, W)
|
| 270 |
+
image_names: Optional list of image names
|
| 271 |
+
dtype: Data type for computation
|
| 272 |
+
|
| 273 |
+
Returns:
|
| 274 |
+
Predicted extrinsic camera parameters
|
| 275 |
+
|
| 276 |
+
TODO:
|
| 277 |
+
- [ ] Use VGGT's vit instead of dinov2 for rank generation
|
| 278 |
+
"""
|
| 279 |
+
device = images.device
|
| 280 |
+
frame_num = images.shape[0]
|
| 281 |
+
|
| 282 |
+
# TODO: use vggt's vit instead of dinov2
|
| 283 |
+
# Select representative frames for feature extraction
|
| 284 |
+
query_frame_indexes = generate_rank_by_dino(
|
| 285 |
+
images, query_frame_num, image_size=518,
|
| 286 |
+
model_name="dinov2_vitb14_reg", device=device,
|
| 287 |
+
spatial_similarity=False
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
# Add the first image to the front if not already present
|
| 291 |
+
if 0 in query_frame_indexes:
|
| 292 |
+
query_frame_indexes.remove(0)
|
| 293 |
+
query_frame_indexes = [0, *query_frame_indexes]
|
| 294 |
+
|
| 295 |
+
# Get initial pose and depth predictions
|
| 296 |
+
|
| 297 |
+
with torch.no_grad(), torch.cuda.amp.autocast(enabled=False, dtype=torch.bfloat16):
|
| 298 |
+
aggregated_tokens_list, patch_start_idx = model.aggregator(images, intermediate_layer_idx=model.cfg.intermediate_layer_idx)
|
| 299 |
+
with torch.cuda.amp.autocast(enabled=False):
|
| 300 |
+
fp32_tokens = [token.float() for token in aggregated_tokens_list]
|
| 301 |
+
pred_all_pose_enc = model.camera_head(fp32_tokens)[-1]
|
| 302 |
+
pred_all_extrinsic, pred_all_intrinsic = pose_encoding_to_extri_intri(pred_all_pose_enc, images.shape[-2:])
|
| 303 |
+
pred_extrinsic = pred_all_extrinsic[0]
|
| 304 |
+
pred_intrinsic = pred_all_intrinsic[0]
|
| 305 |
+
depth_map, depth_conf = model.depth_head(aggregated_tokens_list, images=images, patch_start_idx=patch_start_idx)
|
| 306 |
+
|
| 307 |
+
world_points = unproject_depth_map_to_point_map(depth_map, pred_extrinsic, pred_intrinsic)
|
| 308 |
+
world_points = torch.from_numpy(world_points).to(device)
|
| 309 |
+
world_points_conf = depth_conf.to(device)
|
| 310 |
+
|
| 311 |
+
torch.cuda.empty_cache()
|
| 312 |
+
|
| 313 |
+
# Lists to store predictions
|
| 314 |
+
pred_tracks = []
|
| 315 |
+
pred_vis_scores = []
|
| 316 |
+
pred_conf_scores = []
|
| 317 |
+
pred_world_points = []
|
| 318 |
+
pred_world_points_conf = []
|
| 319 |
+
|
| 320 |
+
# Initialize feature extractors
|
| 321 |
+
extractors = initialize_feature_extractors(max_query_num, det_thres, extractor_method, device)
|
| 322 |
+
|
| 323 |
+
# Process each query frame
|
| 324 |
+
for query_index in query_frame_indexes:
|
| 325 |
+
query_image = images[query_index]
|
| 326 |
+
query_points_round = extract_keypoints(query_image, extractors)
|
| 327 |
+
|
| 328 |
+
# Reorder images to put query image first
|
| 329 |
+
reorder_index = calculate_index_mappings(query_index, frame_num, device=device)
|
| 330 |
+
reorder_images = switch_tensor_order([images], reorder_index, dim=0)[0]
|
| 331 |
+
|
| 332 |
+
# Track points across frames
|
| 333 |
+
reorder_tracks, reorder_vis_score, reorder_conf_score = predict_track(
|
| 334 |
+
model, reorder_images, query_points_round, dtype=dtype, use_tf32_for_track=True, iters=4
|
| 335 |
+
)
|
| 336 |
+
|
| 337 |
+
# Restore original order
|
| 338 |
+
pred_track, pred_vis, pred_score = switch_tensor_order(
|
| 339 |
+
[reorder_tracks, reorder_vis_score, reorder_conf_score], reorder_index, dim=0
|
| 340 |
+
)
|
| 341 |
+
|
| 342 |
+
pred_tracks.append(pred_track)
|
| 343 |
+
pred_vis_scores.append(pred_vis)
|
| 344 |
+
pred_conf_scores.append(pred_score)
|
| 345 |
+
|
| 346 |
+
# Get corresponding 3D points
|
| 347 |
+
query_points_round_long = query_points_round.squeeze(0).long()
|
| 348 |
+
query_world_points = world_points[query_index][
|
| 349 |
+
query_points_round_long[:, 1], query_points_round_long[:, 0]
|
| 350 |
+
]
|
| 351 |
+
query_world_points_conf = world_points_conf[query_index][
|
| 352 |
+
query_points_round_long[:, 1], query_points_round_long[:, 0]
|
| 353 |
+
]
|
| 354 |
+
|
| 355 |
+
pred_world_points.append(query_world_points)
|
| 356 |
+
pred_world_points_conf.append(query_world_points_conf)
|
| 357 |
+
|
| 358 |
+
# Concatenate prediction lists
|
| 359 |
+
pred_tracks = torch.cat(pred_tracks, dim=1)
|
| 360 |
+
pred_vis_scores = torch.cat(pred_vis_scores, dim=1)
|
| 361 |
+
pred_conf_scores = torch.cat(pred_conf_scores, dim=1)
|
| 362 |
+
pred_world_points = torch.cat(pred_world_points, dim=0)
|
| 363 |
+
pred_world_points_conf = torch.cat(pred_world_points_conf, dim=0)
|
| 364 |
+
|
| 365 |
+
# Filter points by confidence
|
| 366 |
+
filtered_flag = pred_world_points_conf > 1.5
|
| 367 |
+
|
| 368 |
+
if filtered_flag.sum() > 1024:
|
| 369 |
+
# well if the number of points is too small, we will not filter
|
| 370 |
+
pred_world_points = pred_world_points[filtered_flag]
|
| 371 |
+
pred_world_points_conf = pred_world_points_conf[filtered_flag]
|
| 372 |
+
|
| 373 |
+
pred_tracks = pred_tracks[:, filtered_flag]
|
| 374 |
+
pred_vis_scores = pred_vis_scores[:, filtered_flag]
|
| 375 |
+
pred_conf_scores = pred_conf_scores[:, filtered_flag]
|
| 376 |
+
|
| 377 |
+
torch.cuda.empty_cache()
|
| 378 |
+
|
| 379 |
+
# Bundle adjustment parameters
|
| 380 |
+
S, _, H, W = images.shape
|
| 381 |
+
image_size = torch.tensor([W, H], dtype=pred_tracks.dtype, device=device)
|
| 382 |
+
|
| 383 |
+
# Run bundle adjustment
|
| 384 |
+
reconstruction = batch_matrix_to_pycolmap(
|
| 385 |
+
pred_world_points,
|
| 386 |
+
pred_extrinsic,
|
| 387 |
+
pred_intrinsic,
|
| 388 |
+
pred_tracks,
|
| 389 |
+
image_size,
|
| 390 |
+
max_reproj_error=max_reproj_error,
|
| 391 |
+
shared_camera=shared_camera
|
| 392 |
+
)
|
| 393 |
+
|
| 394 |
+
ba_options = pycolmap.BundleAdjustmentOptions()
|
| 395 |
+
pycolmap.bundle_adjustment(reconstruction, ba_options)
|
| 396 |
+
_, updated_extrinsic, _, _ = pycolmap_to_batch_matrix(
|
| 397 |
+
reconstruction, device=device, camera_type="SIMPLE_PINHOLE"
|
| 398 |
+
)
|
| 399 |
+
|
| 400 |
+
return updated_extrinsic
|
| 401 |
+
|
| 402 |
+
|
| 403 |
+
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
|
| 407 |
+
|
src/utils/cropping.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (C) 2024-present Naver Corporation. All rights reserved.
|
| 2 |
+
# Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
|
| 3 |
+
#
|
| 4 |
+
# --------------------------------------------------------
|
| 5 |
+
# croppping utilities
|
| 6 |
+
# --------------------------------------------------------
|
| 7 |
+
import PIL.Image
|
| 8 |
+
import os
|
| 9 |
+
os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1"
|
| 10 |
+
import cv2 # noqa
|
| 11 |
+
import numpy as np # noqa
|
| 12 |
+
from src.utils.geometry import colmap_to_opencv_intrinsics, opencv_to_colmap_intrinsics # noqa
|
| 13 |
+
try:
|
| 14 |
+
lanczos = PIL.Image.Resampling.LANCZOS
|
| 15 |
+
bicubic = PIL.Image.Resampling.BICUBIC
|
| 16 |
+
except AttributeError:
|
| 17 |
+
lanczos = PIL.Image.LANCZOS
|
| 18 |
+
bicubic = PIL.Image.BICUBIC
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class ImageList:
|
| 22 |
+
""" Convenience class to aply the same operation to a whole set of images.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
def __init__(self, images):
|
| 26 |
+
if not isinstance(images, (tuple, list, set)):
|
| 27 |
+
images = [images]
|
| 28 |
+
self.images = []
|
| 29 |
+
for image in images:
|
| 30 |
+
if not isinstance(image, PIL.Image.Image):
|
| 31 |
+
image = PIL.Image.fromarray(image)
|
| 32 |
+
self.images.append(image)
|
| 33 |
+
|
| 34 |
+
def __len__(self):
|
| 35 |
+
return len(self.images)
|
| 36 |
+
|
| 37 |
+
def to_pil(self):
|
| 38 |
+
return tuple(self.images) if len(self.images) > 1 else self.images[0]
|
| 39 |
+
|
| 40 |
+
@property
|
| 41 |
+
def size(self):
|
| 42 |
+
sizes = [im.size for im in self.images]
|
| 43 |
+
assert all(sizes[0] == s for s in sizes)
|
| 44 |
+
return sizes[0]
|
| 45 |
+
|
| 46 |
+
def resize(self, *args, **kwargs):
|
| 47 |
+
return ImageList(self._dispatch('resize', *args, **kwargs))
|
| 48 |
+
|
| 49 |
+
def crop(self, *args, **kwargs):
|
| 50 |
+
return ImageList(self._dispatch('crop', *args, **kwargs))
|
| 51 |
+
|
| 52 |
+
def _dispatch(self, func, *args, **kwargs):
|
| 53 |
+
return [getattr(im, func)(*args, **kwargs) for im in self.images]
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def rescale_image(image, camera_intrinsics, output_resolution, force=True):
|
| 57 |
+
""" Jointly rescale a (image, depthmap)
|
| 58 |
+
so that (out_width, out_height) >= output_res
|
| 59 |
+
"""
|
| 60 |
+
image = ImageList(image)
|
| 61 |
+
input_resolution = np.array(image.size) # (W,H)
|
| 62 |
+
output_resolution = np.array(output_resolution)
|
| 63 |
+
# define output resolution
|
| 64 |
+
assert output_resolution.shape == (2,)
|
| 65 |
+
scale_final = max(output_resolution / image.size) + 1e-8
|
| 66 |
+
if scale_final >= 1 and not force: # image is already smaller than what is asked
|
| 67 |
+
return (image.to_pil(), camera_intrinsics)
|
| 68 |
+
output_resolution = np.floor(input_resolution * scale_final).astype(int)
|
| 69 |
+
|
| 70 |
+
# first rescale the image so that it contains the crop
|
| 71 |
+
image = image.resize(tuple(output_resolution), resample=lanczos if scale_final < 1 else bicubic)
|
| 72 |
+
|
| 73 |
+
# no offset here; simple rescaling
|
| 74 |
+
camera_intrinsics = camera_matrix_of_crop(
|
| 75 |
+
camera_intrinsics, input_resolution, output_resolution, scaling=scale_final)
|
| 76 |
+
|
| 77 |
+
return image.to_pil(), camera_intrinsics
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def camera_matrix_of_crop(input_camera_matrix, input_resolution, output_resolution, scaling=1, offset_factor=0.5, offset=None):
|
| 81 |
+
# Margins to offset the origin
|
| 82 |
+
margins = np.asarray(input_resolution) * scaling - output_resolution
|
| 83 |
+
assert np.all(margins >= 0.0)
|
| 84 |
+
if offset is None:
|
| 85 |
+
offset = offset_factor * margins
|
| 86 |
+
|
| 87 |
+
# Generate new camera parameters
|
| 88 |
+
output_camera_matrix_colmap = opencv_to_colmap_intrinsics(input_camera_matrix)
|
| 89 |
+
output_camera_matrix_colmap[:2, :] *= scaling
|
| 90 |
+
output_camera_matrix_colmap[:2, 2] -= offset
|
| 91 |
+
output_camera_matrix = colmap_to_opencv_intrinsics(output_camera_matrix_colmap)
|
| 92 |
+
|
| 93 |
+
return output_camera_matrix
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def crop_image(image, camera_intrinsics, crop_bbox):
|
| 97 |
+
"""
|
| 98 |
+
Return a crop of the input view.
|
| 99 |
+
"""
|
| 100 |
+
image = ImageList(image)
|
| 101 |
+
l, t, r, b = crop_bbox
|
| 102 |
+
|
| 103 |
+
image = image.crop((l, t, r, b))
|
| 104 |
+
|
| 105 |
+
camera_intrinsics = camera_intrinsics.copy()
|
| 106 |
+
camera_intrinsics[0, 2] -= l
|
| 107 |
+
camera_intrinsics[1, 2] -= t
|
| 108 |
+
|
| 109 |
+
return image.to_pil(), camera_intrinsics
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def bbox_from_intrinsics_in_out(input_camera_matrix, output_camera_matrix, output_resolution):
|
| 113 |
+
out_width, out_height = output_resolution
|
| 114 |
+
l, t = np.int32(np.round(input_camera_matrix[:2, 2] - output_camera_matrix[:2, 2]))
|
| 115 |
+
crop_bbox = (l, t, l + out_width, t + out_height)
|
| 116 |
+
return crop_bbox
|
| 117 |
+
|
| 118 |
+
def rescale_image_depthmap(image, depthmap, camera_intrinsics, output_resolution, force=True):
|
| 119 |
+
""" Jointly rescale a (image, depthmap)
|
| 120 |
+
so that (out_width, out_height) >= output_res
|
| 121 |
+
"""
|
| 122 |
+
image = ImageList(image)
|
| 123 |
+
input_resolution = np.array(image.size) # (W,H)
|
| 124 |
+
output_resolution = np.array(output_resolution)
|
| 125 |
+
if depthmap is not None:
|
| 126 |
+
# can also use this with masks instead of depthmaps
|
| 127 |
+
assert tuple(depthmap.shape[:2]) == image.size[::-1]
|
| 128 |
+
|
| 129 |
+
# define output resolution
|
| 130 |
+
assert output_resolution.shape == (2,)
|
| 131 |
+
scale_final = max(output_resolution / image.size) + 1e-8
|
| 132 |
+
if scale_final >= 1 and not force: # image is already smaller than what is asked
|
| 133 |
+
return (image.to_pil(), depthmap, camera_intrinsics)
|
| 134 |
+
output_resolution = np.floor(input_resolution * scale_final).astype(int)
|
| 135 |
+
|
| 136 |
+
# first rescale the image so that it contains the crop
|
| 137 |
+
# breakpoint()
|
| 138 |
+
image = image.resize(tuple(output_resolution), resample=lanczos if scale_final < 1 else bicubic)
|
| 139 |
+
if depthmap is not None:
|
| 140 |
+
depthmap = cv2.resize(depthmap, tuple(output_resolution), fx=scale_final,
|
| 141 |
+
fy=scale_final, interpolation=cv2.INTER_NEAREST)
|
| 142 |
+
|
| 143 |
+
# no offset here; simple rescaling
|
| 144 |
+
camera_intrinsics = camera_matrix_of_crop(
|
| 145 |
+
camera_intrinsics, input_resolution, tuple(output_resolution), scaling=scale_final)
|
| 146 |
+
|
| 147 |
+
return image.to_pil(), depthmap, camera_intrinsics
|
| 148 |
+
|
| 149 |
+
def crop_image_depthmap(image, depthmap, camera_intrinsics, crop_bbox):
|
| 150 |
+
"""
|
| 151 |
+
Return a crop of the input view.
|
| 152 |
+
"""
|
| 153 |
+
image = ImageList(image)
|
| 154 |
+
l, t, r, b = crop_bbox
|
| 155 |
+
|
| 156 |
+
image = image.crop((l, t, r, b))
|
| 157 |
+
depthmap = depthmap[t:b, l:r]
|
| 158 |
+
|
| 159 |
+
camera_intrinsics = camera_intrinsics.copy()
|
| 160 |
+
camera_intrinsics[0, 2] -= l
|
| 161 |
+
camera_intrinsics[1, 2] -= t
|
| 162 |
+
|
| 163 |
+
return image.to_pil(), depthmap, camera_intrinsics
|
src/utils/device.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
|
| 4 |
+
# This source code is licensed under the license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
# Copyright (C) 2024-present Naver Corporation. All rights reserved.
|
| 8 |
+
# Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
|
| 9 |
+
#
|
| 10 |
+
# --------------------------------------------------------
|
| 11 |
+
# utilitary functions for DUSt3R
|
| 12 |
+
# --------------------------------------------------------
|
| 13 |
+
import numpy as np
|
| 14 |
+
import torch
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def todevice(batch, device, callback=None, non_blocking=False):
|
| 18 |
+
"""Transfer some variables to another device (i.e. GPU, CPU:torch, CPU:numpy).
|
| 19 |
+
|
| 20 |
+
batch: list, tuple, dict of tensors or other things
|
| 21 |
+
device: pytorch device or 'numpy'
|
| 22 |
+
callback: function that would be called on every sub-elements.
|
| 23 |
+
"""
|
| 24 |
+
if callback:
|
| 25 |
+
batch = callback(batch)
|
| 26 |
+
|
| 27 |
+
if isinstance(batch, dict):
|
| 28 |
+
return {k: todevice(v, device) for k, v in batch.items()}
|
| 29 |
+
|
| 30 |
+
if isinstance(batch, (tuple, list)):
|
| 31 |
+
return type(batch)(todevice(x, device) for x in batch)
|
| 32 |
+
|
| 33 |
+
x = batch
|
| 34 |
+
if device == "numpy":
|
| 35 |
+
if isinstance(x, torch.Tensor):
|
| 36 |
+
x = x.detach().cpu().numpy()
|
| 37 |
+
elif x is not None:
|
| 38 |
+
if isinstance(x, np.ndarray):
|
| 39 |
+
x = torch.from_numpy(x)
|
| 40 |
+
if torch.is_tensor(x):
|
| 41 |
+
x = x.to(device, non_blocking=non_blocking)
|
| 42 |
+
return x
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
to_device = todevice # alias
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def to_numpy(x):
|
| 49 |
+
return todevice(x, "numpy")
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def to_cpu(x):
|
| 53 |
+
return todevice(x, "cpu")
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def to_cuda(x):
|
| 57 |
+
return todevice(x, "cuda")
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def collate_with_cat(whatever, lists=False):
|
| 61 |
+
if isinstance(whatever, dict):
|
| 62 |
+
return {k: collate_with_cat(vals, lists=lists) for k, vals in whatever.items()}
|
| 63 |
+
|
| 64 |
+
elif isinstance(whatever, (tuple, list)):
|
| 65 |
+
if len(whatever) == 0:
|
| 66 |
+
return whatever
|
| 67 |
+
elem = whatever[0]
|
| 68 |
+
T = type(whatever)
|
| 69 |
+
|
| 70 |
+
if elem is None:
|
| 71 |
+
return None
|
| 72 |
+
if isinstance(elem, (bool, float, int, str)):
|
| 73 |
+
return whatever
|
| 74 |
+
if isinstance(elem, tuple):
|
| 75 |
+
return T(collate_with_cat(x, lists=lists) for x in zip(*whatever))
|
| 76 |
+
if isinstance(elem, dict):
|
| 77 |
+
return {
|
| 78 |
+
k: collate_with_cat([e[k] for e in whatever], lists=lists) for k in elem
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
if isinstance(elem, torch.Tensor):
|
| 82 |
+
return listify(whatever) if lists else torch.cat(whatever)
|
| 83 |
+
if isinstance(elem, np.ndarray):
|
| 84 |
+
return (
|
| 85 |
+
listify(whatever)
|
| 86 |
+
if lists
|
| 87 |
+
else torch.cat([torch.from_numpy(x) for x in whatever])
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
# otherwise, we just chain lists
|
| 91 |
+
return sum(whatever, T())
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def listify(elems):
|
| 95 |
+
return [x for e in elems for x in e]
|
src/utils/geometry.py
ADDED
|
@@ -0,0 +1,428 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (C) 2024-present Naver Corporation. All rights reserved.
|
| 2 |
+
# Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
|
| 3 |
+
#
|
| 4 |
+
# --------------------------------------------------------
|
| 5 |
+
# geometry utilitary functions
|
| 6 |
+
# --------------------------------------------------------
|
| 7 |
+
import torch
|
| 8 |
+
import numpy as np
|
| 9 |
+
from scipy.spatial import cKDTree as KDTree
|
| 10 |
+
|
| 11 |
+
def invalid_to_nans(arr, valid_mask, ndim=999):
|
| 12 |
+
if valid_mask is not None:
|
| 13 |
+
arr = arr.clone()
|
| 14 |
+
arr[~valid_mask] = float('nan')
|
| 15 |
+
if arr.ndim > ndim:
|
| 16 |
+
arr = arr.flatten(-2 - (arr.ndim - ndim), -2)
|
| 17 |
+
return arr
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def invalid_to_zeros(arr, valid_mask, ndim=999):
|
| 21 |
+
if valid_mask is not None:
|
| 22 |
+
arr = arr.clone()
|
| 23 |
+
arr[~valid_mask] = 0
|
| 24 |
+
nnz = valid_mask.view(len(valid_mask), -1).sum(1)
|
| 25 |
+
else:
|
| 26 |
+
nnz = arr.numel() // len(arr) if len(arr) else 0 # number of point per image
|
| 27 |
+
if arr.ndim > ndim:
|
| 28 |
+
arr = arr.flatten(-2 - (arr.ndim - ndim), -2)
|
| 29 |
+
return arr, nnz
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def xy_grid(W, H, device=None, origin=(0, 0), unsqueeze=None, cat_dim=-1, homogeneous=False, **arange_kw):
|
| 33 |
+
""" Output a (H,W,2) array of int32
|
| 34 |
+
with output[j,i,0] = i + origin[0]
|
| 35 |
+
output[j,i,1] = j + origin[1]
|
| 36 |
+
"""
|
| 37 |
+
if device is None:
|
| 38 |
+
# numpy
|
| 39 |
+
arange, meshgrid, stack, ones = np.arange, np.meshgrid, np.stack, np.ones
|
| 40 |
+
else:
|
| 41 |
+
# torch
|
| 42 |
+
arange = lambda *a, **kw: torch.arange(*a, device=device, **kw)
|
| 43 |
+
meshgrid, stack = torch.meshgrid, torch.stack
|
| 44 |
+
ones = lambda *a: torch.ones(*a, device=device)
|
| 45 |
+
|
| 46 |
+
tw, th = [arange(o, o + s, **arange_kw) for s, o in zip((W, H), origin)]
|
| 47 |
+
grid = meshgrid(tw, th, indexing='xy')
|
| 48 |
+
if homogeneous:
|
| 49 |
+
grid = grid + (ones((H, W)),)
|
| 50 |
+
if unsqueeze is not None:
|
| 51 |
+
grid = (grid[0].unsqueeze(unsqueeze), grid[1].unsqueeze(unsqueeze))
|
| 52 |
+
if cat_dim is not None:
|
| 53 |
+
grid = stack(grid, cat_dim)
|
| 54 |
+
return grid
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def geotrf(Trf, pts, ncol=None, norm=False):
|
| 58 |
+
""" Apply a geometric transformation to a list of 3-D points.
|
| 59 |
+
|
| 60 |
+
H: 3x3 or 4x4 projection matrix (typically a Homography)
|
| 61 |
+
p: numpy/torch/tuple of coordinates. Shape must be (...,2) or (...,3)
|
| 62 |
+
|
| 63 |
+
ncol: int. number of columns of the result (2 or 3)
|
| 64 |
+
norm: float. if != 0, the resut is projected on the z=norm plane.
|
| 65 |
+
|
| 66 |
+
Returns an array of projected 2d points.
|
| 67 |
+
"""
|
| 68 |
+
assert Trf.ndim >= 2
|
| 69 |
+
if isinstance(Trf, np.ndarray):
|
| 70 |
+
pts = np.asarray(pts)
|
| 71 |
+
elif isinstance(Trf, torch.Tensor):
|
| 72 |
+
pts = torch.as_tensor(pts, dtype=Trf.dtype)
|
| 73 |
+
|
| 74 |
+
# adapt shape if necessary
|
| 75 |
+
output_reshape = pts.shape[:-1]
|
| 76 |
+
ncol = ncol or pts.shape[-1]
|
| 77 |
+
|
| 78 |
+
# optimized code
|
| 79 |
+
if (isinstance(Trf, torch.Tensor) and isinstance(pts, torch.Tensor) and
|
| 80 |
+
Trf.ndim == 3 and pts.ndim == 4):
|
| 81 |
+
d = pts.shape[3]
|
| 82 |
+
if Trf.shape[-1] == d:
|
| 83 |
+
pts = torch.einsum("bij, bhwj -> bhwi", Trf, pts)
|
| 84 |
+
elif Trf.shape[-1] == d + 1:
|
| 85 |
+
pts = torch.einsum("bij, bhwj -> bhwi", Trf[:, :d, :d], pts) + Trf[:, None, None, :d, d]
|
| 86 |
+
else:
|
| 87 |
+
raise ValueError(f'bad shape, not ending with 3 or 4, for {pts.shape=}')
|
| 88 |
+
else:
|
| 89 |
+
if Trf.ndim >= 3:
|
| 90 |
+
n = Trf.ndim - 2
|
| 91 |
+
assert Trf.shape[:n] == pts.shape[:n], 'batch size does not match'
|
| 92 |
+
Trf = Trf.reshape(-1, Trf.shape[-2], Trf.shape[-1])
|
| 93 |
+
|
| 94 |
+
if pts.ndim > Trf.ndim:
|
| 95 |
+
# Trf == (B,d,d) & pts == (B,H,W,d) --> (B, H*W, d)
|
| 96 |
+
pts = pts.reshape(Trf.shape[0], -1, pts.shape[-1])
|
| 97 |
+
elif pts.ndim == 2:
|
| 98 |
+
# Trf == (B,d,d) & pts == (B,d) --> (B, 1, d)
|
| 99 |
+
pts = pts[:, None, :]
|
| 100 |
+
|
| 101 |
+
if pts.shape[-1] + 1 == Trf.shape[-1]:
|
| 102 |
+
Trf = Trf.swapaxes(-1, -2) # transpose Trf
|
| 103 |
+
pts = pts @ Trf[..., :-1, :] + Trf[..., -1:, :]
|
| 104 |
+
elif pts.shape[-1] == Trf.shape[-1]:
|
| 105 |
+
Trf = Trf.swapaxes(-1, -2) # transpose Trf
|
| 106 |
+
pts = pts @ Trf
|
| 107 |
+
else:
|
| 108 |
+
pts = Trf @ pts.T
|
| 109 |
+
if pts.ndim >= 2:
|
| 110 |
+
pts = pts.swapaxes(-1, -2)
|
| 111 |
+
|
| 112 |
+
if norm:
|
| 113 |
+
pts = pts / pts[..., -1:] # DONT DO /= BECAUSE OF WEIRD PYTORCH BUG
|
| 114 |
+
if norm != 1:
|
| 115 |
+
pts *= norm
|
| 116 |
+
|
| 117 |
+
res = pts[..., :ncol].reshape(*output_reshape, ncol)
|
| 118 |
+
return res
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def inv(mat):
|
| 122 |
+
""" Invert a torch or numpy matrix
|
| 123 |
+
"""
|
| 124 |
+
if isinstance(mat, torch.Tensor):
|
| 125 |
+
return torch.linalg.inv(mat)
|
| 126 |
+
if isinstance(mat, np.ndarray):
|
| 127 |
+
return np.linalg.inv(mat)
|
| 128 |
+
raise ValueError(f'bad matrix type = {type(mat)}')
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def depthmap_to_pts3d(depth, pseudo_focal, pp=None, **_):
|
| 132 |
+
"""
|
| 133 |
+
Args:
|
| 134 |
+
- depthmap (BxHxW array):
|
| 135 |
+
- pseudo_focal: [B,H,W] ; [B,2,H,W] or [B,1,H,W]
|
| 136 |
+
Returns:
|
| 137 |
+
pointmap of absolute coordinates (BxHxWx3 array)
|
| 138 |
+
"""
|
| 139 |
+
|
| 140 |
+
if len(depth.shape) == 4:
|
| 141 |
+
B, H, W, n = depth.shape
|
| 142 |
+
else:
|
| 143 |
+
B, H, W = depth.shape
|
| 144 |
+
n = None
|
| 145 |
+
|
| 146 |
+
if len(pseudo_focal.shape) == 3: # [B,H,W]
|
| 147 |
+
pseudo_focalx = pseudo_focaly = pseudo_focal
|
| 148 |
+
elif len(pseudo_focal.shape) == 4: # [B,2,H,W] or [B,1,H,W]
|
| 149 |
+
pseudo_focalx = pseudo_focal[:, 0]
|
| 150 |
+
if pseudo_focal.shape[1] == 2:
|
| 151 |
+
pseudo_focaly = pseudo_focal[:, 1]
|
| 152 |
+
else:
|
| 153 |
+
pseudo_focaly = pseudo_focalx
|
| 154 |
+
else:
|
| 155 |
+
raise NotImplementedError("Error, unknown input focal shape format.")
|
| 156 |
+
|
| 157 |
+
assert pseudo_focalx.shape == depth.shape[:3]
|
| 158 |
+
assert pseudo_focaly.shape == depth.shape[:3]
|
| 159 |
+
grid_x, grid_y = xy_grid(W, H, cat_dim=0, device=depth.device)[:, None]
|
| 160 |
+
|
| 161 |
+
# set principal point
|
| 162 |
+
if pp is None:
|
| 163 |
+
grid_x = grid_x - (W - 1) / 2
|
| 164 |
+
grid_y = grid_y - (H - 1) / 2
|
| 165 |
+
else:
|
| 166 |
+
grid_x = grid_x.expand(B, -1, -1) - pp[:, 0, None, None]
|
| 167 |
+
grid_y = grid_y.expand(B, -1, -1) - pp[:, 1, None, None]
|
| 168 |
+
|
| 169 |
+
if n is None:
|
| 170 |
+
pts3d = torch.empty((B, H, W, 3), device=depth.device)
|
| 171 |
+
pts3d[..., 0] = depth * grid_x / pseudo_focalx
|
| 172 |
+
pts3d[..., 1] = depth * grid_y / pseudo_focaly
|
| 173 |
+
pts3d[..., 2] = depth
|
| 174 |
+
else:
|
| 175 |
+
pts3d = torch.empty((B, H, W, 3, n), device=depth.device)
|
| 176 |
+
pts3d[..., 0, :] = depth * (grid_x / pseudo_focalx)[..., None]
|
| 177 |
+
pts3d[..., 1, :] = depth * (grid_y / pseudo_focaly)[..., None]
|
| 178 |
+
pts3d[..., 2, :] = depth
|
| 179 |
+
return pts3d
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def depthmap_to_camera_coordinates(depthmap, camera_intrinsics, pseudo_focal=None):
|
| 183 |
+
"""
|
| 184 |
+
Args:
|
| 185 |
+
- depthmap (HxW array):
|
| 186 |
+
- camera_intrinsics: a 3x3 matrix
|
| 187 |
+
Returns:
|
| 188 |
+
pointmap of absolute coordinates (HxWx3 array), and a mask specifying valid pixels.
|
| 189 |
+
"""
|
| 190 |
+
camera_intrinsics = np.float32(camera_intrinsics)
|
| 191 |
+
H, W = depthmap.shape
|
| 192 |
+
|
| 193 |
+
# Compute 3D ray associated with each pixel
|
| 194 |
+
# Strong assumption: there are no skew terms
|
| 195 |
+
assert camera_intrinsics[0, 1] == 0.0
|
| 196 |
+
assert camera_intrinsics[1, 0] == 0.0
|
| 197 |
+
if pseudo_focal is None:
|
| 198 |
+
fu = camera_intrinsics[0, 0]
|
| 199 |
+
fv = camera_intrinsics[1, 1]
|
| 200 |
+
else:
|
| 201 |
+
assert pseudo_focal.shape == (H, W)
|
| 202 |
+
fu = fv = pseudo_focal
|
| 203 |
+
cu = camera_intrinsics[0, 2]
|
| 204 |
+
cv = camera_intrinsics[1, 2]
|
| 205 |
+
|
| 206 |
+
u, v = np.meshgrid(np.arange(W), np.arange(H))
|
| 207 |
+
z_cam = depthmap
|
| 208 |
+
x_cam = (u - cu) * z_cam / fu
|
| 209 |
+
y_cam = (v - cv) * z_cam / fv
|
| 210 |
+
X_cam = np.stack((x_cam, y_cam, z_cam), axis=-1).astype(np.float32)
|
| 211 |
+
|
| 212 |
+
# Mask for valid coordinates
|
| 213 |
+
valid_mask = (depthmap > 0.0)
|
| 214 |
+
return X_cam, valid_mask
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def depthmap_to_absolute_camera_coordinates(depthmap, camera_intrinsics, camera_pose=None, **kw):
|
| 218 |
+
"""
|
| 219 |
+
Args:
|
| 220 |
+
- depthmap (HxW array):
|
| 221 |
+
- camera_intrinsics: a 3x3 matrix
|
| 222 |
+
- camera_pose: a 4x3 or 4x4 cam2world matrix
|
| 223 |
+
Returns:
|
| 224 |
+
pointmap of absolute coordinates (HxWx3 array), and a mask specifying valid pixels."""
|
| 225 |
+
X_cam, valid_mask = depthmap_to_camera_coordinates(depthmap, camera_intrinsics)
|
| 226 |
+
|
| 227 |
+
X_world = X_cam # default
|
| 228 |
+
if camera_pose is not None:
|
| 229 |
+
# R_cam2world = np.float32(camera_params["R_cam2world"])
|
| 230 |
+
# t_cam2world = np.float32(camera_params["t_cam2world"]).squeeze()
|
| 231 |
+
R_cam2world = camera_pose[:3, :3]
|
| 232 |
+
t_cam2world = camera_pose[:3, 3]
|
| 233 |
+
|
| 234 |
+
# Express in absolute coordinates (invalid depth values)
|
| 235 |
+
X_world = np.einsum("ik, vuk -> vui", R_cam2world, X_cam) + t_cam2world[None, None, :]
|
| 236 |
+
|
| 237 |
+
return X_world, valid_mask
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def colmap_to_opencv_intrinsics(K):
|
| 241 |
+
"""
|
| 242 |
+
Modify camera intrinsics to follow a different convention.
|
| 243 |
+
Coordinates of the center of the top-left pixels are by default:
|
| 244 |
+
- (0.5, 0.5) in Colmap
|
| 245 |
+
- (0,0) in OpenCV
|
| 246 |
+
"""
|
| 247 |
+
K = K.copy()
|
| 248 |
+
K[0, 2] -= 0.5
|
| 249 |
+
K[1, 2] -= 0.5
|
| 250 |
+
return K
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def opencv_to_colmap_intrinsics(K):
|
| 254 |
+
"""
|
| 255 |
+
Modify camera intrinsics to follow a different convention.
|
| 256 |
+
Coordinates of the center of the top-left pixels are by default:
|
| 257 |
+
- (0.5, 0.5) in Colmap
|
| 258 |
+
- (0,0) in OpenCV
|
| 259 |
+
"""
|
| 260 |
+
K = K.copy()
|
| 261 |
+
K[0, 2] += 0.5
|
| 262 |
+
K[1, 2] += 0.5
|
| 263 |
+
return K
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
def normalize_pointcloud(pts1, pts2, norm_mode='avg_dis', valid1=None, valid2=None, ret_factor=False):
|
| 267 |
+
""" renorm pointmaps pts1, pts2 with norm_mode
|
| 268 |
+
"""
|
| 269 |
+
assert pts1.ndim >= 3 and pts1.shape[-1] == 3
|
| 270 |
+
assert pts2 is None or (pts2.ndim >= 3 and pts2.shape[-1] == 3)
|
| 271 |
+
norm_mode, dis_mode = norm_mode.split('_')
|
| 272 |
+
|
| 273 |
+
if norm_mode == 'avg':
|
| 274 |
+
# gather all points together (joint normalization)
|
| 275 |
+
nan_pts1, nnz1 = invalid_to_zeros(pts1, valid1, ndim=3)
|
| 276 |
+
nan_pts2, nnz2 = invalid_to_zeros(pts2, valid2, ndim=3) if pts2 is not None else (None, 0)
|
| 277 |
+
all_pts = torch.cat((nan_pts1, nan_pts2), dim=1) if pts2 is not None else nan_pts1
|
| 278 |
+
|
| 279 |
+
# compute distance to origin
|
| 280 |
+
all_dis = all_pts.norm(dim=-1)
|
| 281 |
+
if dis_mode == 'dis':
|
| 282 |
+
pass # do nothing
|
| 283 |
+
elif dis_mode == 'log1p':
|
| 284 |
+
all_dis = torch.log1p(all_dis)
|
| 285 |
+
elif dis_mode == 'warp-log1p':
|
| 286 |
+
# actually warp input points before normalizing them
|
| 287 |
+
log_dis = torch.log1p(all_dis)
|
| 288 |
+
warp_factor = log_dis / all_dis.clip(min=1e-8)
|
| 289 |
+
H1, W1 = pts1.shape[1:-1]
|
| 290 |
+
pts1 = pts1 * warp_factor[:, :W1 * H1].view(-1, H1, W1, 1)
|
| 291 |
+
if pts2 is not None:
|
| 292 |
+
H2, W2 = pts2.shape[1:-1]
|
| 293 |
+
pts2 = pts2 * warp_factor[:, W1 * H1:].view(-1, H2, W2, 1)
|
| 294 |
+
all_dis = log_dis # this is their true distance afterwards
|
| 295 |
+
else:
|
| 296 |
+
raise ValueError(f'bad {dis_mode=}')
|
| 297 |
+
|
| 298 |
+
norm_factor = all_dis.sum(dim=1) / (nnz1 + nnz2 + 1e-8)
|
| 299 |
+
else:
|
| 300 |
+
# gather all points together (joint normalization)
|
| 301 |
+
nan_pts1 = invalid_to_nans(pts1, valid1, ndim=3)
|
| 302 |
+
nan_pts2 = invalid_to_nans(pts2, valid2, ndim=3) if pts2 is not None else None
|
| 303 |
+
all_pts = torch.cat((nan_pts1, nan_pts2), dim=1) if pts2 is not None else nan_pts1
|
| 304 |
+
|
| 305 |
+
# compute distance to origin
|
| 306 |
+
all_dis = all_pts.norm(dim=-1)
|
| 307 |
+
|
| 308 |
+
if norm_mode == 'avg':
|
| 309 |
+
norm_factor = all_dis.nanmean(dim=1)
|
| 310 |
+
elif norm_mode == 'median':
|
| 311 |
+
norm_factor = all_dis.nanmedian(dim=1).values.detach()
|
| 312 |
+
elif norm_mode == 'sqrt':
|
| 313 |
+
norm_factor = all_dis.sqrt().nanmean(dim=1)**2
|
| 314 |
+
else:
|
| 315 |
+
raise ValueError(f'bad {norm_mode=}')
|
| 316 |
+
|
| 317 |
+
norm_factor = norm_factor.clip(min=1e-8)
|
| 318 |
+
while norm_factor.ndim < pts1.ndim:
|
| 319 |
+
norm_factor.unsqueeze_(-1)
|
| 320 |
+
|
| 321 |
+
res = pts1 / norm_factor
|
| 322 |
+
if pts2 is not None:
|
| 323 |
+
res = (res, pts2 / norm_factor)
|
| 324 |
+
if ret_factor:
|
| 325 |
+
res = res + (norm_factor,)
|
| 326 |
+
return res
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
@torch.no_grad()
|
| 330 |
+
def get_joint_pointcloud_depth(z1, z2, valid_mask1, valid_mask2=None, quantile=0.5):
|
| 331 |
+
# set invalid points to NaN
|
| 332 |
+
_z1 = invalid_to_nans(z1, valid_mask1).reshape(len(z1), -1)
|
| 333 |
+
_z2 = invalid_to_nans(z2, valid_mask2).reshape(len(z2), -1) if z2 is not None else None
|
| 334 |
+
_z = torch.cat((_z1, _z2), dim=-1) if z2 is not None else _z1
|
| 335 |
+
|
| 336 |
+
# compute median depth overall (ignoring nans)
|
| 337 |
+
if quantile == 0.5:
|
| 338 |
+
shift_z = torch.nanmedian(_z, dim=-1).values
|
| 339 |
+
else:
|
| 340 |
+
shift_z = torch.nanquantile(_z, quantile, dim=-1)
|
| 341 |
+
return shift_z # (B,)
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
@torch.no_grad()
|
| 345 |
+
def get_joint_pointcloud_center_scale(pts1, pts2, valid_mask1=None, valid_mask2=None, z_only=False, center=True):
|
| 346 |
+
# set invalid points to NaN
|
| 347 |
+
_pts1 = invalid_to_nans(pts1, valid_mask1).reshape(len(pts1), -1, 3)
|
| 348 |
+
_pts2 = invalid_to_nans(pts2, valid_mask2).reshape(len(pts2), -1, 3) if pts2 is not None else None
|
| 349 |
+
_pts = torch.cat((_pts1, _pts2), dim=1) if pts2 is not None else _pts1
|
| 350 |
+
|
| 351 |
+
# compute median center
|
| 352 |
+
_center = torch.nanmedian(_pts, dim=1, keepdim=True).values # (B,1,3)
|
| 353 |
+
if z_only:
|
| 354 |
+
_center[..., :2] = 0 # do not center X and Y
|
| 355 |
+
|
| 356 |
+
# compute median norm
|
| 357 |
+
_norm = ((_pts - _center) if center else _pts).norm(dim=-1)
|
| 358 |
+
scale = torch.nanmedian(_norm, dim=1).values
|
| 359 |
+
return _center[:, None, :, :], scale[:, None, None, None]
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
def find_reciprocal_matches(P1, P2):
|
| 363 |
+
"""
|
| 364 |
+
returns 3 values:
|
| 365 |
+
1 - reciprocal_in_P2: a boolean array of size P2.shape[0], a "True" value indicates a match
|
| 366 |
+
2 - nn2_in_P1: a int array of size P2.shape[0], it contains the indexes of the closest points in P1
|
| 367 |
+
3 - reciprocal_in_P2.sum(): the number of matches
|
| 368 |
+
"""
|
| 369 |
+
tree1 = KDTree(P1)
|
| 370 |
+
tree2 = KDTree(P2)
|
| 371 |
+
|
| 372 |
+
_, nn1_in_P2 = tree2.query(P1, workers=8)
|
| 373 |
+
_, nn2_in_P1 = tree1.query(P2, workers=8)
|
| 374 |
+
|
| 375 |
+
reciprocal_in_P1 = (nn2_in_P1[nn1_in_P2] == np.arange(len(nn1_in_P2)))
|
| 376 |
+
reciprocal_in_P2 = (nn1_in_P2[nn2_in_P1] == np.arange(len(nn2_in_P1)))
|
| 377 |
+
assert reciprocal_in_P1.sum() == reciprocal_in_P2.sum()
|
| 378 |
+
return reciprocal_in_P2, nn2_in_P1, reciprocal_in_P2.sum()
|
| 379 |
+
|
| 380 |
+
|
| 381 |
+
def get_med_dist_between_poses(poses):
|
| 382 |
+
from scipy.spatial.distance import pdist
|
| 383 |
+
return np.median(pdist([p[:3, 3].detach().cpu().numpy() for p in poses]))
|
| 384 |
+
|
| 385 |
+
def get_rel_pos(_rel_xyz, free_space, voxel_size):
|
| 386 |
+
"""
|
| 387 |
+
Args:
|
| 388 |
+
_rel_xyz: torch.tensor
|
| 389 |
+
shape [..., 3]
|
| 390 |
+
free_space: str
|
| 391 |
+
something like 'soft'
|
| 392 |
+
grid: GridBatch
|
| 393 |
+
fvdb grid
|
| 394 |
+
|
| 395 |
+
Returns:
|
| 396 |
+
rel_pos: torch.tensor
|
| 397 |
+
shape [..., 3], the position within a voxel (compared with the voxel center)
|
| 398 |
+
"""
|
| 399 |
+
if free_space == "hard":
|
| 400 |
+
rel_pos = torch.sigmoid(_rel_xyz) * voxel_size
|
| 401 |
+
elif free_space == "soft":
|
| 402 |
+
# free space [-1, 2]
|
| 403 |
+
rel_pos = (torch.sigmoid(_rel_xyz) * 3 - 1) * voxel_size
|
| 404 |
+
elif free_space == "soft-2":
|
| 405 |
+
# free space [-2, 3]
|
| 406 |
+
rel_pos = (torch.sigmoid(_rel_xyz) * 5 - 2) * voxel_size
|
| 407 |
+
elif free_space == "soft-3":
|
| 408 |
+
# free space [-3, 4]
|
| 409 |
+
rel_pos = (torch.sigmoid(_rel_xyz) * 7 - 3) * voxel_size
|
| 410 |
+
elif free_space == "soft-4":
|
| 411 |
+
# free space [-4, 5]
|
| 412 |
+
rel_pos = (torch.sigmoid(_rel_xyz) * 9 - 4) * voxel_size
|
| 413 |
+
elif free_space == "soft-5":
|
| 414 |
+
# free space [-5, 6]
|
| 415 |
+
rel_pos = (torch.sigmoid(_rel_xyz) * 11 - 5) * voxel_size
|
| 416 |
+
elif free_space == "tanh-3":
|
| 417 |
+
# free space [-2.5, 3.5]
|
| 418 |
+
rel_pos = (torch.tanh(_rel_xyz) * 3 + 0.5) * voxel_size
|
| 419 |
+
elif free_space == "free-1":
|
| 420 |
+
rel_pos = _rel_xyz * voxel_size
|
| 421 |
+
elif free_space == "free-2":
|
| 422 |
+
rel_pos = _rel_xyz
|
| 423 |
+
elif free_space == "center":
|
| 424 |
+
rel_pos = (torch.zeros_like(_rel_xyz) + 0.5) * voxel_size
|
| 425 |
+
else:
|
| 426 |
+
raise NotImplementedError
|
| 427 |
+
|
| 428 |
+
return rel_pos
|
src/utils/image.py
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
|
| 4 |
+
# This source code is licensed under the license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
# Copyright (C) 2024-present Naver Corporation. All rights reserved.
|
| 8 |
+
# Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
|
| 9 |
+
#
|
| 10 |
+
# --------------------------------------------------------
|
| 11 |
+
# utilitary functions about images (loading/converting...)
|
| 12 |
+
# --------------------------------------------------------
|
| 13 |
+
import os
|
| 14 |
+
|
| 15 |
+
import numpy as np
|
| 16 |
+
import PIL.Image
|
| 17 |
+
import torch
|
| 18 |
+
import torchvision.transforms as tvf
|
| 19 |
+
from PIL.ImageOps import exif_transpose
|
| 20 |
+
from PIL import Image
|
| 21 |
+
import torchvision
|
| 22 |
+
|
| 23 |
+
os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1"
|
| 24 |
+
import cv2
|
| 25 |
+
|
| 26 |
+
try:
|
| 27 |
+
from pillow_heif import register_heif_opener
|
| 28 |
+
|
| 29 |
+
register_heif_opener()
|
| 30 |
+
heif_support_enabled = True
|
| 31 |
+
except ImportError:
|
| 32 |
+
heif_support_enabled = False
|
| 33 |
+
|
| 34 |
+
ImgNorm = tvf.Compose([tvf.ToTensor(), tvf.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def imread_cv2(path, options=cv2.IMREAD_COLOR):
|
| 38 |
+
"""Open an image or a depthmap with opencv-python."""
|
| 39 |
+
if path.endswith((".exr", "EXR")):
|
| 40 |
+
options = cv2.IMREAD_ANYDEPTH
|
| 41 |
+
img = cv2.imread(path, options)
|
| 42 |
+
if img is None:
|
| 43 |
+
raise IOError(f"Could not load image={path} with {options=}")
|
| 44 |
+
if img.ndim == 3:
|
| 45 |
+
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
| 46 |
+
return img
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def rgb(ftensor, true_shape=None):
|
| 50 |
+
if isinstance(ftensor, list):
|
| 51 |
+
return [rgb(x, true_shape=true_shape) for x in ftensor]
|
| 52 |
+
if isinstance(ftensor, torch.Tensor):
|
| 53 |
+
ftensor = ftensor.detach().cpu().numpy() # H,W,3
|
| 54 |
+
if ftensor.ndim == 3 and ftensor.shape[0] == 3:
|
| 55 |
+
ftensor = ftensor.transpose(1, 2, 0)
|
| 56 |
+
elif ftensor.ndim == 4 and ftensor.shape[1] == 3:
|
| 57 |
+
ftensor = ftensor.transpose(0, 2, 3, 1)
|
| 58 |
+
if true_shape is not None:
|
| 59 |
+
H, W = true_shape
|
| 60 |
+
ftensor = ftensor[:H, :W]
|
| 61 |
+
if ftensor.dtype == np.uint8:
|
| 62 |
+
img = np.float32(ftensor) / 255
|
| 63 |
+
else:
|
| 64 |
+
img = (ftensor * 0.5) + 0.5
|
| 65 |
+
return img.clip(min=0, max=1)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def _resize_pil_image(img, long_edge_size):
|
| 69 |
+
S = max(img.size)
|
| 70 |
+
if S > long_edge_size:
|
| 71 |
+
interp = PIL.Image.LANCZOS
|
| 72 |
+
elif S <= long_edge_size:
|
| 73 |
+
interp = PIL.Image.BICUBIC
|
| 74 |
+
new_size = tuple(int(round(x * long_edge_size / S)) for x in img.size)
|
| 75 |
+
return img.resize(new_size, interp)
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def load_images(folder_or_list, size, square_ok=False, verbose=True, rotate_clockwise_90=False, crop_to_landscape=False):
|
| 79 |
+
"""open and convert all images in a list or folder to proper input format for DUSt3R"""
|
| 80 |
+
if isinstance(folder_or_list, str):
|
| 81 |
+
if verbose:
|
| 82 |
+
print(f">> Loading images from {folder_or_list}")
|
| 83 |
+
root, folder_content = folder_or_list, sorted(os.listdir(folder_or_list))
|
| 84 |
+
|
| 85 |
+
elif isinstance(folder_or_list, list):
|
| 86 |
+
if verbose:
|
| 87 |
+
print(f">> Loading a list of {len(folder_or_list)} images")
|
| 88 |
+
root, folder_content = "", folder_or_list
|
| 89 |
+
|
| 90 |
+
else:
|
| 91 |
+
raise ValueError(f"bad {folder_or_list=} ({type(folder_or_list)})")
|
| 92 |
+
|
| 93 |
+
supported_images_extensions = [".jpg", ".jpeg", ".png"]
|
| 94 |
+
if heif_support_enabled:
|
| 95 |
+
supported_images_extensions += [".heic", ".heif"]
|
| 96 |
+
supported_images_extensions = tuple(supported_images_extensions)
|
| 97 |
+
|
| 98 |
+
imgs = []
|
| 99 |
+
for path in folder_content:
|
| 100 |
+
if not path.lower().endswith(supported_images_extensions):
|
| 101 |
+
continue
|
| 102 |
+
img = exif_transpose(PIL.Image.open(os.path.join(root, path))).convert("RGB")
|
| 103 |
+
if rotate_clockwise_90:
|
| 104 |
+
img = img.rotate(-90, expand=True)
|
| 105 |
+
if crop_to_landscape:
|
| 106 |
+
# Crop to a landscape aspect ratio (e.g., 16:9)
|
| 107 |
+
desired_aspect_ratio = 4 / 3
|
| 108 |
+
width, height = img.size
|
| 109 |
+
current_aspect_ratio = width / height
|
| 110 |
+
|
| 111 |
+
if current_aspect_ratio > desired_aspect_ratio:
|
| 112 |
+
# Wider than landscape: crop width
|
| 113 |
+
new_width = int(height * desired_aspect_ratio)
|
| 114 |
+
left = (width - new_width) // 2
|
| 115 |
+
right = left + new_width
|
| 116 |
+
top = 0
|
| 117 |
+
bottom = height
|
| 118 |
+
else:
|
| 119 |
+
# Taller than landscape: crop height
|
| 120 |
+
new_height = int(width / desired_aspect_ratio)
|
| 121 |
+
top = (height - new_height) // 2
|
| 122 |
+
bottom = top + new_height
|
| 123 |
+
left = 0
|
| 124 |
+
right = width
|
| 125 |
+
|
| 126 |
+
img = img.crop((left, top, right, bottom))
|
| 127 |
+
|
| 128 |
+
W1, H1 = img.size
|
| 129 |
+
if size == 224:
|
| 130 |
+
# resize short side to 224 (then crop)
|
| 131 |
+
img = _resize_pil_image(img, round(size * max(W1 / H1, H1 / W1)))
|
| 132 |
+
else:
|
| 133 |
+
# resize long side to 512
|
| 134 |
+
img = _resize_pil_image(img, size)
|
| 135 |
+
W, H = img.size
|
| 136 |
+
cx, cy = W // 2, H // 2
|
| 137 |
+
if size == 224:
|
| 138 |
+
half = min(cx, cy)
|
| 139 |
+
img = img.crop((cx - half, cy - half, cx + half, cy + half))
|
| 140 |
+
else:
|
| 141 |
+
halfw, halfh = ((2 * cx) // 16) * 8, ((2 * cy) // 16) * 8
|
| 142 |
+
if not (square_ok) and W == H:
|
| 143 |
+
halfh = 3 * halfw / 4
|
| 144 |
+
img = img.crop((cx - halfw, cy - halfh, cx + halfw, cy + halfh))
|
| 145 |
+
|
| 146 |
+
W2, H2 = img.size
|
| 147 |
+
if verbose:
|
| 148 |
+
print(f" - adding {path} with resolution {W1}x{H1} --> {W2}x{H2}")
|
| 149 |
+
imgs.append(
|
| 150 |
+
dict(
|
| 151 |
+
img=ImgNorm(img)[None],
|
| 152 |
+
true_shape=np.int32([img.size[::-1]]),
|
| 153 |
+
idx=len(imgs),
|
| 154 |
+
instance=str(len(imgs)),
|
| 155 |
+
)
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
assert imgs, "no images foud at " + root
|
| 159 |
+
if verbose:
|
| 160 |
+
print(f" (Found {len(imgs)} images)")
|
| 161 |
+
return imgs
|
| 162 |
+
|
| 163 |
+
def process_image(img_path):
|
| 164 |
+
img = Image.open(img_path)
|
| 165 |
+
if img.mode == 'RGBA':
|
| 166 |
+
# Convert RGBA to RGB by removing alpha channel
|
| 167 |
+
img = img.convert('RGB')
|
| 168 |
+
# Resize to maintain aspect ratio and then center crop to 448x448
|
| 169 |
+
width, height = img.size
|
| 170 |
+
if width > height:
|
| 171 |
+
new_height = 448
|
| 172 |
+
new_width = int(width * (new_height / height))
|
| 173 |
+
else:
|
| 174 |
+
new_width = 448
|
| 175 |
+
new_height = int(height * (new_width / width))
|
| 176 |
+
img = img.resize((new_width, new_height))
|
| 177 |
+
|
| 178 |
+
# Center crop
|
| 179 |
+
left = (new_width - 448) // 2
|
| 180 |
+
top = (new_height - 448) // 2
|
| 181 |
+
right = left + 448
|
| 182 |
+
bottom = top + 448
|
| 183 |
+
img = img.crop((left, top, right, bottom))
|
| 184 |
+
img_tensor = torchvision.transforms.ToTensor()(img) * 2.0 - 1.0 # [-1, 1]
|
| 185 |
+
return img_tensor
|
src/utils/misc.py
ADDED
|
@@ -0,0 +1,531 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
|
| 4 |
+
# This source code is licensed under the license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
# Copyright (C) 2022-present Naver Corporation. All rights reserved.
|
| 8 |
+
# Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
|
| 9 |
+
#
|
| 10 |
+
# --------------------------------------------------------
|
| 11 |
+
# utilitary functions for CroCo
|
| 12 |
+
# --------------------------------------------------------
|
| 13 |
+
# References:
|
| 14 |
+
# MAE: https://github.com/facebookresearch/mae
|
| 15 |
+
# DeiT: https://github.com/facebookresearch/deit
|
| 16 |
+
# BEiT: https://github.com/microsoft/unilm/tree/master/beit
|
| 17 |
+
# --------------------------------------------------------
|
| 18 |
+
|
| 19 |
+
import builtins
|
| 20 |
+
import datetime
|
| 21 |
+
import json
|
| 22 |
+
import math
|
| 23 |
+
import os
|
| 24 |
+
import time
|
| 25 |
+
from collections import defaultdict, deque
|
| 26 |
+
from pathlib import Path
|
| 27 |
+
|
| 28 |
+
import numpy as np
|
| 29 |
+
import torch
|
| 30 |
+
import torch.distributed as dist
|
| 31 |
+
from torch import inf
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class SmoothedValue(object):
|
| 35 |
+
"""Track a series of values and provide access to smoothed values over a
|
| 36 |
+
window or the global series average.
|
| 37 |
+
"""
|
| 38 |
+
|
| 39 |
+
def __init__(self, window_size=20, fmt=None):
|
| 40 |
+
if fmt is None:
|
| 41 |
+
fmt = "{median:.4f} ({global_avg:.4f})"
|
| 42 |
+
self.deque = deque(maxlen=window_size)
|
| 43 |
+
self.total = 0.0
|
| 44 |
+
self.count = 0
|
| 45 |
+
self.fmt = fmt
|
| 46 |
+
|
| 47 |
+
def update(self, value, n=1):
|
| 48 |
+
self.deque.append(value)
|
| 49 |
+
self.count += n
|
| 50 |
+
self.total += value * n
|
| 51 |
+
|
| 52 |
+
def synchronize_between_processes(self):
|
| 53 |
+
"""
|
| 54 |
+
Warning: does not synchronize the deque!
|
| 55 |
+
"""
|
| 56 |
+
if not is_dist_avail_and_initialized():
|
| 57 |
+
return
|
| 58 |
+
t = torch.tensor([self.count, self.total], dtype=torch.float64, device="cuda")
|
| 59 |
+
dist.barrier()
|
| 60 |
+
dist.all_reduce(t)
|
| 61 |
+
t = t.tolist()
|
| 62 |
+
self.count = int(t[0])
|
| 63 |
+
self.total = t[1]
|
| 64 |
+
|
| 65 |
+
@property
|
| 66 |
+
def median(self):
|
| 67 |
+
d = torch.tensor(list(self.deque))
|
| 68 |
+
return d.median().item()
|
| 69 |
+
|
| 70 |
+
@property
|
| 71 |
+
def avg(self):
|
| 72 |
+
d = torch.tensor(list(self.deque), dtype=torch.float32)
|
| 73 |
+
return d.mean().item()
|
| 74 |
+
|
| 75 |
+
@property
|
| 76 |
+
def global_avg(self):
|
| 77 |
+
return self.total / self.count
|
| 78 |
+
|
| 79 |
+
@property
|
| 80 |
+
def max(self):
|
| 81 |
+
return max(self.deque)
|
| 82 |
+
|
| 83 |
+
@property
|
| 84 |
+
def value(self):
|
| 85 |
+
return self.deque[-1]
|
| 86 |
+
|
| 87 |
+
def __str__(self):
|
| 88 |
+
return self.fmt.format(
|
| 89 |
+
median=self.median,
|
| 90 |
+
avg=self.avg,
|
| 91 |
+
global_avg=self.global_avg,
|
| 92 |
+
max=self.max,
|
| 93 |
+
value=self.value,
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
class MetricLogger(object):
|
| 98 |
+
def __init__(self, delimiter="\t"):
|
| 99 |
+
self.meters = defaultdict(SmoothedValue)
|
| 100 |
+
self.delimiter = delimiter
|
| 101 |
+
|
| 102 |
+
def update(self, **kwargs):
|
| 103 |
+
for k, v in kwargs.items():
|
| 104 |
+
if v is None:
|
| 105 |
+
continue
|
| 106 |
+
if isinstance(v, torch.Tensor):
|
| 107 |
+
v = v.item()
|
| 108 |
+
assert isinstance(v, (float, int))
|
| 109 |
+
self.meters[k].update(v)
|
| 110 |
+
|
| 111 |
+
def __getattr__(self, attr):
|
| 112 |
+
if attr in self.meters:
|
| 113 |
+
return self.meters[attr]
|
| 114 |
+
if attr in self.__dict__:
|
| 115 |
+
return self.__dict__[attr]
|
| 116 |
+
raise AttributeError(
|
| 117 |
+
"'{}' object has no attribute '{}'".format(type(self).__name__, attr)
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
def __str__(self):
|
| 121 |
+
loss_str = []
|
| 122 |
+
for name, meter in self.meters.items():
|
| 123 |
+
loss_str.append("{}: {}".format(name, str(meter)))
|
| 124 |
+
return self.delimiter.join(loss_str)
|
| 125 |
+
|
| 126 |
+
def synchronize_between_processes(self):
|
| 127 |
+
for meter in self.meters.values():
|
| 128 |
+
meter.synchronize_between_processes()
|
| 129 |
+
|
| 130 |
+
def add_meter(self, name, meter):
|
| 131 |
+
self.meters[name] = meter
|
| 132 |
+
|
| 133 |
+
def log_every(self, iterable, print_freq, header=None, max_iter=None):
|
| 134 |
+
i = 0
|
| 135 |
+
if not header:
|
| 136 |
+
header = ""
|
| 137 |
+
start_time = time.time()
|
| 138 |
+
end = time.time()
|
| 139 |
+
iter_time = SmoothedValue(fmt="{avg:.4f}")
|
| 140 |
+
data_time = SmoothedValue(fmt="{avg:.4f}")
|
| 141 |
+
len_iterable = min(len(iterable), max_iter) if max_iter else len(iterable)
|
| 142 |
+
space_fmt = ":" + str(len(str(len_iterable))) + "d"
|
| 143 |
+
log_msg = [
|
| 144 |
+
header,
|
| 145 |
+
"[{0" + space_fmt + "}/{1}]",
|
| 146 |
+
"eta: {eta}",
|
| 147 |
+
"{meters}",
|
| 148 |
+
"time: {time}",
|
| 149 |
+
"data: {data}",
|
| 150 |
+
]
|
| 151 |
+
if torch.cuda.is_available():
|
| 152 |
+
log_msg.append("max mem: {memory:.0f}")
|
| 153 |
+
log_msg = self.delimiter.join(log_msg)
|
| 154 |
+
MB = 1024.0 * 1024.0
|
| 155 |
+
for it, obj in enumerate(iterable):
|
| 156 |
+
data_time.update(time.time() - end)
|
| 157 |
+
yield obj
|
| 158 |
+
iter_time.update(time.time() - end)
|
| 159 |
+
if i % print_freq == 0 or i == len_iterable - 1:
|
| 160 |
+
eta_seconds = iter_time.global_avg * (len_iterable - i)
|
| 161 |
+
eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
|
| 162 |
+
if torch.cuda.is_available():
|
| 163 |
+
print(
|
| 164 |
+
log_msg.format(
|
| 165 |
+
i,
|
| 166 |
+
len_iterable,
|
| 167 |
+
eta=eta_string,
|
| 168 |
+
meters=str(self),
|
| 169 |
+
time=str(iter_time),
|
| 170 |
+
data=str(data_time),
|
| 171 |
+
memory=torch.cuda.max_memory_allocated() / MB,
|
| 172 |
+
)
|
| 173 |
+
)
|
| 174 |
+
else:
|
| 175 |
+
print(
|
| 176 |
+
log_msg.format(
|
| 177 |
+
i,
|
| 178 |
+
len_iterable,
|
| 179 |
+
eta=eta_string,
|
| 180 |
+
meters=str(self),
|
| 181 |
+
time=str(iter_time),
|
| 182 |
+
data=str(data_time),
|
| 183 |
+
)
|
| 184 |
+
)
|
| 185 |
+
i += 1
|
| 186 |
+
end = time.time()
|
| 187 |
+
if max_iter and it >= max_iter:
|
| 188 |
+
break
|
| 189 |
+
total_time = time.time() - start_time
|
| 190 |
+
total_time_str = str(datetime.timedelta(seconds=int(total_time)))
|
| 191 |
+
print(
|
| 192 |
+
"{} Total time: {} ({:.4f} s / it)".format(
|
| 193 |
+
header, total_time_str, total_time / len_iterable
|
| 194 |
+
)
|
| 195 |
+
)
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def setup_for_distributed(is_master):
|
| 199 |
+
"""
|
| 200 |
+
This function disables printing when not in master process
|
| 201 |
+
"""
|
| 202 |
+
builtin_print = builtins.print
|
| 203 |
+
|
| 204 |
+
def print(*args, **kwargs):
|
| 205 |
+
force = kwargs.pop("force", False)
|
| 206 |
+
force = force or (get_world_size() > 8)
|
| 207 |
+
if is_master or force:
|
| 208 |
+
now = datetime.datetime.now().time()
|
| 209 |
+
builtin_print("[{}] ".format(now), end="") # print with time stamp
|
| 210 |
+
builtin_print(*args, **kwargs)
|
| 211 |
+
|
| 212 |
+
builtins.print = print
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def is_dist_avail_and_initialized():
|
| 216 |
+
if not dist.is_available():
|
| 217 |
+
return False
|
| 218 |
+
if not dist.is_initialized():
|
| 219 |
+
return False
|
| 220 |
+
return True
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def get_world_size():
|
| 224 |
+
if not is_dist_avail_and_initialized():
|
| 225 |
+
return 1
|
| 226 |
+
return dist.get_world_size()
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def get_rank():
|
| 230 |
+
if not is_dist_avail_and_initialized():
|
| 231 |
+
return 0
|
| 232 |
+
return dist.get_rank()
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
def is_main_process():
|
| 236 |
+
return get_rank() == 0
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def save_on_master(*args, **kwargs):
|
| 240 |
+
if is_main_process():
|
| 241 |
+
torch.save(*args, **kwargs)
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
def init_distributed_mode(args):
|
| 245 |
+
nodist = args.nodist if hasattr(args, "nodist") else False
|
| 246 |
+
if "RANK" in os.environ and "WORLD_SIZE" in os.environ and not nodist:
|
| 247 |
+
args.rank = int(os.environ["RANK"])
|
| 248 |
+
args.world_size = int(os.environ["WORLD_SIZE"])
|
| 249 |
+
args.gpu = int(os.environ["LOCAL_RANK"])
|
| 250 |
+
else:
|
| 251 |
+
print("Not using distributed mode")
|
| 252 |
+
setup_for_distributed(is_master=True) # hack
|
| 253 |
+
args.distributed = False
|
| 254 |
+
return
|
| 255 |
+
|
| 256 |
+
args.distributed = True
|
| 257 |
+
|
| 258 |
+
torch.cuda.set_device(args.gpu)
|
| 259 |
+
args.dist_backend = "nccl"
|
| 260 |
+
print(
|
| 261 |
+
"| distributed init (rank {}): {}, gpu {}".format(
|
| 262 |
+
args.rank, args.dist_url, args.gpu
|
| 263 |
+
),
|
| 264 |
+
flush=True,
|
| 265 |
+
)
|
| 266 |
+
torch.distributed.init_process_group(
|
| 267 |
+
backend=args.dist_backend,
|
| 268 |
+
init_method=args.dist_url,
|
| 269 |
+
world_size=args.world_size,
|
| 270 |
+
rank=args.rank,
|
| 271 |
+
)
|
| 272 |
+
torch.distributed.barrier()
|
| 273 |
+
setup_for_distributed(args.rank == 0)
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
class NativeScalerWithGradNormCount:
|
| 277 |
+
state_dict_key = "amp_scaler"
|
| 278 |
+
|
| 279 |
+
def __init__(self, enabled=True):
|
| 280 |
+
self._scaler = torch.cuda.amp.GradScaler(enabled=enabled)
|
| 281 |
+
|
| 282 |
+
def __call__(
|
| 283 |
+
self,
|
| 284 |
+
loss,
|
| 285 |
+
optimizer,
|
| 286 |
+
clip_grad=None,
|
| 287 |
+
parameters=None,
|
| 288 |
+
create_graph=False,
|
| 289 |
+
update_grad=True,
|
| 290 |
+
):
|
| 291 |
+
self._scaler.scale(loss).backward(create_graph=create_graph)
|
| 292 |
+
if update_grad:
|
| 293 |
+
if clip_grad is not None:
|
| 294 |
+
assert parameters is not None
|
| 295 |
+
self._scaler.unscale_(
|
| 296 |
+
optimizer
|
| 297 |
+
) # unscale the gradients of optimizer's assigned params in-place
|
| 298 |
+
norm = torch.nn.utils.clip_grad_norm_(parameters, clip_grad)
|
| 299 |
+
else:
|
| 300 |
+
self._scaler.unscale_(optimizer)
|
| 301 |
+
# TODO: FIXME: get_grad_norm_ is a poorly-implemented func that is very slow, and its return not even used
|
| 302 |
+
# norm = get_grad_norm_(parameters)
|
| 303 |
+
norm = None
|
| 304 |
+
self._scaler.step(optimizer)
|
| 305 |
+
self._scaler.update()
|
| 306 |
+
else:
|
| 307 |
+
norm = None
|
| 308 |
+
return norm
|
| 309 |
+
|
| 310 |
+
def state_dict(self):
|
| 311 |
+
return self._scaler.state_dict()
|
| 312 |
+
|
| 313 |
+
def load_state_dict(self, state_dict):
|
| 314 |
+
self._scaler.load_state_dict(state_dict)
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
def get_grad_norm_(parameters, norm_type: float = 2.0) -> torch.Tensor:
|
| 318 |
+
if isinstance(parameters, torch.Tensor):
|
| 319 |
+
parameters = [parameters]
|
| 320 |
+
parameters = [p for p in parameters if p.grad is not None]
|
| 321 |
+
norm_type = float(norm_type)
|
| 322 |
+
if len(parameters) == 0:
|
| 323 |
+
return torch.tensor(0.0)
|
| 324 |
+
device = parameters[0].grad.device
|
| 325 |
+
if norm_type == inf:
|
| 326 |
+
total_norm = max(p.grad.detach().abs().max().to(device) for p in parameters)
|
| 327 |
+
else:
|
| 328 |
+
total_norm = torch.norm(
|
| 329 |
+
torch.stack(
|
| 330 |
+
[torch.norm(p.grad.detach(), norm_type).to(device) for p in parameters]
|
| 331 |
+
),
|
| 332 |
+
norm_type,
|
| 333 |
+
)
|
| 334 |
+
return total_norm
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
def save_model(
|
| 338 |
+
args, epoch, model_without_ddp, optimizer, loss_scaler, fname=None, best_so_far=None
|
| 339 |
+
):
|
| 340 |
+
output_dir = Path(args.output_dir)
|
| 341 |
+
if fname is None:
|
| 342 |
+
fname = str(epoch)
|
| 343 |
+
checkpoint_path = output_dir / ("checkpoint-%s.pth" % fname)
|
| 344 |
+
to_save = {
|
| 345 |
+
"model": model_without_ddp.state_dict(),
|
| 346 |
+
"optimizer": optimizer.state_dict(),
|
| 347 |
+
"scaler": loss_scaler.state_dict(),
|
| 348 |
+
"args": args,
|
| 349 |
+
"epoch": epoch,
|
| 350 |
+
}
|
| 351 |
+
if best_so_far is not None:
|
| 352 |
+
to_save["best_so_far"] = best_so_far
|
| 353 |
+
print(f">> Saving model to {checkpoint_path} ...")
|
| 354 |
+
save_on_master(to_save, checkpoint_path)
|
| 355 |
+
|
| 356 |
+
|
| 357 |
+
def load_model(args, model_without_ddp, optimizer, loss_scaler):
|
| 358 |
+
args.start_epoch = 0
|
| 359 |
+
best_so_far = None
|
| 360 |
+
if args.resume is not None:
|
| 361 |
+
if args.resume.startswith("https"):
|
| 362 |
+
checkpoint = torch.hub.load_state_dict_from_url(
|
| 363 |
+
args.resume, map_location="cpu", check_hash=True
|
| 364 |
+
)
|
| 365 |
+
else:
|
| 366 |
+
checkpoint = torch.load(args.resume, map_location="cpu")
|
| 367 |
+
print("Resume checkpoint %s" % args.resume)
|
| 368 |
+
model_without_ddp.load_state_dict(checkpoint["model"], strict=False)
|
| 369 |
+
args.start_epoch = checkpoint["epoch"] + 1
|
| 370 |
+
optimizer.load_state_dict(checkpoint["optimizer"])
|
| 371 |
+
if "scaler" in checkpoint:
|
| 372 |
+
loss_scaler.load_state_dict(checkpoint["scaler"])
|
| 373 |
+
if "best_so_far" in checkpoint:
|
| 374 |
+
best_so_far = checkpoint["best_so_far"]
|
| 375 |
+
print(" & best_so_far={:g}".format(best_so_far))
|
| 376 |
+
else:
|
| 377 |
+
print("")
|
| 378 |
+
print("With optim & sched! start_epoch={:d}".format(args.start_epoch), end="")
|
| 379 |
+
return best_so_far
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
def all_reduce_mean(x):
|
| 383 |
+
world_size = get_world_size()
|
| 384 |
+
if world_size > 1:
|
| 385 |
+
x_reduce = torch.tensor(x).cuda()
|
| 386 |
+
dist.all_reduce(x_reduce)
|
| 387 |
+
x_reduce /= world_size
|
| 388 |
+
return x_reduce.item()
|
| 389 |
+
else:
|
| 390 |
+
return x
|
| 391 |
+
|
| 392 |
+
|
| 393 |
+
def _replace(text, src, tgt, rm=""):
|
| 394 |
+
"""Advanced string replacement.
|
| 395 |
+
Given a text:
|
| 396 |
+
- replace all elements in src by the corresponding element in tgt
|
| 397 |
+
- remove all elements in rm
|
| 398 |
+
"""
|
| 399 |
+
if len(tgt) == 1:
|
| 400 |
+
tgt = tgt * len(src)
|
| 401 |
+
assert len(src) == len(tgt), f"'{src}' and '{tgt}' should have the same len"
|
| 402 |
+
for s, t in zip(src, tgt):
|
| 403 |
+
text = text.replace(s, t)
|
| 404 |
+
for c in rm:
|
| 405 |
+
text = text.replace(c, "")
|
| 406 |
+
return text
|
| 407 |
+
|
| 408 |
+
|
| 409 |
+
def filename(obj):
|
| 410 |
+
"""transform a python obj or cmd into a proper filename.
|
| 411 |
+
- \1 gets replaced by slash '/'
|
| 412 |
+
- \2 gets replaced by comma ','
|
| 413 |
+
"""
|
| 414 |
+
if not isinstance(obj, str):
|
| 415 |
+
obj = repr(obj)
|
| 416 |
+
obj = str(obj).replace("()", "")
|
| 417 |
+
obj = _replace(obj, "_,(*/\1\2", "-__x%/,", rm=" )'\"")
|
| 418 |
+
assert all(len(s) < 256 for s in obj.split(os.sep)), (
|
| 419 |
+
"filename too long (>256 characters):\n" + obj
|
| 420 |
+
)
|
| 421 |
+
return obj
|
| 422 |
+
|
| 423 |
+
|
| 424 |
+
def _get_num_layer_for_vit(var_name, enc_depth, dec_depth):
|
| 425 |
+
if var_name in ("cls_token", "mask_token", "pos_embed", "global_tokens"):
|
| 426 |
+
return 0
|
| 427 |
+
elif var_name.startswith("patch_embed"):
|
| 428 |
+
return 0
|
| 429 |
+
elif var_name.startswith("enc_blocks"):
|
| 430 |
+
layer_id = int(var_name.split(".")[1])
|
| 431 |
+
return layer_id + 1
|
| 432 |
+
elif var_name.startswith("decoder_embed") or var_name.startswith(
|
| 433 |
+
"enc_norm"
|
| 434 |
+
): # part of the last black
|
| 435 |
+
return enc_depth
|
| 436 |
+
elif var_name.startswith("dec_blocks"):
|
| 437 |
+
layer_id = int(var_name.split(".")[1])
|
| 438 |
+
return enc_depth + layer_id + 1
|
| 439 |
+
elif var_name.startswith("dec_norm"): # part of the last block
|
| 440 |
+
return enc_depth + dec_depth
|
| 441 |
+
elif any(var_name.startswith(k) for k in ["head", "prediction_head"]):
|
| 442 |
+
return enc_depth + dec_depth + 1
|
| 443 |
+
else:
|
| 444 |
+
raise NotImplementedError(var_name)
|
| 445 |
+
|
| 446 |
+
|
| 447 |
+
def get_parameter_groups(
|
| 448 |
+
model, weight_decay, layer_decay=1.0, skip_list=(), no_lr_scale_list=[]
|
| 449 |
+
):
|
| 450 |
+
parameter_group_names = {}
|
| 451 |
+
parameter_group_vars = {}
|
| 452 |
+
enc_depth, dec_depth = None, None
|
| 453 |
+
# prepare layer decay values
|
| 454 |
+
assert layer_decay == 1.0 or 0.0 < layer_decay < 1.0
|
| 455 |
+
if layer_decay < 1.0:
|
| 456 |
+
enc_depth = model.enc_depth
|
| 457 |
+
dec_depth = model.dec_depth if hasattr(model, "dec_blocks") else 0
|
| 458 |
+
num_layers = enc_depth + dec_depth
|
| 459 |
+
layer_decay_values = list(
|
| 460 |
+
layer_decay ** (num_layers + 1 - i) for i in range(num_layers + 2)
|
| 461 |
+
)
|
| 462 |
+
|
| 463 |
+
for name, param in model.named_parameters():
|
| 464 |
+
if not param.requires_grad:
|
| 465 |
+
continue # frozen weights
|
| 466 |
+
|
| 467 |
+
# Assign weight decay values
|
| 468 |
+
if len(param.shape) == 1 or name.endswith(".bias") or name in skip_list:
|
| 469 |
+
group_name = "no_decay"
|
| 470 |
+
this_weight_decay = 0.0
|
| 471 |
+
else:
|
| 472 |
+
group_name = "decay"
|
| 473 |
+
this_weight_decay = weight_decay
|
| 474 |
+
|
| 475 |
+
# Assign layer ID for LR scaling
|
| 476 |
+
if layer_decay < 1.0:
|
| 477 |
+
skip_scale = False
|
| 478 |
+
layer_id = _get_num_layer_for_vit(name, enc_depth, dec_depth)
|
| 479 |
+
group_name = "layer_%d_%s" % (layer_id, group_name)
|
| 480 |
+
if name in no_lr_scale_list:
|
| 481 |
+
skip_scale = True
|
| 482 |
+
group_name = f"{group_name}_no_lr_scale"
|
| 483 |
+
else:
|
| 484 |
+
layer_id = 0
|
| 485 |
+
skip_scale = True
|
| 486 |
+
|
| 487 |
+
if group_name not in parameter_group_names:
|
| 488 |
+
if not skip_scale:
|
| 489 |
+
scale = layer_decay_values[layer_id]
|
| 490 |
+
else:
|
| 491 |
+
scale = 1.0
|
| 492 |
+
|
| 493 |
+
parameter_group_names[group_name] = {
|
| 494 |
+
"weight_decay": this_weight_decay,
|
| 495 |
+
"params": [],
|
| 496 |
+
"lr_scale": scale,
|
| 497 |
+
}
|
| 498 |
+
parameter_group_vars[group_name] = {
|
| 499 |
+
"weight_decay": this_weight_decay,
|
| 500 |
+
"params": [],
|
| 501 |
+
"lr_scale": scale,
|
| 502 |
+
}
|
| 503 |
+
|
| 504 |
+
parameter_group_vars[group_name]["params"].append(param)
|
| 505 |
+
parameter_group_names[group_name]["params"].append(name)
|
| 506 |
+
print("Param groups = %s" % json.dumps(parameter_group_names, indent=2))
|
| 507 |
+
return list(parameter_group_vars.values())
|
| 508 |
+
|
| 509 |
+
|
| 510 |
+
def adjust_learning_rate(optimizer, epoch, args):
|
| 511 |
+
"""Decay the learning rate with half-cycle cosine after warmup"""
|
| 512 |
+
|
| 513 |
+
if epoch < args.warmup_epochs:
|
| 514 |
+
lr = args.lr * epoch / args.warmup_epochs
|
| 515 |
+
else:
|
| 516 |
+
lr = args.min_lr + (args.lr - args.min_lr) * 0.5 * (
|
| 517 |
+
1.0
|
| 518 |
+
+ math.cos(
|
| 519 |
+
math.pi
|
| 520 |
+
* (epoch - args.warmup_epochs)
|
| 521 |
+
/ (args.epochs - args.warmup_epochs)
|
| 522 |
+
)
|
| 523 |
+
)
|
| 524 |
+
|
| 525 |
+
for param_group in optimizer.param_groups:
|
| 526 |
+
if "lr_scale" in param_group:
|
| 527 |
+
param_group["lr"] = lr * param_group["lr_scale"]
|
| 528 |
+
else:
|
| 529 |
+
param_group["lr"] = lr
|
| 530 |
+
|
| 531 |
+
return lr
|
src/utils/point.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from torch import Tensor
|
| 3 |
+
|
| 4 |
+
def get_normal_map(depth_map: torch.Tensor, intrinsic: torch.Tensor) -> torch.Tensor:
|
| 5 |
+
"""
|
| 6 |
+
Convert a depth map to camera coordinates.
|
| 7 |
+
|
| 8 |
+
Args:
|
| 9 |
+
depth_map (torch.Tensor): Depth map of shape (H, W).
|
| 10 |
+
intrinsic (torch.Tensor): Camera intrinsic matrix of shape (3, 3).
|
| 11 |
+
|
| 12 |
+
Returns:
|
| 13 |
+
tuple[torch.Tensor, torch.Tensor]: Camera coordinates (H, W, 3)
|
| 14 |
+
"""
|
| 15 |
+
B, H, W = depth_map.shape
|
| 16 |
+
assert intrinsic.shape == (B, 3, 3), "Intrinsic matrix must be Bx3x3"
|
| 17 |
+
assert (intrinsic[:, 0, 1] == 0).all() and (intrinsic[:, 1, 0] == 0).all(), "Intrinsic matrix must have zero skew"
|
| 18 |
+
|
| 19 |
+
# Intrinsic parameters
|
| 20 |
+
fu = intrinsic[:, 0, 0] * W # (B,)
|
| 21 |
+
fv = intrinsic[:, 1, 1] * H # (B,)
|
| 22 |
+
cu = intrinsic[:, 0, 2] * W # (B,)
|
| 23 |
+
cv = intrinsic[:, 1, 2] * H # (B,)
|
| 24 |
+
|
| 25 |
+
# Generate grid of pixel coordinates
|
| 26 |
+
u = torch.arange(W, device=depth_map.device)[None, None, :].expand(B, H, W)
|
| 27 |
+
v = torch.arange(H, device=depth_map.device)[None, :, None].expand(B, H, W)
|
| 28 |
+
|
| 29 |
+
# Unproject to camera coordinates (B, H, W)
|
| 30 |
+
x_cam = (u - cu[:, None, None]) * depth_map / fu[:, None, None]
|
| 31 |
+
y_cam = (v - cv[:, None, None]) * depth_map / fv[:, None, None]
|
| 32 |
+
z_cam = depth_map
|
| 33 |
+
|
| 34 |
+
# Stack to form camera coordinates (B, H, W, 3)
|
| 35 |
+
cam_coords = torch.stack((x_cam, y_cam, z_cam), dim=-1).to(dtype=torch.float32)
|
| 36 |
+
|
| 37 |
+
output = torch.zeros_like(cam_coords)
|
| 38 |
+
# Calculate dx using batch dimension (B, H-2, W-2, 3)
|
| 39 |
+
dx = cam_coords[:, 2:, 1:-1] - cam_coords[:, :-2, 1:-1]
|
| 40 |
+
# Calculate dy using batch dimension (B, H-2, W-2, 3)
|
| 41 |
+
dy = cam_coords[:, 1:-1, 2:] - cam_coords[:, 1:-1, :-2]
|
| 42 |
+
# Cross product and normalization (B, H-2, W-2, 3)
|
| 43 |
+
normal_map = torch.nn.functional.normalize(torch.cross(dx, dy, dim=-1), dim=-1)
|
| 44 |
+
# Assign the computed normal map to the output tensor
|
| 45 |
+
output[:, 1:-1, 1:-1, :] = normal_map
|
| 46 |
+
|
| 47 |
+
return output
|
src/utils/pose.py
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import numpy as np
|
| 3 |
+
from src.model.encoder.vggt.utils.rotation import mat_to_quat
|
| 4 |
+
from src.model.encoder.vggt.utils.geometry import closed_form_inverse_se3, unproject_depth_map_to_point_map
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def convert_pt3d_RT_to_opencv(Rot, Trans):
|
| 8 |
+
"""
|
| 9 |
+
Convert Point3D extrinsic matrices to OpenCV convention.
|
| 10 |
+
|
| 11 |
+
Args:
|
| 12 |
+
Rot: 3D rotation matrix in Point3D format
|
| 13 |
+
Trans: 3D translation vector in Point3D format
|
| 14 |
+
|
| 15 |
+
Returns:
|
| 16 |
+
extri_opencv: 3x4 extrinsic matrix in OpenCV format
|
| 17 |
+
"""
|
| 18 |
+
rot_pt3d = np.array(Rot)
|
| 19 |
+
trans_pt3d = np.array(Trans)
|
| 20 |
+
|
| 21 |
+
trans_pt3d[:2] *= -1
|
| 22 |
+
rot_pt3d[:, :2] *= -1
|
| 23 |
+
rot_pt3d = rot_pt3d.transpose(1, 0)
|
| 24 |
+
extri_opencv = np.hstack((rot_pt3d, trans_pt3d[:, None]))
|
| 25 |
+
return extri_opencv
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def build_pair_index(N, B=1):
|
| 29 |
+
"""
|
| 30 |
+
Build indices for all possible pairs of frames.
|
| 31 |
+
|
| 32 |
+
Args:
|
| 33 |
+
N: Number of frames
|
| 34 |
+
B: Batch size
|
| 35 |
+
|
| 36 |
+
Returns:
|
| 37 |
+
i1, i2: Indices for all possible pairs
|
| 38 |
+
"""
|
| 39 |
+
i1_, i2_ = torch.combinations(torch.arange(N), 2, with_replacement=False).unbind(-1)
|
| 40 |
+
i1, i2 = [(i[None] + torch.arange(B)[:, None] * N).reshape(-1) for i in [i1_, i2_]]
|
| 41 |
+
return i1, i2
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def rotation_angle(rot_gt, rot_pred, batch_size=None, eps=1e-15):
|
| 45 |
+
"""
|
| 46 |
+
Calculate rotation angle error between ground truth and predicted rotations.
|
| 47 |
+
|
| 48 |
+
Args:
|
| 49 |
+
rot_gt: Ground truth rotation matrices
|
| 50 |
+
rot_pred: Predicted rotation matrices
|
| 51 |
+
batch_size: Batch size for reshaping the result
|
| 52 |
+
eps: Small value to avoid numerical issues
|
| 53 |
+
|
| 54 |
+
Returns:
|
| 55 |
+
Rotation angle error in degrees
|
| 56 |
+
"""
|
| 57 |
+
q_pred = mat_to_quat(rot_pred)
|
| 58 |
+
q_gt = mat_to_quat(rot_gt)
|
| 59 |
+
|
| 60 |
+
loss_q = (1 - (q_pred * q_gt).sum(dim=1) ** 2).clamp(min=eps)
|
| 61 |
+
err_q = torch.arccos(1 - 2 * loss_q)
|
| 62 |
+
|
| 63 |
+
rel_rangle_deg = err_q * 180 / np.pi
|
| 64 |
+
|
| 65 |
+
if batch_size is not None:
|
| 66 |
+
rel_rangle_deg = rel_rangle_deg.reshape(batch_size, -1)
|
| 67 |
+
|
| 68 |
+
return rel_rangle_deg
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def translation_angle(tvec_gt, tvec_pred, batch_size=None, ambiguity=True):
|
| 72 |
+
"""
|
| 73 |
+
Calculate translation angle error between ground truth and predicted translations.
|
| 74 |
+
|
| 75 |
+
Args:
|
| 76 |
+
tvec_gt: Ground truth translation vectors
|
| 77 |
+
tvec_pred: Predicted translation vectors
|
| 78 |
+
batch_size: Batch size for reshaping the result
|
| 79 |
+
ambiguity: Whether to handle direction ambiguity
|
| 80 |
+
|
| 81 |
+
Returns:
|
| 82 |
+
Translation angle error in degrees
|
| 83 |
+
"""
|
| 84 |
+
rel_tangle_deg = compare_translation_by_angle(tvec_gt, tvec_pred)
|
| 85 |
+
rel_tangle_deg = rel_tangle_deg * 180.0 / np.pi
|
| 86 |
+
|
| 87 |
+
if ambiguity:
|
| 88 |
+
rel_tangle_deg = torch.min(rel_tangle_deg, (180 - rel_tangle_deg).abs())
|
| 89 |
+
|
| 90 |
+
if batch_size is not None:
|
| 91 |
+
rel_tangle_deg = rel_tangle_deg.reshape(batch_size, -1)
|
| 92 |
+
|
| 93 |
+
return rel_tangle_deg
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def compare_translation_by_angle(t_gt, t, eps=1e-15, default_err=1e6):
|
| 97 |
+
"""
|
| 98 |
+
Normalize the translation vectors and compute the angle between them.
|
| 99 |
+
|
| 100 |
+
Args:
|
| 101 |
+
t_gt: Ground truth translation vectors
|
| 102 |
+
t: Predicted translation vectors
|
| 103 |
+
eps: Small value to avoid division by zero
|
| 104 |
+
default_err: Default error value for invalid cases
|
| 105 |
+
|
| 106 |
+
Returns:
|
| 107 |
+
Angular error between translation vectors in radians
|
| 108 |
+
"""
|
| 109 |
+
t_norm = torch.norm(t, dim=1, keepdim=True)
|
| 110 |
+
t = t / (t_norm + eps)
|
| 111 |
+
|
| 112 |
+
t_gt_norm = torch.norm(t_gt, dim=1, keepdim=True)
|
| 113 |
+
t_gt = t_gt / (t_gt_norm + eps)
|
| 114 |
+
|
| 115 |
+
loss_t = torch.clamp_min(1.0 - torch.sum(t * t_gt, dim=1) ** 2, eps)
|
| 116 |
+
err_t = torch.acos(torch.sqrt(1 - loss_t))
|
| 117 |
+
|
| 118 |
+
err_t[torch.isnan(err_t) | torch.isinf(err_t)] = default_err
|
| 119 |
+
return err_t
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def calculate_auc(r_error, t_error, max_threshold=30, return_list=False):
|
| 123 |
+
"""
|
| 124 |
+
Calculate the Area Under the Curve (AUC) for the given error arrays using PyTorch.
|
| 125 |
+
|
| 126 |
+
Args:
|
| 127 |
+
r_error: torch.Tensor representing R error values (Degree)
|
| 128 |
+
t_error: torch.Tensor representing T error values (Degree)
|
| 129 |
+
max_threshold: Maximum threshold value for binning the histogram
|
| 130 |
+
return_list: Whether to return the normalized histogram as well
|
| 131 |
+
|
| 132 |
+
Returns:
|
| 133 |
+
AUC value, and optionally the normalized histogram
|
| 134 |
+
"""
|
| 135 |
+
error_matrix = torch.stack((r_error, t_error), dim=1)
|
| 136 |
+
max_errors, _ = torch.max(error_matrix, dim=1)
|
| 137 |
+
histogram = torch.histc(
|
| 138 |
+
max_errors, bins=max_threshold + 1, min=0, max=max_threshold
|
| 139 |
+
)
|
| 140 |
+
num_pairs = float(max_errors.size(0))
|
| 141 |
+
normalized_histogram = histogram / num_pairs
|
| 142 |
+
|
| 143 |
+
if return_list:
|
| 144 |
+
return (
|
| 145 |
+
torch.cumsum(normalized_histogram, dim=0).mean(),
|
| 146 |
+
normalized_histogram,
|
| 147 |
+
)
|
| 148 |
+
return torch.cumsum(normalized_histogram, dim=0).mean()
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def calculate_auc_np(r_error, t_error, max_threshold=30):
|
| 152 |
+
"""
|
| 153 |
+
Calculate the Area Under the Curve (AUC) for the given error arrays using NumPy.
|
| 154 |
+
|
| 155 |
+
Args:
|
| 156 |
+
r_error: numpy array representing R error values (Degree)
|
| 157 |
+
t_error: numpy array representing T error values (Degree)
|
| 158 |
+
max_threshold: Maximum threshold value for binning the histogram
|
| 159 |
+
|
| 160 |
+
Returns:
|
| 161 |
+
AUC value and the normalized histogram
|
| 162 |
+
"""
|
| 163 |
+
error_matrix = np.concatenate((r_error[:, None], t_error[:, None]), axis=1)
|
| 164 |
+
max_errors = np.max(error_matrix, axis=1)
|
| 165 |
+
bins = np.arange(max_threshold + 1)
|
| 166 |
+
histogram, _ = np.histogram(max_errors, bins=bins)
|
| 167 |
+
num_pairs = float(len(max_errors))
|
| 168 |
+
normalized_histogram = histogram.astype(float) / num_pairs
|
| 169 |
+
return np.mean(np.cumsum(normalized_histogram)), normalized_histogram
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def se3_to_relative_pose_error(pred_se3, gt_se3, num_frames):
|
| 173 |
+
"""
|
| 174 |
+
Compute rotation and translation errors between predicted and ground truth poses.
|
| 175 |
+
|
| 176 |
+
Args:
|
| 177 |
+
pred_se3: Predicted SE(3) transformations
|
| 178 |
+
gt_se3: Ground truth SE(3) transformations
|
| 179 |
+
num_frames: Number of frames
|
| 180 |
+
|
| 181 |
+
Returns:
|
| 182 |
+
Rotation and translation angle errors in degrees
|
| 183 |
+
"""
|
| 184 |
+
pair_idx_i1, pair_idx_i2 = build_pair_index(num_frames)
|
| 185 |
+
|
| 186 |
+
# Compute relative camera poses between pairs
|
| 187 |
+
# We use closed_form_inverse to avoid potential numerical loss by torch.inverse()
|
| 188 |
+
relative_pose_gt = closed_form_inverse_se3(gt_se3[pair_idx_i1]).bmm(
|
| 189 |
+
gt_se3[pair_idx_i2]
|
| 190 |
+
)
|
| 191 |
+
relative_pose_pred = closed_form_inverse_se3(pred_se3[pair_idx_i1]).bmm(
|
| 192 |
+
pred_se3[pair_idx_i2]
|
| 193 |
+
)
|
| 194 |
+
|
| 195 |
+
# Compute the difference in rotation and translation
|
| 196 |
+
rel_rangle_deg = rotation_angle(
|
| 197 |
+
relative_pose_gt[:, :3, :3], relative_pose_pred[:, :3, :3]
|
| 198 |
+
)
|
| 199 |
+
rel_tangle_deg = translation_angle(
|
| 200 |
+
relative_pose_gt[:, :3, 3], relative_pose_pred[:, :3, 3]
|
| 201 |
+
)
|
| 202 |
+
|
| 203 |
+
return rel_rangle_deg, rel_tangle_deg
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def align_to_first_camera(camera_poses):
|
| 207 |
+
"""
|
| 208 |
+
Align all camera poses to the first camera's coordinate frame.
|
| 209 |
+
|
| 210 |
+
Args:
|
| 211 |
+
camera_poses: Tensor of shape (N, 4, 4) containing camera poses as SE3 transformations
|
| 212 |
+
|
| 213 |
+
Returns:
|
| 214 |
+
Tensor of shape (N, 4, 4) containing aligned camera poses
|
| 215 |
+
"""
|
| 216 |
+
first_cam_extrinsic_inv = closed_form_inverse_se3(camera_poses[0][None])
|
| 217 |
+
aligned_poses = torch.matmul(camera_poses, first_cam_extrinsic_inv)
|
| 218 |
+
return aligned_poses
|
src/utils/render.py
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2022 Google LLC
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# https://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
import numpy as np
|
| 16 |
+
import os
|
| 17 |
+
import enum
|
| 18 |
+
import types
|
| 19 |
+
from typing import List, Mapping, Optional, Text, Tuple, Union
|
| 20 |
+
import copy
|
| 21 |
+
from PIL import Image
|
| 22 |
+
# import mediapy as media
|
| 23 |
+
from matplotlib import cm
|
| 24 |
+
from tqdm import tqdm
|
| 25 |
+
|
| 26 |
+
import torch
|
| 27 |
+
|
| 28 |
+
def normalize(x: np.ndarray) -> np.ndarray:
|
| 29 |
+
"""Normalization helper function."""
|
| 30 |
+
return x / np.linalg.norm(x)
|
| 31 |
+
|
| 32 |
+
def pad_poses(p: np.ndarray) -> np.ndarray:
|
| 33 |
+
"""Pad [..., 3, 4] pose matrices with a homogeneous bottom row [0,0,0,1]."""
|
| 34 |
+
bottom = np.broadcast_to([0, 0, 0, 1.], p[..., :1, :4].shape)
|
| 35 |
+
return np.concatenate([p[..., :3, :4], bottom], axis=-2)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def unpad_poses(p: np.ndarray) -> np.ndarray:
|
| 39 |
+
"""Remove the homogeneous bottom row from [..., 4, 4] pose matrices."""
|
| 40 |
+
return p[..., :3, :4]
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def recenter_poses(poses: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
| 44 |
+
"""Recenter poses around the origin."""
|
| 45 |
+
cam2world = average_pose(poses)
|
| 46 |
+
transform = np.linalg.inv(pad_poses(cam2world))
|
| 47 |
+
poses = transform @ pad_poses(poses)
|
| 48 |
+
return unpad_poses(poses), transform
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def average_pose(poses: np.ndarray) -> np.ndarray:
|
| 52 |
+
"""New pose using average position, z-axis, and up vector of input poses."""
|
| 53 |
+
position = poses[:, :3, 3].mean(0)
|
| 54 |
+
z_axis = poses[:, :3, 2].mean(0)
|
| 55 |
+
up = poses[:, :3, 1].mean(0)
|
| 56 |
+
cam2world = viewmatrix(z_axis, up, position)
|
| 57 |
+
return cam2world
|
| 58 |
+
|
| 59 |
+
def viewmatrix(lookdir: np.ndarray, up: np.ndarray,
|
| 60 |
+
position: np.ndarray) -> np.ndarray:
|
| 61 |
+
"""Construct lookat view matrix."""
|
| 62 |
+
vec2 = normalize(lookdir)
|
| 63 |
+
vec0 = normalize(np.cross(up, vec2))
|
| 64 |
+
vec1 = normalize(np.cross(vec2, vec0))
|
| 65 |
+
m = np.stack([vec0, vec1, vec2, position], axis=1)
|
| 66 |
+
return m
|
| 67 |
+
|
| 68 |
+
def focus_point_fn(poses: np.ndarray) -> np.ndarray:
|
| 69 |
+
"""Calculate nearest point to all focal axes in poses."""
|
| 70 |
+
directions, origins = poses[:, :3, 2:3], poses[:, :3, 3:4]
|
| 71 |
+
m = np.eye(3) - directions * np.transpose(directions, [0, 2, 1])
|
| 72 |
+
mt_m = np.transpose(m, [0, 2, 1]) @ m
|
| 73 |
+
focus_pt = np.linalg.inv(mt_m.mean(0)) @ (mt_m @ origins).mean(0)[:, 0]
|
| 74 |
+
return focus_pt
|
| 75 |
+
|
| 76 |
+
def transform_poses_pca(poses: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
| 77 |
+
"""Transforms poses so principal components lie on XYZ axes.
|
| 78 |
+
|
| 79 |
+
Args:
|
| 80 |
+
poses: a (N, 3, 4) array containing the cameras' camera to world transforms.
|
| 81 |
+
|
| 82 |
+
Returns:
|
| 83 |
+
A tuple (poses, transform), with the transformed poses and the applied
|
| 84 |
+
camera_to_world transforms.
|
| 85 |
+
"""
|
| 86 |
+
t = poses[:, :3, 3]
|
| 87 |
+
t_mean = t.mean(axis=0)
|
| 88 |
+
t = t - t_mean
|
| 89 |
+
|
| 90 |
+
eigval, eigvec = np.linalg.eig(t.T @ t)
|
| 91 |
+
# Sort eigenvectors in order of largest to smallest eigenvalue.
|
| 92 |
+
inds = np.argsort(eigval)[::-1]
|
| 93 |
+
eigvec = eigvec[:, inds]
|
| 94 |
+
rot = eigvec.T
|
| 95 |
+
if np.linalg.det(rot) < 0:
|
| 96 |
+
rot = np.diag(np.array([1, 1, -1])) @ rot
|
| 97 |
+
|
| 98 |
+
transform = np.concatenate([rot, rot @ -t_mean[:, None]], -1)
|
| 99 |
+
poses_recentered = unpad_poses(transform @ pad_poses(poses))
|
| 100 |
+
transform = np.concatenate([transform, np.eye(4)[3:]], axis=0)
|
| 101 |
+
|
| 102 |
+
# Flip coordinate system if z component of y-axis is negative
|
| 103 |
+
if poses_recentered.mean(axis=0)[2, 1] < 0:
|
| 104 |
+
poses_recentered = np.diag(np.array([1, -1, -1])) @ poses_recentered
|
| 105 |
+
transform = np.diag(np.array([1, -1, -1, 1])) @ transform
|
| 106 |
+
|
| 107 |
+
return poses_recentered, transform
|
| 108 |
+
# points = np.random.rand(3,100)
|
| 109 |
+
# points_h = np.concatenate((points,np.ones_like(points[:1])), axis=0)
|
| 110 |
+
# (poses_recentered @ points_h)[0]
|
| 111 |
+
# (transform @ pad_poses(poses) @ points_h)[0,:3]
|
| 112 |
+
# import pdb; pdb.set_trace()
|
| 113 |
+
|
| 114 |
+
# # Just make sure it's it in the [-1, 1]^3 cube
|
| 115 |
+
# scale_factor = 1. / np.max(np.abs(poses_recentered[:, :3, 3]))
|
| 116 |
+
# poses_recentered[:, :3, 3] *= scale_factor
|
| 117 |
+
# transform = np.diag(np.array([scale_factor] * 3 + [1])) @ transform
|
| 118 |
+
|
| 119 |
+
# return poses_recentered, transform
|
| 120 |
+
|
| 121 |
+
def generate_ellipse_path(poses: np.ndarray,
|
| 122 |
+
n_frames: int = 120,
|
| 123 |
+
const_speed: bool = True,
|
| 124 |
+
z_variation: float = 0.,
|
| 125 |
+
z_phase: float = 0.) -> np.ndarray:
|
| 126 |
+
"""Generate an elliptical render path based on the given poses."""
|
| 127 |
+
# Calculate the focal point for the path (cameras point toward this).
|
| 128 |
+
center = focus_point_fn(poses)
|
| 129 |
+
# Path height sits at z=0 (in middle of zero-mean capture pattern).
|
| 130 |
+
offset = np.array([center[0], center[1], 0])
|
| 131 |
+
|
| 132 |
+
# Calculate scaling for ellipse axes based on input camera positions.
|
| 133 |
+
sc = np.percentile(np.abs(poses[:, :3, 3] - offset), 90, axis=0)
|
| 134 |
+
# Use ellipse that is symmetric about the focal point in xy.
|
| 135 |
+
low = -sc + offset
|
| 136 |
+
high = sc + offset
|
| 137 |
+
# Optional height variation need not be symmetric
|
| 138 |
+
z_low = np.percentile((poses[:, :3, 3]), 10, axis=0)
|
| 139 |
+
z_high = np.percentile((poses[:, :3, 3]), 90, axis=0)
|
| 140 |
+
|
| 141 |
+
def get_positions(theta):
|
| 142 |
+
# Interpolate between bounds with trig functions to get ellipse in x-y.
|
| 143 |
+
# Optionally also interpolate in z to change camera height along path.
|
| 144 |
+
return np.stack([
|
| 145 |
+
low[0] + (high - low)[0] * (np.cos(theta) * .5 + .5),
|
| 146 |
+
low[1] + (high - low)[1] * (np.sin(theta) * .5 + .5),
|
| 147 |
+
z_variation * (z_low[2] + (z_high - z_low)[2] *
|
| 148 |
+
(np.cos(theta + 2 * np.pi * z_phase) * .5 + .5)),
|
| 149 |
+
], -1)
|
| 150 |
+
|
| 151 |
+
theta = np.linspace(0, 2. * np.pi, n_frames + 1, endpoint=True)
|
| 152 |
+
positions = get_positions(theta)
|
| 153 |
+
|
| 154 |
+
#if const_speed:
|
| 155 |
+
|
| 156 |
+
# # Resample theta angles so that the velocity is closer to constant.
|
| 157 |
+
# lengths = np.linalg.norm(positions[1:] - positions[:-1], axis=-1)
|
| 158 |
+
# theta = stepfun.sample(None, theta, np.log(lengths), n_frames + 1)
|
| 159 |
+
# positions = get_positions(theta)
|
| 160 |
+
|
| 161 |
+
# Throw away duplicated last position.
|
| 162 |
+
positions = positions[:-1]
|
| 163 |
+
|
| 164 |
+
# Set path's up vector to axis closest to average of input pose up vectors.
|
| 165 |
+
avg_up = poses[:, :3, 1].mean(0)
|
| 166 |
+
avg_up = avg_up / np.linalg.norm(avg_up)
|
| 167 |
+
ind_up = np.argmax(np.abs(avg_up))
|
| 168 |
+
up = np.eye(3)[ind_up] * np.sign(avg_up[ind_up])
|
| 169 |
+
|
| 170 |
+
return np.stack([viewmatrix(p - center, up, p) for p in positions])
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def generate_path(viewpoint_cameras, n_frames=480):
|
| 174 |
+
# c2ws = np.array([np.linalg.inv(np.asarray((cam.world_view_transform.T).cpu().numpy())) for cam in viewpoint_cameras])
|
| 175 |
+
c2ws = viewpoint_cameras.cpu().numpy()
|
| 176 |
+
pose = c2ws[:,:3,:] @ np.diag([1, -1, -1, 1])
|
| 177 |
+
pose_recenter, colmap_to_world_transform = transform_poses_pca(pose)
|
| 178 |
+
|
| 179 |
+
# generate new poses
|
| 180 |
+
new_poses = generate_ellipse_path(poses=pose_recenter, n_frames=n_frames)
|
| 181 |
+
# warp back to orignal scale
|
| 182 |
+
new_poses = np.linalg.inv(colmap_to_world_transform) @ pad_poses(new_poses)
|
| 183 |
+
|
| 184 |
+
return new_poses
|
| 185 |
+
|
| 186 |
+
# traj = []
|
| 187 |
+
# for c2w in new_poses:
|
| 188 |
+
# c2w = c2w @ np.diag([1, -1, -1, 1])
|
| 189 |
+
# cam = copy.deepcopy(viewpoint_cameras[0])
|
| 190 |
+
# cam.image_height = int(cam.image_height / 2) * 2
|
| 191 |
+
# cam.image_width = int(cam.image_width / 2) * 2
|
| 192 |
+
# cam.world_view_transform = torch.from_numpy(np.linalg.inv(c2w).T).float().cuda()
|
| 193 |
+
# cam.full_proj_transform = (cam.world_view_transform.unsqueeze(0).bmm(cam.projection_matrix.unsqueeze(0))).squeeze(0)
|
| 194 |
+
# cam.camera_center = cam.world_view_transform.inverse()[3, :3]
|
| 195 |
+
# traj.append(cam)
|
| 196 |
+
|
| 197 |
+
# return traj
|
| 198 |
+
|
| 199 |
+
def load_img(pth: str) -> np.ndarray:
|
| 200 |
+
"""Load an image and cast to float32."""
|
| 201 |
+
with open(pth, 'rb') as f:
|
| 202 |
+
image = np.array(Image.open(f), dtype=np.float32)
|
| 203 |
+
return image
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def create_videos(base_dir, input_dir, out_name, num_frames=480):
|
| 207 |
+
"""Creates videos out of the images saved to disk."""
|
| 208 |
+
# Last two parts of checkpoint path are experiment name and scene name.
|
| 209 |
+
video_prefix = f'{out_name}'
|
| 210 |
+
|
| 211 |
+
zpad = max(5, len(str(num_frames - 1)))
|
| 212 |
+
idx_to_str = lambda idx: str(idx).zfill(zpad)
|
| 213 |
+
|
| 214 |
+
os.makedirs(base_dir, exist_ok=True)
|
| 215 |
+
render_dist_curve_fn = np.log
|
| 216 |
+
|
| 217 |
+
# Load one example frame to get image shape and depth range.
|
| 218 |
+
depth_file = os.path.join(input_dir, 'vis', f'depth_{idx_to_str(0)}.tiff')
|
| 219 |
+
depth_frame = load_img(depth_file)
|
| 220 |
+
shape = depth_frame.shape
|
| 221 |
+
p = 3
|
| 222 |
+
distance_limits = np.percentile(depth_frame.flatten(), [p, 100 - p])
|
| 223 |
+
lo, hi = [render_dist_curve_fn(x) for x in distance_limits]
|
| 224 |
+
print(f'Video shape is {shape[:2]}')
|
| 225 |
+
|
| 226 |
+
video_kwargs = {
|
| 227 |
+
'shape': shape[:2],
|
| 228 |
+
'codec': 'h264',
|
| 229 |
+
'fps': 60,
|
| 230 |
+
'crf': 18,
|
| 231 |
+
}
|
| 232 |
+
|
| 233 |
+
for k in ['depth', 'normal', 'color']:
|
| 234 |
+
video_file = os.path.join(base_dir, f'{video_prefix}_{k}.mp4')
|
| 235 |
+
input_format = 'gray' if k == 'alpha' else 'rgb'
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
file_ext = 'png' if k in ['color', 'normal'] else 'tiff'
|
| 239 |
+
idx = 0
|
| 240 |
+
|
| 241 |
+
if k == 'color':
|
| 242 |
+
file0 = os.path.join(input_dir, 'renders', f'{idx_to_str(0)}.{file_ext}')
|
| 243 |
+
else:
|
| 244 |
+
file0 = os.path.join(input_dir, 'vis', f'{k}_{idx_to_str(0)}.{file_ext}')
|
| 245 |
+
|
| 246 |
+
if not os.path.exists(file0):
|
| 247 |
+
print(f'Images missing for tag {k}')
|
| 248 |
+
continue
|
| 249 |
+
print(f'Making video {video_file}...')
|
| 250 |
+
with media.VideoWriter(
|
| 251 |
+
video_file, **video_kwargs, input_format=input_format) as writer:
|
| 252 |
+
for idx in tqdm(range(num_frames)):
|
| 253 |
+
# img_file = os.path.join(input_dir, f'{k}_{idx_to_str(idx)}.{file_ext}')
|
| 254 |
+
if k == 'color':
|
| 255 |
+
img_file = os.path.join(input_dir, 'renders', f'{idx_to_str(idx)}.{file_ext}')
|
| 256 |
+
else:
|
| 257 |
+
img_file = os.path.join(input_dir, 'vis', f'{k}_{idx_to_str(idx)}.{file_ext}')
|
| 258 |
+
|
| 259 |
+
if not os.path.exists(img_file):
|
| 260 |
+
ValueError(f'Image file {img_file} does not exist.')
|
| 261 |
+
img = load_img(img_file)
|
| 262 |
+
if k in ['color', 'normal']:
|
| 263 |
+
img = img / 255.
|
| 264 |
+
elif k.startswith('depth'):
|
| 265 |
+
img = render_dist_curve_fn(img)
|
| 266 |
+
img = np.clip((img - np.minimum(lo, hi)) / np.abs(hi - lo), 0, 1)
|
| 267 |
+
img = cm.get_cmap('turbo')(img)[..., :3]
|
| 268 |
+
|
| 269 |
+
frame = (np.clip(np.nan_to_num(img), 0., 1.) * 255.).astype(np.uint8)
|
| 270 |
+
writer.add_image(frame)
|
| 271 |
+
idx += 1
|
| 272 |
+
|
| 273 |
+
def save_img_u8(img, pth):
|
| 274 |
+
"""Save an image (probably RGB) in [0, 1] to disk as a uint8 PNG."""
|
| 275 |
+
with open(pth, 'wb') as f:
|
| 276 |
+
Image.fromarray(
|
| 277 |
+
(np.clip(np.nan_to_num(img), 0., 1.) * 255.).astype(np.uint8)).save(
|
| 278 |
+
f, 'PNG')
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
def save_img_f32(depthmap, pth):
|
| 282 |
+
"""Save an image (probably a depthmap) to disk as a float32 TIFF."""
|
| 283 |
+
with open(pth, 'wb') as f:
|
| 284 |
+
Image.fromarray(np.nan_to_num(depthmap).astype(np.float32)).save(f, 'TIFF')
|
src/utils/tensor_to_pycolmap.py
ADDED
|
@@ -0,0 +1,320 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
#
|
| 4 |
+
# This source code is licensed under the license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
import torch.nn as nn
|
| 10 |
+
import torch.nn.functional as F
|
| 11 |
+
|
| 12 |
+
import numpy as np
|
| 13 |
+
import pycolmap
|
| 14 |
+
|
| 15 |
+
# TODO: frame_idx should start from 1 instead of 0 in colmap
|
| 16 |
+
def batch_matrix_to_pycolmap(
|
| 17 |
+
points3d,
|
| 18 |
+
extrinsics,
|
| 19 |
+
intrinsics,
|
| 20 |
+
tracks,
|
| 21 |
+
image_size,
|
| 22 |
+
masks=None,
|
| 23 |
+
max_reproj_error=None,
|
| 24 |
+
max_points3D_val=3000,
|
| 25 |
+
shared_camera=False,
|
| 26 |
+
camera_type="SIMPLE_PINHOLE",
|
| 27 |
+
extra_params=None,
|
| 28 |
+
):
|
| 29 |
+
"""
|
| 30 |
+
Convert Batched Pytorch Tensors to PyCOLMAP
|
| 31 |
+
|
| 32 |
+
Check https://github.com/colmap/pycolmap for more details about its format
|
| 33 |
+
"""
|
| 34 |
+
|
| 35 |
+
# points3d: Px3
|
| 36 |
+
# extrinsics: Nx3x4
|
| 37 |
+
# intrinsics: Nx3x3
|
| 38 |
+
# tracks: NxPx2
|
| 39 |
+
# masks: NxP
|
| 40 |
+
# image_size: 2, assume all the frames have been padded to the same size
|
| 41 |
+
# where N is the number of frames and P is the number of tracks
|
| 42 |
+
|
| 43 |
+
N, P, _ = tracks.shape
|
| 44 |
+
assert len(extrinsics) == N
|
| 45 |
+
assert len(intrinsics) == N
|
| 46 |
+
assert len(points3d) == P
|
| 47 |
+
assert image_size.shape[0] == 2
|
| 48 |
+
|
| 49 |
+
projected_points_2d, projected_points_cam = project_3D_points(points3d, extrinsics, intrinsics, return_points_cam=True)
|
| 50 |
+
projected_diff = (projected_points_2d - tracks).norm(dim=-1)
|
| 51 |
+
projected_points_2d[projected_points_cam[:, -1] <= 0] = 1e6
|
| 52 |
+
reproj_mask = projected_diff < max_reproj_error
|
| 53 |
+
|
| 54 |
+
if masks is not None:
|
| 55 |
+
masks = torch.logical_and(masks, reproj_mask)
|
| 56 |
+
else:
|
| 57 |
+
masks = reproj_mask
|
| 58 |
+
|
| 59 |
+
extrinsics = extrinsics.cpu().numpy()
|
| 60 |
+
intrinsics = intrinsics.cpu().numpy()
|
| 61 |
+
|
| 62 |
+
if extra_params is not None:
|
| 63 |
+
extra_params = extra_params.cpu().numpy()
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
tracks = tracks.cpu().numpy()
|
| 67 |
+
points3d = points3d.cpu().numpy()
|
| 68 |
+
image_size = image_size.cpu().numpy()
|
| 69 |
+
|
| 70 |
+
# Reconstruction object, following the format of PyCOLMAP/COLMAP
|
| 71 |
+
reconstruction = pycolmap.Reconstruction()
|
| 72 |
+
|
| 73 |
+
masks = masks.cpu().numpy()
|
| 74 |
+
|
| 75 |
+
inlier_num = masks.sum(0)
|
| 76 |
+
valid_mask = inlier_num >= 2 # a track is invalid if without two inliers
|
| 77 |
+
valid_idx = np.nonzero(valid_mask)[0]
|
| 78 |
+
|
| 79 |
+
# Only add 3D points that have sufficient 2D points
|
| 80 |
+
for vidx in valid_idx:
|
| 81 |
+
reconstruction.add_point3D(
|
| 82 |
+
points3d[vidx], pycolmap.Track(), np.zeros(3)
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
num_points3D = len(valid_idx)
|
| 86 |
+
|
| 87 |
+
camera = None
|
| 88 |
+
# frame idx
|
| 89 |
+
for fidx in range(N):
|
| 90 |
+
# set camera
|
| 91 |
+
if camera is None or (not shared_camera):
|
| 92 |
+
if camera_type == "SIMPLE_RADIAL":
|
| 93 |
+
focal = (intrinsics[fidx][0, 0] + intrinsics[fidx][1, 1]) / 2
|
| 94 |
+
pycolmap_intri = np.array(
|
| 95 |
+
[
|
| 96 |
+
focal,
|
| 97 |
+
intrinsics[fidx][0, 2],
|
| 98 |
+
intrinsics[fidx][1, 2],
|
| 99 |
+
extra_params[fidx][0],
|
| 100 |
+
]
|
| 101 |
+
)
|
| 102 |
+
elif camera_type == "SIMPLE_PINHOLE":
|
| 103 |
+
focal = (intrinsics[fidx][0, 0] + intrinsics[fidx][1, 1]) / 2
|
| 104 |
+
pycolmap_intri = np.array(
|
| 105 |
+
[
|
| 106 |
+
focal,
|
| 107 |
+
intrinsics[fidx][0, 2],
|
| 108 |
+
intrinsics[fidx][1, 2],
|
| 109 |
+
]
|
| 110 |
+
)
|
| 111 |
+
else:
|
| 112 |
+
raise ValueError(
|
| 113 |
+
f"Camera type {camera_type} is not supported yet"
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
camera = pycolmap.Camera(
|
| 117 |
+
model=camera_type,
|
| 118 |
+
width=image_size[0],
|
| 119 |
+
height=image_size[1],
|
| 120 |
+
params=pycolmap_intri,
|
| 121 |
+
camera_id=fidx,
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
# add camera
|
| 125 |
+
reconstruction.add_camera(camera)
|
| 126 |
+
|
| 127 |
+
# set image
|
| 128 |
+
cam_from_world = pycolmap.Rigid3d(
|
| 129 |
+
pycolmap.Rotation3d(extrinsics[fidx][:3, :3]),
|
| 130 |
+
extrinsics[fidx][:3, 3],
|
| 131 |
+
) # Rot and Trans
|
| 132 |
+
image = pycolmap.Image(
|
| 133 |
+
id=fidx,
|
| 134 |
+
name=f"image_{fidx}",
|
| 135 |
+
camera_id=camera.camera_id,
|
| 136 |
+
cam_from_world=cam_from_world,
|
| 137 |
+
)
|
| 138 |
+
|
| 139 |
+
points2D_list = []
|
| 140 |
+
|
| 141 |
+
point2D_idx = 0
|
| 142 |
+
# NOTE point3D_id start by 1
|
| 143 |
+
for point3D_id in range(1, num_points3D + 1):
|
| 144 |
+
original_track_idx = valid_idx[point3D_id - 1]
|
| 145 |
+
|
| 146 |
+
if (
|
| 147 |
+
reconstruction.points3D[point3D_id].xyz < max_points3D_val
|
| 148 |
+
).all():
|
| 149 |
+
if masks[fidx][original_track_idx]:
|
| 150 |
+
# It seems we don't need +0.5 for BA
|
| 151 |
+
point2D_xy = tracks[fidx][original_track_idx]
|
| 152 |
+
# Please note when adding the Point2D object
|
| 153 |
+
# It not only requires the 2D xy location, but also the id to 3D point
|
| 154 |
+
points2D_list.append(
|
| 155 |
+
pycolmap.Point2D(point2D_xy, point3D_id)
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
# add element
|
| 159 |
+
track = reconstruction.points3D[point3D_id].track
|
| 160 |
+
track.add_element(fidx, point2D_idx)
|
| 161 |
+
point2D_idx += 1
|
| 162 |
+
|
| 163 |
+
assert point2D_idx == len(points2D_list)
|
| 164 |
+
|
| 165 |
+
try:
|
| 166 |
+
image.points2D = pycolmap.ListPoint2D(points2D_list)
|
| 167 |
+
image.registered = True
|
| 168 |
+
except:
|
| 169 |
+
print(f"frame {fidx} is out of BA")
|
| 170 |
+
image.registered = False
|
| 171 |
+
|
| 172 |
+
# add image
|
| 173 |
+
reconstruction.add_image(image)
|
| 174 |
+
|
| 175 |
+
return reconstruction
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def pycolmap_to_batch_matrix(
|
| 179 |
+
reconstruction, device="cuda", camera_type="SIMPLE_PINHOLE"
|
| 180 |
+
):
|
| 181 |
+
"""
|
| 182 |
+
Convert a PyCOLMAP Reconstruction Object to batched PyTorch tensors.
|
| 183 |
+
|
| 184 |
+
Args:
|
| 185 |
+
reconstruction (pycolmap.Reconstruction): The reconstruction object from PyCOLMAP.
|
| 186 |
+
device (str): The device to place the tensors on (default: "cuda").
|
| 187 |
+
camera_type (str): The type of camera model used (default: "SIMPLE_PINHOLE").
|
| 188 |
+
|
| 189 |
+
Returns:
|
| 190 |
+
tuple: A tuple containing points3D, extrinsics, intrinsics, and optionally extra_params.
|
| 191 |
+
"""
|
| 192 |
+
|
| 193 |
+
num_images = len(reconstruction.images)
|
| 194 |
+
max_points3D_id = max(reconstruction.point3D_ids())
|
| 195 |
+
points3D = np.zeros((max_points3D_id, 3))
|
| 196 |
+
|
| 197 |
+
for point3D_id in reconstruction.points3D:
|
| 198 |
+
points3D[point3D_id - 1] = reconstruction.points3D[point3D_id].xyz
|
| 199 |
+
points3D = torch.from_numpy(points3D).to(device)
|
| 200 |
+
|
| 201 |
+
extrinsics = []
|
| 202 |
+
intrinsics = []
|
| 203 |
+
|
| 204 |
+
extra_params = [] if camera_type == "SIMPLE_RADIAL" else None
|
| 205 |
+
|
| 206 |
+
for i in range(num_images):
|
| 207 |
+
# Extract and append extrinsics
|
| 208 |
+
pyimg = reconstruction.images[i]
|
| 209 |
+
pycam = reconstruction.cameras[pyimg.camera_id]
|
| 210 |
+
matrix = pyimg.cam_from_world.matrix()
|
| 211 |
+
extrinsics.append(matrix)
|
| 212 |
+
|
| 213 |
+
# Extract and append intrinsics
|
| 214 |
+
calibration_matrix = pycam.calibration_matrix()
|
| 215 |
+
intrinsics.append(calibration_matrix)
|
| 216 |
+
|
| 217 |
+
if camera_type == "SIMPLE_RADIAL":
|
| 218 |
+
extra_params.append(pycam.params[-1])
|
| 219 |
+
|
| 220 |
+
# Convert lists to torch tensors
|
| 221 |
+
extrinsics = torch.from_numpy(np.stack(extrinsics)).to(device)
|
| 222 |
+
|
| 223 |
+
intrinsics = torch.from_numpy(np.stack(intrinsics)).to(device)
|
| 224 |
+
|
| 225 |
+
if camera_type == "SIMPLE_RADIAL":
|
| 226 |
+
extra_params = torch.from_numpy(np.stack(extra_params)).to(device)
|
| 227 |
+
extra_params = extra_params[:, None]
|
| 228 |
+
|
| 229 |
+
return points3D, extrinsics, intrinsics, extra_params
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
def project_3D_points(
|
| 236 |
+
points3D,
|
| 237 |
+
extrinsics,
|
| 238 |
+
intrinsics=None,
|
| 239 |
+
extra_params=None,
|
| 240 |
+
return_points_cam=False,
|
| 241 |
+
default=0,
|
| 242 |
+
only_points_cam=False,
|
| 243 |
+
):
|
| 244 |
+
"""
|
| 245 |
+
Transforms 3D points to 2D using extrinsic and intrinsic parameters.
|
| 246 |
+
Args:
|
| 247 |
+
points3D (torch.Tensor): 3D points of shape Px3.
|
| 248 |
+
extrinsics (torch.Tensor): Extrinsic parameters of shape Bx3x4.
|
| 249 |
+
intrinsics (torch.Tensor): Intrinsic parameters of shape Bx3x3.
|
| 250 |
+
extra_params (torch.Tensor): Extra parameters of shape BxN, which is used for radial distortion.
|
| 251 |
+
Returns:
|
| 252 |
+
torch.Tensor: Transformed 2D points of shape BxNx2.
|
| 253 |
+
"""
|
| 254 |
+
with torch.cuda.amp.autocast(dtype=torch.double):
|
| 255 |
+
N = points3D.shape[0] # Number of points
|
| 256 |
+
B = extrinsics.shape[0] # Batch size, i.e., number of cameras
|
| 257 |
+
points3D_homogeneous = torch.cat(
|
| 258 |
+
[points3D, torch.ones_like(points3D[..., 0:1])], dim=1
|
| 259 |
+
) # Nx4
|
| 260 |
+
# Reshape for batch processing
|
| 261 |
+
points3D_homogeneous = points3D_homogeneous.unsqueeze(0).expand(
|
| 262 |
+
B, -1, -1
|
| 263 |
+
) # BxNx4
|
| 264 |
+
|
| 265 |
+
# Step 1: Apply extrinsic parameters
|
| 266 |
+
# Transform 3D points to camera coordinate system for all cameras
|
| 267 |
+
points_cam = torch.bmm(
|
| 268 |
+
extrinsics, points3D_homogeneous.transpose(-1, -2)
|
| 269 |
+
)
|
| 270 |
+
|
| 271 |
+
if only_points_cam:
|
| 272 |
+
return points_cam
|
| 273 |
+
|
| 274 |
+
# Step 2: Apply intrinsic parameters and (optional) distortion
|
| 275 |
+
points2D = img_from_cam(intrinsics, points_cam, extra_params)
|
| 276 |
+
|
| 277 |
+
if return_points_cam:
|
| 278 |
+
return points2D, points_cam
|
| 279 |
+
return points2D
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
def img_from_cam(intrinsics, points_cam, extra_params=None, default=0.0):
|
| 283 |
+
"""
|
| 284 |
+
Applies intrinsic parameters and optional distortion to the given 3D points.
|
| 285 |
+
|
| 286 |
+
Args:
|
| 287 |
+
intrinsics (torch.Tensor): Intrinsic camera parameters of shape Bx3x3.
|
| 288 |
+
points_cam (torch.Tensor): 3D points in camera coordinates of shape Bx3xN.
|
| 289 |
+
extra_params (torch.Tensor, optional): Distortion parameters of shape BxN, where N can be 1, 2, or 4.
|
| 290 |
+
default (float, optional): Default value to replace NaNs in the output.
|
| 291 |
+
|
| 292 |
+
Returns:
|
| 293 |
+
points2D (torch.Tensor): 2D points in pixel coordinates of shape BxNx2.
|
| 294 |
+
"""
|
| 295 |
+
|
| 296 |
+
# Normalize by the third coordinate (homogeneous division)
|
| 297 |
+
points_cam = points_cam / points_cam[:, 2:3, :]
|
| 298 |
+
# Extract uv
|
| 299 |
+
uv = points_cam[:, :2, :]
|
| 300 |
+
|
| 301 |
+
# Apply distortion if extra_params are provided
|
| 302 |
+
if extra_params is not None:
|
| 303 |
+
uu, vv = apply_distortion(extra_params, uv[:, 0], uv[:, 1])
|
| 304 |
+
uv = torch.stack([uu, vv], dim=1)
|
| 305 |
+
|
| 306 |
+
# Prepare points_cam for batch matrix multiplication
|
| 307 |
+
points_cam_homo = torch.cat(
|
| 308 |
+
(uv, torch.ones_like(uv[:, :1, :])), dim=1
|
| 309 |
+
) # Bx3xN
|
| 310 |
+
# Apply intrinsic parameters using batch matrix multiplication
|
| 311 |
+
points2D_homo = torch.bmm(intrinsics, points_cam_homo) # Bx3xN
|
| 312 |
+
|
| 313 |
+
# Extract x and y coordinates
|
| 314 |
+
points2D = points2D_homo[:, :2, :] # Bx2xN
|
| 315 |
+
|
| 316 |
+
# Replace NaNs with default value
|
| 317 |
+
points2D = torch.nan_to_num(points2D, nan=default)
|
| 318 |
+
|
| 319 |
+
return points2D.transpose(1, 2) # BxNx2
|
| 320 |
+
|
src/utils/transforms.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
|
| 4 |
+
# This source code is licensed under the license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
# Copyright (C) 2024-present Naver Corporation. All rights reserved.
|
| 8 |
+
# Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
|
| 9 |
+
#
|
| 10 |
+
# --------------------------------------------------------
|
| 11 |
+
# DUST3R default transforms
|
| 12 |
+
# --------------------------------------------------------
|
| 13 |
+
import torchvision.transforms as tvf
|
| 14 |
+
|
| 15 |
+
from model.dataset.utils.image import ImgNorm
|
| 16 |
+
|
| 17 |
+
# define the standard image transforms
|
| 18 |
+
ColorJitter = tvf.Compose([tvf.ColorJitter(0.5, 0.5, 0.5, 0.1), ImgNorm])
|
src/utils/viz.py
ADDED
|
@@ -0,0 +1,377 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
# All rights reserved.
|
| 3 |
+
|
| 4 |
+
# This source code is licensed under the license found in the
|
| 5 |
+
# LICENSE file in the root directory of this source tree.
|
| 6 |
+
|
| 7 |
+
# Copyright (C) 2024-present Naver Corporation. All rights reserved.
|
| 8 |
+
# Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
|
| 9 |
+
#
|
| 10 |
+
# --------------------------------------------------------
|
| 11 |
+
# Visualization utilities using trimesh
|
| 12 |
+
# --------------------------------------------------------
|
| 13 |
+
import numpy as np
|
| 14 |
+
import PIL.Image
|
| 15 |
+
import torch
|
| 16 |
+
from scipy.spatial.transform import Rotation
|
| 17 |
+
|
| 18 |
+
from model.dataset.utils.device import to_numpy
|
| 19 |
+
from model.dataset.utils.geometry import geotrf, get_med_dist_between_poses
|
| 20 |
+
from model.dataset.utils.image import rgb
|
| 21 |
+
|
| 22 |
+
try:
|
| 23 |
+
import trimesh
|
| 24 |
+
except ImportError:
|
| 25 |
+
print("/!\\ module trimesh is not installed, cannot visualize results /!\\")
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def cat_3d(vecs):
|
| 29 |
+
if isinstance(vecs, (np.ndarray, torch.Tensor)):
|
| 30 |
+
vecs = [vecs]
|
| 31 |
+
return np.concatenate([p.reshape(-1, 3) for p in to_numpy(vecs)])
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def show_raw_pointcloud(pts3d, colors, point_size=2):
|
| 35 |
+
scene = trimesh.Scene()
|
| 36 |
+
|
| 37 |
+
pct = trimesh.PointCloud(cat_3d(pts3d), colors=cat_3d(colors))
|
| 38 |
+
scene.add_geometry(pct)
|
| 39 |
+
|
| 40 |
+
scene.show(line_settings={"point_size": point_size})
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def pts3d_to_trimesh(img, pts3d, valid=None):
|
| 44 |
+
H, W, THREE = img.shape
|
| 45 |
+
assert THREE == 3
|
| 46 |
+
assert img.shape == pts3d.shape
|
| 47 |
+
|
| 48 |
+
vertices = pts3d.reshape(-1, 3)
|
| 49 |
+
|
| 50 |
+
# make squares: each pixel == 2 triangles
|
| 51 |
+
idx = np.arange(len(vertices)).reshape(H, W)
|
| 52 |
+
idx1 = idx[:-1, :-1].ravel() # top-left corner
|
| 53 |
+
idx2 = idx[:-1, +1:].ravel() # right-left corner
|
| 54 |
+
idx3 = idx[+1:, :-1].ravel() # bottom-left corner
|
| 55 |
+
idx4 = idx[+1:, +1:].ravel() # bottom-right corner
|
| 56 |
+
faces = np.concatenate(
|
| 57 |
+
(
|
| 58 |
+
np.c_[idx1, idx2, idx3],
|
| 59 |
+
np.c_[
|
| 60 |
+
idx3, idx2, idx1
|
| 61 |
+
], # same triangle, but backward (cheap solution to cancel face culling)
|
| 62 |
+
np.c_[idx2, idx3, idx4],
|
| 63 |
+
np.c_[
|
| 64 |
+
idx4, idx3, idx2
|
| 65 |
+
], # same triangle, but backward (cheap solution to cancel face culling)
|
| 66 |
+
),
|
| 67 |
+
axis=0,
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
# prepare triangle colors
|
| 71 |
+
face_colors = np.concatenate(
|
| 72 |
+
(
|
| 73 |
+
img[:-1, :-1].reshape(-1, 3),
|
| 74 |
+
img[:-1, :-1].reshape(-1, 3),
|
| 75 |
+
img[+1:, +1:].reshape(-1, 3),
|
| 76 |
+
img[+1:, +1:].reshape(-1, 3),
|
| 77 |
+
),
|
| 78 |
+
axis=0,
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
# remove invalid faces
|
| 82 |
+
if valid is not None:
|
| 83 |
+
assert valid.shape == (H, W)
|
| 84 |
+
valid_idxs = valid.ravel()
|
| 85 |
+
valid_faces = valid_idxs[faces].all(axis=-1)
|
| 86 |
+
faces = faces[valid_faces]
|
| 87 |
+
face_colors = face_colors[valid_faces]
|
| 88 |
+
|
| 89 |
+
assert len(faces) == len(face_colors)
|
| 90 |
+
return dict(vertices=vertices, face_colors=face_colors, faces=faces)
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def cat_meshes(meshes):
|
| 94 |
+
vertices, faces, colors = zip(
|
| 95 |
+
*[(m["vertices"], m["faces"], m["face_colors"]) for m in meshes]
|
| 96 |
+
)
|
| 97 |
+
n_vertices = np.cumsum([0] + [len(v) for v in vertices])
|
| 98 |
+
for i in range(len(faces)):
|
| 99 |
+
faces[i][:] += n_vertices[i]
|
| 100 |
+
|
| 101 |
+
vertices = np.concatenate(vertices)
|
| 102 |
+
colors = np.concatenate(colors)
|
| 103 |
+
faces = np.concatenate(faces)
|
| 104 |
+
return dict(vertices=vertices, face_colors=colors, faces=faces)
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def show_duster_pairs(view1, view2, pred1, pred2):
|
| 108 |
+
import matplotlib.pyplot as pl
|
| 109 |
+
|
| 110 |
+
pl.ion()
|
| 111 |
+
|
| 112 |
+
for e in range(len(view1["instance"])):
|
| 113 |
+
i = view1["idx"][e]
|
| 114 |
+
j = view2["idx"][e]
|
| 115 |
+
img1 = rgb(view1["img"][e])
|
| 116 |
+
img2 = rgb(view2["img"][e])
|
| 117 |
+
conf1 = pred1["conf"][e].squeeze()
|
| 118 |
+
conf2 = pred2["conf"][e].squeeze()
|
| 119 |
+
score = conf1.mean() * conf2.mean()
|
| 120 |
+
print(f">> Showing pair #{e} {i}-{j} {score=:g}")
|
| 121 |
+
pl.clf()
|
| 122 |
+
pl.subplot(221).imshow(img1)
|
| 123 |
+
pl.subplot(223).imshow(img2)
|
| 124 |
+
pl.subplot(222).imshow(conf1, vmin=1, vmax=30)
|
| 125 |
+
pl.subplot(224).imshow(conf2, vmin=1, vmax=30)
|
| 126 |
+
pts1 = pred1["pts3d"][e]
|
| 127 |
+
pts2 = pred2["pts3d_in_other_view"][e]
|
| 128 |
+
pl.subplots_adjust(0, 0, 1, 1, 0, 0)
|
| 129 |
+
if input("show pointcloud? (y/n) ") == "y":
|
| 130 |
+
show_raw_pointcloud(cat(pts1, pts2), cat(img1, img2), point_size=5)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def auto_cam_size(im_poses):
|
| 134 |
+
return 0.1 * get_med_dist_between_poses(im_poses)
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
class SceneViz:
|
| 138 |
+
def __init__(self):
|
| 139 |
+
self.scene = trimesh.Scene()
|
| 140 |
+
|
| 141 |
+
def add_pointcloud(self, pts3d, color, mask=None):
|
| 142 |
+
pts3d = to_numpy(pts3d)
|
| 143 |
+
mask = to_numpy(mask)
|
| 144 |
+
if mask is None:
|
| 145 |
+
mask = [slice(None)] * len(pts3d)
|
| 146 |
+
pts = np.concatenate([p[m] for p, m in zip(pts3d, mask)])
|
| 147 |
+
pct = trimesh.PointCloud(pts.reshape(-1, 3))
|
| 148 |
+
|
| 149 |
+
if isinstance(color, (list, np.ndarray, torch.Tensor)):
|
| 150 |
+
color = to_numpy(color)
|
| 151 |
+
col = np.concatenate([p[m] for p, m in zip(color, mask)])
|
| 152 |
+
assert col.shape == pts.shape
|
| 153 |
+
pct.visual.vertex_colors = uint8(col.reshape(-1, 3))
|
| 154 |
+
else:
|
| 155 |
+
assert len(color) == 3
|
| 156 |
+
pct.visual.vertex_colors = np.broadcast_to(uint8(color), pts.shape)
|
| 157 |
+
|
| 158 |
+
self.scene.add_geometry(pct)
|
| 159 |
+
return self
|
| 160 |
+
|
| 161 |
+
def add_camera(
|
| 162 |
+
self,
|
| 163 |
+
pose_c2w,
|
| 164 |
+
focal=None,
|
| 165 |
+
color=(0, 0, 0),
|
| 166 |
+
image=None,
|
| 167 |
+
imsize=None,
|
| 168 |
+
cam_size=0.03,
|
| 169 |
+
):
|
| 170 |
+
pose_c2w, focal, color, image = to_numpy((pose_c2w, focal, color, image))
|
| 171 |
+
add_scene_cam(self.scene, pose_c2w, color, image, focal, screen_width=cam_size)
|
| 172 |
+
return self
|
| 173 |
+
|
| 174 |
+
def add_cameras(
|
| 175 |
+
self, poses, focals=None, images=None, imsizes=None, colors=None, **kw
|
| 176 |
+
):
|
| 177 |
+
def get(arr, idx):
|
| 178 |
+
return None if arr is None else arr[idx]
|
| 179 |
+
|
| 180 |
+
for i, pose_c2w in enumerate(poses):
|
| 181 |
+
self.add_camera(
|
| 182 |
+
pose_c2w,
|
| 183 |
+
get(focals, i),
|
| 184 |
+
image=get(images, i),
|
| 185 |
+
color=get(colors, i),
|
| 186 |
+
imsize=get(imsizes, i),
|
| 187 |
+
**kw,
|
| 188 |
+
)
|
| 189 |
+
return self
|
| 190 |
+
|
| 191 |
+
def show(self, point_size=2, viewer=None):
|
| 192 |
+
return self.scene.show(viewer=viewer, line_settings={"point_size": point_size})
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def show_raw_pointcloud_with_cams(
|
| 196 |
+
imgs, pts3d, mask, focals, cams2world, point_size=2, cam_size=0.05, cam_color=None
|
| 197 |
+
):
|
| 198 |
+
"""Visualization of a pointcloud with cameras
|
| 199 |
+
imgs = (N, H, W, 3) or N-size list of [(H,W,3), ...]
|
| 200 |
+
pts3d = (N, H, W, 3) or N-size list of [(H,W,3), ...]
|
| 201 |
+
focals = (N,) or N-size list of [focal, ...]
|
| 202 |
+
cams2world = (N,4,4) or N-size list of [(4,4), ...]
|
| 203 |
+
"""
|
| 204 |
+
assert len(pts3d) == len(mask) <= len(imgs) <= len(cams2world) == len(focals)
|
| 205 |
+
pts3d = to_numpy(pts3d)
|
| 206 |
+
imgs = to_numpy(imgs)
|
| 207 |
+
focals = to_numpy(focals)
|
| 208 |
+
cams2world = to_numpy(cams2world)
|
| 209 |
+
|
| 210 |
+
scene = trimesh.Scene()
|
| 211 |
+
|
| 212 |
+
# full pointcloud
|
| 213 |
+
pts = np.concatenate([p[m] for p, m in zip(pts3d, mask)])
|
| 214 |
+
col = np.concatenate([p[m] for p, m in zip(imgs, mask)])
|
| 215 |
+
pct = trimesh.PointCloud(pts.reshape(-1, 3), colors=col.reshape(-1, 3))
|
| 216 |
+
scene.add_geometry(pct)
|
| 217 |
+
|
| 218 |
+
# add each camera
|
| 219 |
+
for i, pose_c2w in enumerate(cams2world):
|
| 220 |
+
if isinstance(cam_color, list):
|
| 221 |
+
camera_edge_color = cam_color[i]
|
| 222 |
+
else:
|
| 223 |
+
camera_edge_color = cam_color or CAM_COLORS[i % len(CAM_COLORS)]
|
| 224 |
+
add_scene_cam(
|
| 225 |
+
scene,
|
| 226 |
+
pose_c2w,
|
| 227 |
+
camera_edge_color,
|
| 228 |
+
imgs[i] if i < len(imgs) else None,
|
| 229 |
+
focals[i],
|
| 230 |
+
screen_width=cam_size,
|
| 231 |
+
)
|
| 232 |
+
|
| 233 |
+
scene.show(line_settings={"point_size": point_size})
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def add_scene_cam(
|
| 237 |
+
scene, pose_c2w, edge_color, image=None, focal=None, imsize=None, screen_width=0.03
|
| 238 |
+
):
|
| 239 |
+
if image is not None:
|
| 240 |
+
H, W, THREE = image.shape
|
| 241 |
+
assert THREE == 3
|
| 242 |
+
if image.dtype != np.uint8:
|
| 243 |
+
image = np.uint8(255 * image)
|
| 244 |
+
elif imsize is not None:
|
| 245 |
+
W, H = imsize
|
| 246 |
+
elif focal is not None:
|
| 247 |
+
H = W = focal / 1.1
|
| 248 |
+
else:
|
| 249 |
+
H = W = 1
|
| 250 |
+
|
| 251 |
+
if focal is None:
|
| 252 |
+
focal = min(H, W) * 1.1 # default value
|
| 253 |
+
elif isinstance(focal, np.ndarray):
|
| 254 |
+
focal = focal[0]
|
| 255 |
+
|
| 256 |
+
# create fake camera
|
| 257 |
+
height = focal * screen_width / H
|
| 258 |
+
width = screen_width * 0.5**0.5
|
| 259 |
+
rot45 = np.eye(4)
|
| 260 |
+
rot45[:3, :3] = Rotation.from_euler("z", np.deg2rad(45)).as_matrix()
|
| 261 |
+
rot45[2, 3] = -height # set the tip of the cone = optical center
|
| 262 |
+
aspect_ratio = np.eye(4)
|
| 263 |
+
aspect_ratio[0, 0] = W / H
|
| 264 |
+
transform = pose_c2w @ OPENGL @ aspect_ratio @ rot45
|
| 265 |
+
cam = trimesh.creation.cone(width, height, sections=4) # , transform=transform)
|
| 266 |
+
|
| 267 |
+
# this is the image
|
| 268 |
+
if image is not None:
|
| 269 |
+
vertices = geotrf(transform, cam.vertices[[4, 5, 1, 3]])
|
| 270 |
+
faces = np.array([[0, 1, 2], [0, 2, 3], [2, 1, 0], [3, 2, 0]])
|
| 271 |
+
img = trimesh.Trimesh(vertices=vertices, faces=faces)
|
| 272 |
+
uv_coords = np.float32([[0, 0], [1, 0], [1, 1], [0, 1]])
|
| 273 |
+
img.visual = trimesh.visual.TextureVisuals(
|
| 274 |
+
uv_coords, image=PIL.Image.fromarray(image)
|
| 275 |
+
)
|
| 276 |
+
scene.add_geometry(img)
|
| 277 |
+
|
| 278 |
+
# this is the camera mesh
|
| 279 |
+
rot2 = np.eye(4)
|
| 280 |
+
rot2[:3, :3] = Rotation.from_euler("z", np.deg2rad(2)).as_matrix()
|
| 281 |
+
vertices = np.r_[cam.vertices, 0.95 * cam.vertices, geotrf(rot2, cam.vertices)]
|
| 282 |
+
vertices = geotrf(transform, vertices)
|
| 283 |
+
faces = []
|
| 284 |
+
for face in cam.faces:
|
| 285 |
+
if 0 in face:
|
| 286 |
+
continue
|
| 287 |
+
a, b, c = face
|
| 288 |
+
a2, b2, c2 = face + len(cam.vertices)
|
| 289 |
+
a3, b3, c3 = face + 2 * len(cam.vertices)
|
| 290 |
+
|
| 291 |
+
# add 3 pseudo-edges
|
| 292 |
+
faces.append((a, b, b2))
|
| 293 |
+
faces.append((a, a2, c))
|
| 294 |
+
faces.append((c2, b, c))
|
| 295 |
+
|
| 296 |
+
faces.append((a, b, b3))
|
| 297 |
+
faces.append((a, a3, c))
|
| 298 |
+
faces.append((c3, b, c))
|
| 299 |
+
|
| 300 |
+
# no culling
|
| 301 |
+
faces += [(c, b, a) for a, b, c in faces]
|
| 302 |
+
|
| 303 |
+
cam = trimesh.Trimesh(vertices=vertices, faces=faces)
|
| 304 |
+
cam.visual.face_colors[:, :3] = edge_color
|
| 305 |
+
scene.add_geometry(cam)
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
def cat(a, b):
|
| 309 |
+
return np.concatenate((a.reshape(-1, 3), b.reshape(-1, 3)))
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
OPENGL = np.array([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]])
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
CAM_COLORS = [
|
| 316 |
+
(255, 0, 0),
|
| 317 |
+
(0, 0, 255),
|
| 318 |
+
(0, 255, 0),
|
| 319 |
+
(255, 0, 255),
|
| 320 |
+
(255, 204, 0),
|
| 321 |
+
(0, 204, 204),
|
| 322 |
+
(128, 255, 255),
|
| 323 |
+
(255, 128, 255),
|
| 324 |
+
(255, 255, 128),
|
| 325 |
+
(0, 0, 0),
|
| 326 |
+
(128, 128, 128),
|
| 327 |
+
]
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
def uint8(colors):
|
| 331 |
+
if not isinstance(colors, np.ndarray):
|
| 332 |
+
colors = np.array(colors)
|
| 333 |
+
if np.issubdtype(colors.dtype, np.floating):
|
| 334 |
+
colors *= 255
|
| 335 |
+
assert 0 <= colors.min() and colors.max() < 256
|
| 336 |
+
return np.uint8(colors)
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
def segment_sky(image):
|
| 340 |
+
import cv2
|
| 341 |
+
from scipy import ndimage
|
| 342 |
+
|
| 343 |
+
# Convert to HSV
|
| 344 |
+
image = to_numpy(image)
|
| 345 |
+
if np.issubdtype(image.dtype, np.floating):
|
| 346 |
+
image = np.uint8(255 * image.clip(min=0, max=1))
|
| 347 |
+
hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
|
| 348 |
+
|
| 349 |
+
# Define range for blue color and create mask
|
| 350 |
+
lower_blue = np.array([0, 0, 100])
|
| 351 |
+
upper_blue = np.array([30, 255, 255])
|
| 352 |
+
mask = cv2.inRange(hsv, lower_blue, upper_blue).view(bool)
|
| 353 |
+
|
| 354 |
+
# add luminous gray
|
| 355 |
+
mask |= (hsv[:, :, 1] < 10) & (hsv[:, :, 2] > 150)
|
| 356 |
+
mask |= (hsv[:, :, 1] < 30) & (hsv[:, :, 2] > 180)
|
| 357 |
+
mask |= (hsv[:, :, 1] < 50) & (hsv[:, :, 2] > 220)
|
| 358 |
+
|
| 359 |
+
# Morphological operations
|
| 360 |
+
kernel = np.ones((5, 5), np.uint8)
|
| 361 |
+
mask2 = ndimage.binary_opening(mask, structure=kernel)
|
| 362 |
+
|
| 363 |
+
# keep only largest CC
|
| 364 |
+
_, labels, stats, _ = cv2.connectedComponentsWithStats(
|
| 365 |
+
mask2.view(np.uint8), connectivity=8
|
| 366 |
+
)
|
| 367 |
+
cc_sizes = stats[1:, cv2.CC_STAT_AREA]
|
| 368 |
+
order = cc_sizes.argsort()[::-1] # bigger first
|
| 369 |
+
i = 0
|
| 370 |
+
selection = []
|
| 371 |
+
while i < len(order) and cc_sizes[order[i]] > cc_sizes[order[0]] / 2:
|
| 372 |
+
selection.append(1 + order[i])
|
| 373 |
+
i += 1
|
| 374 |
+
mask3 = np.in1d(labels, selection).reshape(labels.shape)
|
| 375 |
+
|
| 376 |
+
# Apply mask
|
| 377 |
+
return torch.from_numpy(mask3)
|
src/visualization/annotation.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
from string import ascii_letters, digits, punctuation
|
| 3 |
+
|
| 4 |
+
import numpy as np
|
| 5 |
+
import torch
|
| 6 |
+
from einops import rearrange
|
| 7 |
+
from jaxtyping import Float
|
| 8 |
+
from PIL import Image, ImageDraw, ImageFont
|
| 9 |
+
from torch import Tensor
|
| 10 |
+
|
| 11 |
+
from .layout import vcat
|
| 12 |
+
|
| 13 |
+
EXPECTED_CHARACTERS = digits + punctuation + ascii_letters
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def draw_label(
|
| 17 |
+
text: str,
|
| 18 |
+
font: Path,
|
| 19 |
+
font_size: int,
|
| 20 |
+
device: torch.device = torch.device("cpu"),
|
| 21 |
+
) -> Float[Tensor, "3 height width"]:
|
| 22 |
+
"""Draw a black label on a white background with no border."""
|
| 23 |
+
try:
|
| 24 |
+
font = ImageFont.truetype(str(font), font_size)
|
| 25 |
+
except OSError:
|
| 26 |
+
font = ImageFont.load_default()
|
| 27 |
+
left, _, right, _ = font.getbbox(text)
|
| 28 |
+
width = right - left
|
| 29 |
+
_, top, _, bottom = font.getbbox(EXPECTED_CHARACTERS)
|
| 30 |
+
height = bottom - top
|
| 31 |
+
image = Image.new("RGB", (width, height), color="white")
|
| 32 |
+
draw = ImageDraw.Draw(image)
|
| 33 |
+
draw.text((0, 0), text, font=font, fill="black")
|
| 34 |
+
image = torch.tensor(np.array(image) / 255, dtype=torch.float32, device=device)
|
| 35 |
+
return rearrange(image, "h w c -> c h w")
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def add_label(
|
| 39 |
+
image: Float[Tensor, "3 width height"],
|
| 40 |
+
label: str,
|
| 41 |
+
font: Path = Path("assets/Inter-Regular.otf"),
|
| 42 |
+
font_size: int = 24,
|
| 43 |
+
) -> Float[Tensor, "3 width_with_label height_with_label"]:
|
| 44 |
+
return vcat(
|
| 45 |
+
draw_label(label, font, font_size, image.device),
|
| 46 |
+
image,
|
| 47 |
+
align="left",
|
| 48 |
+
gap=4,
|
| 49 |
+
)
|
src/visualization/camera_trajectory/interpolation.py
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from einops import einsum, rearrange, reduce
|
| 3 |
+
from jaxtyping import Float
|
| 4 |
+
from scipy.spatial.transform import Rotation as R
|
| 5 |
+
from torch import Tensor
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def interpolate_intrinsics(
|
| 9 |
+
initial: Float[Tensor, "*#batch 3 3"],
|
| 10 |
+
final: Float[Tensor, "*#batch 3 3"],
|
| 11 |
+
t: Float[Tensor, " time_step"],
|
| 12 |
+
) -> Float[Tensor, "*batch time_step 3 3"]:
|
| 13 |
+
initial = rearrange(initial, "... i j -> ... () i j")
|
| 14 |
+
final = rearrange(final, "... i j -> ... () i j")
|
| 15 |
+
t = rearrange(t, "t -> t () ()")
|
| 16 |
+
return initial + (final - initial) * t
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def intersect_rays(
|
| 20 |
+
a_origins: Float[Tensor, "*#batch dim"],
|
| 21 |
+
a_directions: Float[Tensor, "*#batch dim"],
|
| 22 |
+
b_origins: Float[Tensor, "*#batch dim"],
|
| 23 |
+
b_directions: Float[Tensor, "*#batch dim"],
|
| 24 |
+
) -> Float[Tensor, "*batch dim"]:
|
| 25 |
+
"""Compute the least-squares intersection of rays. Uses the math from here:
|
| 26 |
+
https://math.stackexchange.com/a/1762491/286022
|
| 27 |
+
"""
|
| 28 |
+
|
| 29 |
+
# Broadcast and stack the tensors.
|
| 30 |
+
a_origins, a_directions, b_origins, b_directions = torch.broadcast_tensors(
|
| 31 |
+
a_origins, a_directions, b_origins, b_directions
|
| 32 |
+
)
|
| 33 |
+
origins = torch.stack((a_origins, b_origins), dim=-2)
|
| 34 |
+
directions = torch.stack((a_directions, b_directions), dim=-2)
|
| 35 |
+
|
| 36 |
+
# Compute n_i * n_i^T - eye(3) from the equation.
|
| 37 |
+
n = einsum(directions, directions, "... n i, ... n j -> ... n i j")
|
| 38 |
+
n = n - torch.eye(3, dtype=origins.dtype, device=origins.device)
|
| 39 |
+
|
| 40 |
+
# Compute the left-hand side of the equation.
|
| 41 |
+
lhs = reduce(n, "... n i j -> ... i j", "sum")
|
| 42 |
+
|
| 43 |
+
# Compute the right-hand side of the equation.
|
| 44 |
+
rhs = einsum(n, origins, "... n i j, ... n j -> ... n i")
|
| 45 |
+
rhs = reduce(rhs, "... n i -> ... i", "sum")
|
| 46 |
+
|
| 47 |
+
# Left-matrix-multiply both sides by the inverse of lhs to find p.
|
| 48 |
+
return torch.linalg.lstsq(lhs, rhs).solution
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def normalize(a: Float[Tensor, "*#batch dim"]) -> Float[Tensor, "*#batch dim"]:
|
| 52 |
+
return a / a.norm(dim=-1, keepdim=True)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def generate_coordinate_frame(
|
| 56 |
+
y: Float[Tensor, "*#batch 3"],
|
| 57 |
+
z: Float[Tensor, "*#batch 3"],
|
| 58 |
+
) -> Float[Tensor, "*batch 3 3"]:
|
| 59 |
+
"""Generate a coordinate frame given perpendicular, unit-length Y and Z vectors."""
|
| 60 |
+
y, z = torch.broadcast_tensors(y, z)
|
| 61 |
+
return torch.stack([y.cross(z), y, z], dim=-1)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def generate_rotation_coordinate_frame(
|
| 65 |
+
a: Float[Tensor, "*#batch 3"],
|
| 66 |
+
b: Float[Tensor, "*#batch 3"],
|
| 67 |
+
eps: float = 1e-4,
|
| 68 |
+
) -> Float[Tensor, "*batch 3 3"]:
|
| 69 |
+
"""Generate a coordinate frame where the Y direction is normal to the plane defined
|
| 70 |
+
by unit vectors a and b. The other axes are arbitrary."""
|
| 71 |
+
device = a.device
|
| 72 |
+
|
| 73 |
+
# Replace every entry in b that's parallel to the corresponding entry in a with an
|
| 74 |
+
# arbitrary vector.
|
| 75 |
+
b = b.detach().clone()
|
| 76 |
+
parallel = (einsum(a, b, "... i, ... i -> ...").abs() - 1).abs() < eps
|
| 77 |
+
b[parallel] = torch.tensor([0, 0, 1], dtype=b.dtype, device=device)
|
| 78 |
+
parallel = (einsum(a, b, "... i, ... i -> ...").abs() - 1).abs() < eps
|
| 79 |
+
b[parallel] = torch.tensor([0, 1, 0], dtype=b.dtype, device=device)
|
| 80 |
+
|
| 81 |
+
# Generate the coordinate frame. The initial cross product defines the plane.
|
| 82 |
+
return generate_coordinate_frame(normalize(a.cross(b)), a)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def matrix_to_euler(
|
| 86 |
+
rotations: Float[Tensor, "*batch 3 3"],
|
| 87 |
+
pattern: str,
|
| 88 |
+
) -> Float[Tensor, "*batch 3"]:
|
| 89 |
+
*batch, _, _ = rotations.shape
|
| 90 |
+
rotations = rotations.reshape(-1, 3, 3)
|
| 91 |
+
angles_np = R.from_matrix(rotations.detach().cpu().numpy()).as_euler(pattern)
|
| 92 |
+
rotations = torch.tensor(angles_np, dtype=rotations.dtype, device=rotations.device)
|
| 93 |
+
return rotations.reshape(*batch, 3)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def euler_to_matrix(
|
| 97 |
+
rotations: Float[Tensor, "*batch 3"],
|
| 98 |
+
pattern: str,
|
| 99 |
+
) -> Float[Tensor, "*batch 3 3"]:
|
| 100 |
+
*batch, _ = rotations.shape
|
| 101 |
+
rotations = rotations.reshape(-1, 3)
|
| 102 |
+
matrix_np = R.from_euler(pattern, rotations.detach().cpu().numpy()).as_matrix()
|
| 103 |
+
rotations = torch.tensor(matrix_np, dtype=rotations.dtype, device=rotations.device)
|
| 104 |
+
return rotations.reshape(*batch, 3, 3)
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def extrinsics_to_pivot_parameters(
|
| 108 |
+
extrinsics: Float[Tensor, "*#batch 4 4"],
|
| 109 |
+
pivot_coordinate_frame: Float[Tensor, "*#batch 3 3"],
|
| 110 |
+
pivot_point: Float[Tensor, "*#batch 3"],
|
| 111 |
+
) -> Float[Tensor, "*batch 5"]:
|
| 112 |
+
"""Convert the extrinsics to a representation with 5 degrees of freedom:
|
| 113 |
+
1. Distance from pivot point in the "X" (look cross pivot axis) direction.
|
| 114 |
+
2. Distance from pivot point in the "Y" (pivot axis) direction.
|
| 115 |
+
3. Distance from pivot point in the Z (look) direction
|
| 116 |
+
4. Angle in plane
|
| 117 |
+
5. Twist (rotation not in plane)
|
| 118 |
+
"""
|
| 119 |
+
|
| 120 |
+
# The pivot coordinate frame's Z axis is normal to the plane.
|
| 121 |
+
pivot_axis = pivot_coordinate_frame[..., :, 1]
|
| 122 |
+
|
| 123 |
+
# Compute the translation elements of the pivot parametrization.
|
| 124 |
+
translation_frame = generate_coordinate_frame(pivot_axis, extrinsics[..., :3, 2])
|
| 125 |
+
origin = extrinsics[..., :3, 3]
|
| 126 |
+
delta = pivot_point - origin
|
| 127 |
+
translation = einsum(translation_frame, delta, "... i j, ... i -> ... j")
|
| 128 |
+
|
| 129 |
+
# Add the rotation elements of the pivot parametrization.
|
| 130 |
+
inverted = pivot_coordinate_frame.inverse() @ extrinsics[..., :3, :3]
|
| 131 |
+
y, _, z = matrix_to_euler(inverted, "YXZ").unbind(dim=-1)
|
| 132 |
+
|
| 133 |
+
return torch.cat([translation, y[..., None], z[..., None]], dim=-1)
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def pivot_parameters_to_extrinsics(
|
| 137 |
+
parameters: Float[Tensor, "*#batch 5"],
|
| 138 |
+
pivot_coordinate_frame: Float[Tensor, "*#batch 3 3"],
|
| 139 |
+
pivot_point: Float[Tensor, "*#batch 3"],
|
| 140 |
+
) -> Float[Tensor, "*batch 4 4"]:
|
| 141 |
+
translation, y, z = parameters.split((3, 1, 1), dim=-1)
|
| 142 |
+
|
| 143 |
+
euler = torch.cat((y, torch.zeros_like(y), z), dim=-1)
|
| 144 |
+
rotation = pivot_coordinate_frame @ euler_to_matrix(euler, "YXZ")
|
| 145 |
+
|
| 146 |
+
# The pivot coordinate frame's Z axis is normal to the plane.
|
| 147 |
+
pivot_axis = pivot_coordinate_frame[..., :, 1]
|
| 148 |
+
|
| 149 |
+
translation_frame = generate_coordinate_frame(pivot_axis, rotation[..., :3, 2])
|
| 150 |
+
delta = einsum(translation_frame, translation, "... i j, ... j -> ... i")
|
| 151 |
+
origin = pivot_point - delta
|
| 152 |
+
|
| 153 |
+
*batch, _ = origin.shape
|
| 154 |
+
extrinsics = torch.eye(4, dtype=parameters.dtype, device=parameters.device)
|
| 155 |
+
extrinsics = extrinsics.broadcast_to((*batch, 4, 4)).clone()
|
| 156 |
+
extrinsics[..., 3, 3] = 1
|
| 157 |
+
extrinsics[..., :3, :3] = rotation
|
| 158 |
+
extrinsics[..., :3, 3] = origin
|
| 159 |
+
return extrinsics
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def interpolate_circular(
|
| 163 |
+
a: Float[Tensor, "*#batch"],
|
| 164 |
+
b: Float[Tensor, "*#batch"],
|
| 165 |
+
t: Float[Tensor, "*#batch"],
|
| 166 |
+
) -> Float[Tensor, " *batch"]:
|
| 167 |
+
a, b, t = torch.broadcast_tensors(a, b, t)
|
| 168 |
+
|
| 169 |
+
tau = 2 * torch.pi
|
| 170 |
+
a = a % tau
|
| 171 |
+
b = b % tau
|
| 172 |
+
|
| 173 |
+
# Consider piecewise edge cases.
|
| 174 |
+
d = (b - a).abs()
|
| 175 |
+
a_left = a - tau
|
| 176 |
+
d_left = (b - a_left).abs()
|
| 177 |
+
a_right = a + tau
|
| 178 |
+
d_right = (b - a_right).abs()
|
| 179 |
+
use_d = (d < d_left) & (d < d_right)
|
| 180 |
+
use_d_left = (d_left < d_right) & (~use_d)
|
| 181 |
+
use_d_right = (~use_d) & (~use_d_left)
|
| 182 |
+
|
| 183 |
+
result = a + (b - a) * t
|
| 184 |
+
result[use_d_left] = (a_left + (b - a_left) * t)[use_d_left]
|
| 185 |
+
result[use_d_right] = (a_right + (b - a_right) * t)[use_d_right]
|
| 186 |
+
|
| 187 |
+
return result
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def interpolate_pivot_parameters(
|
| 191 |
+
initial: Float[Tensor, "*#batch 5"],
|
| 192 |
+
final: Float[Tensor, "*#batch 5"],
|
| 193 |
+
t: Float[Tensor, " time_step"],
|
| 194 |
+
) -> Float[Tensor, "*batch time_step 5"]:
|
| 195 |
+
initial = rearrange(initial, "... d -> ... () d")
|
| 196 |
+
final = rearrange(final, "... d -> ... () d")
|
| 197 |
+
t = rearrange(t, "t -> t ()")
|
| 198 |
+
ti, ri = initial.split((3, 2), dim=-1)
|
| 199 |
+
tf, rf = final.split((3, 2), dim=-1)
|
| 200 |
+
|
| 201 |
+
t_lerp = ti + (tf - ti) * t
|
| 202 |
+
r_lerp = interpolate_circular(ri, rf, t)
|
| 203 |
+
|
| 204 |
+
return torch.cat((t_lerp, r_lerp), dim=-1)
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
@torch.no_grad()
|
| 208 |
+
def interpolate_extrinsics(
|
| 209 |
+
initial: Float[Tensor, "*#batch 4 4"],
|
| 210 |
+
final: Float[Tensor, "*#batch 4 4"],
|
| 211 |
+
t: Float[Tensor, " time_step"],
|
| 212 |
+
eps: float = 1e-4,
|
| 213 |
+
) -> Float[Tensor, "*batch time_step 4 4"]:
|
| 214 |
+
"""Interpolate extrinsics by rotating around their "focus point," which is the
|
| 215 |
+
least-squares intersection between the look vectors of the initial and final
|
| 216 |
+
extrinsics.
|
| 217 |
+
"""
|
| 218 |
+
|
| 219 |
+
initial = initial.type(torch.float64)
|
| 220 |
+
final = final.type(torch.float64)
|
| 221 |
+
t = t.type(torch.float64)
|
| 222 |
+
|
| 223 |
+
# Based on the dot product between the look vectors, pick from one of two cases:
|
| 224 |
+
# 1. Look vectors are parallel: interpolate about their origins' midpoint.
|
| 225 |
+
# 3. Look vectors aren't parallel: interpolate about their focus point.
|
| 226 |
+
initial_look = initial[..., :3, 2]
|
| 227 |
+
final_look = final[..., :3, 2]
|
| 228 |
+
dot_products = einsum(initial_look, final_look, "... i, ... i -> ...")
|
| 229 |
+
parallel_mask = (dot_products.abs() - 1).abs() < eps
|
| 230 |
+
|
| 231 |
+
# Pick focus points.
|
| 232 |
+
initial_origin = initial[..., :3, 3]
|
| 233 |
+
final_origin = final[..., :3, 3]
|
| 234 |
+
pivot_point = 0.5 * (initial_origin + final_origin)
|
| 235 |
+
pivot_point[~parallel_mask] = intersect_rays(
|
| 236 |
+
initial_origin[~parallel_mask],
|
| 237 |
+
initial_look[~parallel_mask],
|
| 238 |
+
final_origin[~parallel_mask],
|
| 239 |
+
final_look[~parallel_mask],
|
| 240 |
+
)
|
| 241 |
+
|
| 242 |
+
# Convert to pivot parameters.
|
| 243 |
+
pivot_frame = generate_rotation_coordinate_frame(initial_look, final_look, eps=eps)
|
| 244 |
+
initial_params = extrinsics_to_pivot_parameters(initial, pivot_frame, pivot_point)
|
| 245 |
+
final_params = extrinsics_to_pivot_parameters(final, pivot_frame, pivot_point)
|
| 246 |
+
|
| 247 |
+
# Interpolate the pivot parameters.
|
| 248 |
+
interpolated_params = interpolate_pivot_parameters(initial_params, final_params, t)
|
| 249 |
+
|
| 250 |
+
# Convert back.
|
| 251 |
+
return pivot_parameters_to_extrinsics(
|
| 252 |
+
interpolated_params.type(torch.float32),
|
| 253 |
+
rearrange(pivot_frame, "... i j -> ... () i j").type(torch.float32),
|
| 254 |
+
rearrange(pivot_point, "... xyz -> ... () xyz").type(torch.float32),
|
| 255 |
+
)
|
src/visualization/camera_trajectory/spin.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import numpy as np
|
| 2 |
+
import torch
|
| 3 |
+
from einops import repeat
|
| 4 |
+
from jaxtyping import Float
|
| 5 |
+
from scipy.spatial.transform import Rotation as R
|
| 6 |
+
from torch import Tensor
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def generate_spin(
|
| 10 |
+
num_frames: int,
|
| 11 |
+
device: torch.device,
|
| 12 |
+
elevation: float,
|
| 13 |
+
radius: float,
|
| 14 |
+
) -> Float[Tensor, "frame 4 4"]:
|
| 15 |
+
# Translate back along the camera's look vector.
|
| 16 |
+
tf_translation = torch.eye(4, dtype=torch.float32, device=device)
|
| 17 |
+
tf_translation[:2] *= -1
|
| 18 |
+
tf_translation[2, 3] = -radius
|
| 19 |
+
|
| 20 |
+
# Generate the transformation for the azimuth.
|
| 21 |
+
phi = 2 * np.pi * (np.arange(num_frames) / num_frames)
|
| 22 |
+
rotation_vectors = np.stack([np.zeros_like(phi), phi, np.zeros_like(phi)], axis=-1)
|
| 23 |
+
|
| 24 |
+
azimuth = R.from_rotvec(rotation_vectors).as_matrix()
|
| 25 |
+
azimuth = torch.tensor(azimuth, dtype=torch.float32, device=device)
|
| 26 |
+
tf_azimuth = torch.eye(4, dtype=torch.float32, device=device)
|
| 27 |
+
tf_azimuth = repeat(tf_azimuth, "i j -> b i j", b=num_frames).clone()
|
| 28 |
+
tf_azimuth[:, :3, :3] = azimuth
|
| 29 |
+
|
| 30 |
+
# Generate the transformation for the elevation.
|
| 31 |
+
deg_elevation = np.deg2rad(elevation)
|
| 32 |
+
elevation = R.from_rotvec(np.array([deg_elevation, 0, 0], dtype=np.float32))
|
| 33 |
+
elevation = torch.tensor(elevation.as_matrix())
|
| 34 |
+
tf_elevation = torch.eye(4, dtype=torch.float32, device=device)
|
| 35 |
+
tf_elevation[:3, :3] = elevation
|
| 36 |
+
|
| 37 |
+
return tf_azimuth @ tf_elevation @ tf_translation
|
src/visualization/camera_trajectory/wobble.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from einops import rearrange
|
| 3 |
+
from jaxtyping import Float
|
| 4 |
+
from torch import Tensor
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
@torch.no_grad()
|
| 8 |
+
def generate_wobble_transformation(
|
| 9 |
+
radius: Float[Tensor, "*#batch"],
|
| 10 |
+
t: Float[Tensor, " time_step"],
|
| 11 |
+
num_rotations: int = 1,
|
| 12 |
+
scale_radius_with_t: bool = True,
|
| 13 |
+
) -> Float[Tensor, "*batch time_step 4 4"]:
|
| 14 |
+
# Generate a translation in the image plane.
|
| 15 |
+
tf = torch.eye(4, dtype=torch.float32, device=t.device)
|
| 16 |
+
tf = tf.broadcast_to((*radius.shape, t.shape[0], 4, 4)).clone()
|
| 17 |
+
radius = radius[..., None]
|
| 18 |
+
if scale_radius_with_t:
|
| 19 |
+
radius = radius * t
|
| 20 |
+
tf[..., 0, 3] = torch.sin(2 * torch.pi * num_rotations * t) * radius
|
| 21 |
+
tf[..., 1, 3] = -torch.cos(2 * torch.pi * num_rotations * t) * radius
|
| 22 |
+
return tf
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
@torch.no_grad()
|
| 26 |
+
def generate_wobble(
|
| 27 |
+
extrinsics: Float[Tensor, "*#batch 4 4"],
|
| 28 |
+
radius: Float[Tensor, "*#batch"],
|
| 29 |
+
t: Float[Tensor, " time_step"],
|
| 30 |
+
) -> Float[Tensor, "*batch time_step 4 4"]:
|
| 31 |
+
tf = generate_wobble_transformation(radius, t)
|
| 32 |
+
return rearrange(extrinsics, "... i j -> ... () i j") @ tf
|
src/visualization/color_map.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from colorspacious import cspace_convert
|
| 3 |
+
from einops import rearrange
|
| 4 |
+
from jaxtyping import Float
|
| 5 |
+
from matplotlib import cm
|
| 6 |
+
from torch import Tensor
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def apply_color_map(
|
| 10 |
+
x: Float[Tensor, " *batch"],
|
| 11 |
+
color_map: str = "inferno",
|
| 12 |
+
) -> Float[Tensor, "*batch 3"]:
|
| 13 |
+
cmap = cm.get_cmap(color_map)
|
| 14 |
+
|
| 15 |
+
# Convert to NumPy so that Matplotlib color maps can be used.
|
| 16 |
+
mapped = cmap(x.detach().clip(min=0, max=1).cpu().numpy())[..., :3]
|
| 17 |
+
|
| 18 |
+
# Convert back to the original format.
|
| 19 |
+
return torch.tensor(mapped, device=x.device, dtype=torch.float32)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def apply_color_map_to_image(
|
| 23 |
+
image: Float[Tensor, "*batch height width"],
|
| 24 |
+
color_map: str = "inferno",
|
| 25 |
+
) -> Float[Tensor, "*batch 3 height with"]:
|
| 26 |
+
image = apply_color_map(image, color_map)
|
| 27 |
+
return rearrange(image, "... h w c -> ... c h w")
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def apply_color_map_2d(
|
| 31 |
+
x: Float[Tensor, "*#batch"],
|
| 32 |
+
y: Float[Tensor, "*#batch"],
|
| 33 |
+
) -> Float[Tensor, "*batch 3"]:
|
| 34 |
+
red = cspace_convert((189, 0, 0), "sRGB255", "CIELab")
|
| 35 |
+
blue = cspace_convert((0, 45, 255), "sRGB255", "CIELab")
|
| 36 |
+
white = cspace_convert((255, 255, 255), "sRGB255", "CIELab")
|
| 37 |
+
x_np = x.detach().clip(min=0, max=1).cpu().numpy()[..., None]
|
| 38 |
+
y_np = y.detach().clip(min=0, max=1).cpu().numpy()[..., None]
|
| 39 |
+
|
| 40 |
+
# Interpolate between red and blue on the x axis.
|
| 41 |
+
interpolated = x_np * red + (1 - x_np) * blue
|
| 42 |
+
|
| 43 |
+
# Interpolate between color and white on the y axis.
|
| 44 |
+
interpolated = y_np * interpolated + (1 - y_np) * white
|
| 45 |
+
|
| 46 |
+
# Convert to RGB.
|
| 47 |
+
rgb = cspace_convert(interpolated, "CIELab", "sRGB1")
|
| 48 |
+
return torch.tensor(rgb, device=x.device, dtype=torch.float32).clip(min=0, max=1)
|
src/visualization/colors.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from PIL import ImageColor
|
| 2 |
+
|
| 3 |
+
# https://sashamaps.net/docs/resources/20-colors/
|
| 4 |
+
DISTINCT_COLORS = [
|
| 5 |
+
"#e6194b",
|
| 6 |
+
"#3cb44b",
|
| 7 |
+
"#ffe119",
|
| 8 |
+
"#4363d8",
|
| 9 |
+
"#f58231",
|
| 10 |
+
"#911eb4",
|
| 11 |
+
"#46f0f0",
|
| 12 |
+
"#f032e6",
|
| 13 |
+
"#bcf60c",
|
| 14 |
+
"#fabebe",
|
| 15 |
+
"#008080",
|
| 16 |
+
"#e6beff",
|
| 17 |
+
"#9a6324",
|
| 18 |
+
"#fffac8",
|
| 19 |
+
"#800000",
|
| 20 |
+
"#aaffc3",
|
| 21 |
+
"#808000",
|
| 22 |
+
"#ffd8b1",
|
| 23 |
+
"#000075",
|
| 24 |
+
"#808080",
|
| 25 |
+
"#ffffff",
|
| 26 |
+
"#000000",
|
| 27 |
+
]
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def get_distinct_color(index: int) -> tuple[float, float, float]:
|
| 31 |
+
hex = DISTINCT_COLORS[index % len(DISTINCT_COLORS)]
|
| 32 |
+
return tuple(x / 255 for x in ImageColor.getcolor(hex, "RGB"))
|
src/visualization/drawing/cameras.py
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from einops import einsum, rearrange, repeat
|
| 5 |
+
from jaxtyping import Float
|
| 6 |
+
from torch import Tensor
|
| 7 |
+
|
| 8 |
+
from ...geometry.projection import unproject
|
| 9 |
+
from ..annotation import add_label
|
| 10 |
+
from .lines import draw_lines
|
| 11 |
+
from .types import Scalar, sanitize_scalar
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def draw_cameras(
|
| 15 |
+
resolution: int,
|
| 16 |
+
extrinsics: Float[Tensor, "batch 4 4"],
|
| 17 |
+
intrinsics: Float[Tensor, "batch 3 3"],
|
| 18 |
+
color: Float[Tensor, "batch 3"],
|
| 19 |
+
near: Optional[Scalar] = None,
|
| 20 |
+
far: Optional[Scalar] = None,
|
| 21 |
+
margin: float = 0.1, # relative to AABB
|
| 22 |
+
frustum_scale: float = 0.05, # relative to image resolution
|
| 23 |
+
) -> Float[Tensor, "3 3 height width"]:
|
| 24 |
+
device = extrinsics.device
|
| 25 |
+
|
| 26 |
+
# Compute scene bounds.
|
| 27 |
+
minima, maxima = compute_aabb(extrinsics, intrinsics, near, far)
|
| 28 |
+
scene_minima, scene_maxima = compute_equal_aabb_with_margin(
|
| 29 |
+
minima, maxima, margin=margin
|
| 30 |
+
)
|
| 31 |
+
span = (scene_maxima - scene_minima).max()
|
| 32 |
+
|
| 33 |
+
# Compute frustum locations.
|
| 34 |
+
corner_depth = (span * frustum_scale)[None]
|
| 35 |
+
frustum_corners = unproject_frustum_corners(extrinsics, intrinsics, corner_depth)
|
| 36 |
+
if near is not None:
|
| 37 |
+
near_corners = unproject_frustum_corners(extrinsics, intrinsics, near)
|
| 38 |
+
if far is not None:
|
| 39 |
+
far_corners = unproject_frustum_corners(extrinsics, intrinsics, far)
|
| 40 |
+
|
| 41 |
+
# Project the cameras onto each axis-aligned plane.
|
| 42 |
+
projections = []
|
| 43 |
+
for projected_axis in range(3):
|
| 44 |
+
image = torch.zeros(
|
| 45 |
+
(3, resolution, resolution),
|
| 46 |
+
dtype=torch.float32,
|
| 47 |
+
device=device,
|
| 48 |
+
)
|
| 49 |
+
image_x_axis = (projected_axis + 1) % 3
|
| 50 |
+
image_y_axis = (projected_axis + 2) % 3
|
| 51 |
+
|
| 52 |
+
def project(points: Float[Tensor, "*batch 3"]) -> Float[Tensor, "*batch 2"]:
|
| 53 |
+
x = points[..., image_x_axis]
|
| 54 |
+
y = points[..., image_y_axis]
|
| 55 |
+
return torch.stack([x, y], dim=-1)
|
| 56 |
+
|
| 57 |
+
x_range, y_range = torch.stack(
|
| 58 |
+
(project(scene_minima), project(scene_maxima)), dim=-1
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
# Draw near and far planes.
|
| 62 |
+
if near is not None:
|
| 63 |
+
projected_near_corners = project(near_corners)
|
| 64 |
+
image = draw_lines(
|
| 65 |
+
image,
|
| 66 |
+
rearrange(projected_near_corners, "b p xy -> (b p) xy"),
|
| 67 |
+
rearrange(projected_near_corners.roll(1, 1), "b p xy -> (b p) xy"),
|
| 68 |
+
color=0.25,
|
| 69 |
+
width=2,
|
| 70 |
+
x_range=x_range,
|
| 71 |
+
y_range=y_range,
|
| 72 |
+
)
|
| 73 |
+
if far is not None:
|
| 74 |
+
projected_far_corners = project(far_corners)
|
| 75 |
+
image = draw_lines(
|
| 76 |
+
image,
|
| 77 |
+
rearrange(projected_far_corners, "b p xy -> (b p) xy"),
|
| 78 |
+
rearrange(projected_far_corners.roll(1, 1), "b p xy -> (b p) xy"),
|
| 79 |
+
color=0.25,
|
| 80 |
+
width=2,
|
| 81 |
+
x_range=x_range,
|
| 82 |
+
y_range=y_range,
|
| 83 |
+
)
|
| 84 |
+
if near is not None and far is not None:
|
| 85 |
+
image = draw_lines(
|
| 86 |
+
image,
|
| 87 |
+
rearrange(projected_near_corners, "b p xy -> (b p) xy"),
|
| 88 |
+
rearrange(projected_far_corners, "b p xy -> (b p) xy"),
|
| 89 |
+
color=0.25,
|
| 90 |
+
width=2,
|
| 91 |
+
x_range=x_range,
|
| 92 |
+
y_range=y_range,
|
| 93 |
+
)
|
| 94 |
+
|
| 95 |
+
# Draw the camera frustums themselves.
|
| 96 |
+
projected_origins = project(extrinsics[:, :3, 3])
|
| 97 |
+
projected_frustum_corners = project(frustum_corners)
|
| 98 |
+
start = [
|
| 99 |
+
repeat(projected_origins, "b xy -> (b p) xy", p=4),
|
| 100 |
+
rearrange(projected_frustum_corners.roll(1, 1), "b p xy -> (b p) xy"),
|
| 101 |
+
]
|
| 102 |
+
start = rearrange(torch.cat(start, dim=0), "(r b p) xy -> (b r p) xy", r=2, p=4)
|
| 103 |
+
image = draw_lines(
|
| 104 |
+
image,
|
| 105 |
+
start,
|
| 106 |
+
repeat(projected_frustum_corners, "b p xy -> (b r p) xy", r=2),
|
| 107 |
+
color=repeat(color, "b c -> (b r p) c", r=2, p=4),
|
| 108 |
+
width=2,
|
| 109 |
+
x_range=x_range,
|
| 110 |
+
y_range=y_range,
|
| 111 |
+
)
|
| 112 |
+
|
| 113 |
+
x_name = "XYZ"[image_x_axis]
|
| 114 |
+
y_name = "XYZ"[image_y_axis]
|
| 115 |
+
image = add_label(image, f"{x_name}{y_name} Projection")
|
| 116 |
+
|
| 117 |
+
# TODO: Draw axis indicators.
|
| 118 |
+
projections.append(image)
|
| 119 |
+
|
| 120 |
+
return torch.stack(projections)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def compute_aabb(
|
| 124 |
+
extrinsics: Float[Tensor, "batch 4 4"],
|
| 125 |
+
intrinsics: Float[Tensor, "batch 3 3"],
|
| 126 |
+
near: Optional[Scalar] = None,
|
| 127 |
+
far: Optional[Scalar] = None,
|
| 128 |
+
) -> tuple[
|
| 129 |
+
Float[Tensor, "3"], # minima of the scene
|
| 130 |
+
Float[Tensor, "3"], # maxima of the scene
|
| 131 |
+
]:
|
| 132 |
+
"""Compute an axis-aligned bounding box for the camera frustums."""
|
| 133 |
+
|
| 134 |
+
device = extrinsics.device
|
| 135 |
+
|
| 136 |
+
# These points are included in the AABB.
|
| 137 |
+
points = [extrinsics[:, :3, 3]]
|
| 138 |
+
|
| 139 |
+
if near is not None:
|
| 140 |
+
near = sanitize_scalar(near, device)
|
| 141 |
+
corners = unproject_frustum_corners(extrinsics, intrinsics, near)
|
| 142 |
+
points.append(rearrange(corners, "b p xyz -> (b p) xyz"))
|
| 143 |
+
|
| 144 |
+
if far is not None:
|
| 145 |
+
far = sanitize_scalar(far, device)
|
| 146 |
+
corners = unproject_frustum_corners(extrinsics, intrinsics, far)
|
| 147 |
+
points.append(rearrange(corners, "b p xyz -> (b p) xyz"))
|
| 148 |
+
|
| 149 |
+
points = torch.cat(points, dim=0)
|
| 150 |
+
return points.min(dim=0).values, points.max(dim=0).values
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def compute_equal_aabb_with_margin(
|
| 154 |
+
minima: Float[Tensor, "*#batch 3"],
|
| 155 |
+
maxima: Float[Tensor, "*#batch 3"],
|
| 156 |
+
margin: float = 0.1,
|
| 157 |
+
) -> tuple[
|
| 158 |
+
Float[Tensor, "*batch 3"], # minima of the scene
|
| 159 |
+
Float[Tensor, "*batch 3"], # maxima of the scene
|
| 160 |
+
]:
|
| 161 |
+
midpoint = (maxima + minima) * 0.5
|
| 162 |
+
span = (maxima - minima).max() * (1 + margin)
|
| 163 |
+
scene_minima = midpoint - 0.5 * span
|
| 164 |
+
scene_maxima = midpoint + 0.5 * span
|
| 165 |
+
return scene_minima, scene_maxima
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def unproject_frustum_corners(
|
| 169 |
+
extrinsics: Float[Tensor, "batch 4 4"],
|
| 170 |
+
intrinsics: Float[Tensor, "batch 3 3"],
|
| 171 |
+
depth: Float[Tensor, "#batch"],
|
| 172 |
+
) -> Float[Tensor, "batch 4 3"]:
|
| 173 |
+
device = extrinsics.device
|
| 174 |
+
|
| 175 |
+
# Get coordinates for the corners. Following them in a circle makes a rectangle.
|
| 176 |
+
xy = torch.linspace(0, 1, 2, device=device)
|
| 177 |
+
xy = torch.stack(torch.meshgrid(xy, xy, indexing="xy"), dim=-1)
|
| 178 |
+
xy = rearrange(xy, "i j xy -> (i j) xy")
|
| 179 |
+
xy = xy[torch.tensor([0, 1, 3, 2], device=device)]
|
| 180 |
+
|
| 181 |
+
# Get ray directions in camera space.
|
| 182 |
+
directions = unproject(
|
| 183 |
+
xy,
|
| 184 |
+
torch.ones(1, dtype=torch.float32, device=device),
|
| 185 |
+
rearrange(intrinsics, "b i j -> b () i j"),
|
| 186 |
+
)
|
| 187 |
+
|
| 188 |
+
# Divide by the z coordinate so that multiplying by depth will produce orthographic
|
| 189 |
+
# depth (z depth) as opposed to Euclidean depth (distance from the camera).
|
| 190 |
+
directions = directions / directions[..., -1:]
|
| 191 |
+
directions = einsum(extrinsics[..., :3, :3], directions, "b i j, b r j -> b r i")
|
| 192 |
+
|
| 193 |
+
origins = rearrange(extrinsics[:, :3, 3], "b xyz -> b () xyz")
|
| 194 |
+
depth = rearrange(depth, "b -> b () ()")
|
| 195 |
+
return origins + depth * directions
|
src/visualization/drawing/coordinate_conversion.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional, Protocol, runtime_checkable
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from jaxtyping import Float
|
| 5 |
+
from torch import Tensor
|
| 6 |
+
|
| 7 |
+
from .types import Pair, sanitize_pair
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
@runtime_checkable
|
| 11 |
+
class ConversionFunction(Protocol):
|
| 12 |
+
def __call__(
|
| 13 |
+
self,
|
| 14 |
+
xy: Float[Tensor, "*batch 2"],
|
| 15 |
+
) -> Float[Tensor, "*batch 2"]:
|
| 16 |
+
pass
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def generate_conversions(
|
| 20 |
+
shape: tuple[int, int],
|
| 21 |
+
device: torch.device,
|
| 22 |
+
x_range: Optional[Pair] = None,
|
| 23 |
+
y_range: Optional[Pair] = None,
|
| 24 |
+
) -> tuple[
|
| 25 |
+
ConversionFunction, # conversion from world coordinates to pixel coordinates
|
| 26 |
+
ConversionFunction, # conversion from pixel coordinates to world coordinates
|
| 27 |
+
]:
|
| 28 |
+
h, w = shape
|
| 29 |
+
x_range = sanitize_pair((0, w) if x_range is None else x_range, device)
|
| 30 |
+
y_range = sanitize_pair((0, h) if y_range is None else y_range, device)
|
| 31 |
+
minima, maxima = torch.stack((x_range, y_range), dim=-1)
|
| 32 |
+
wh = torch.tensor((w, h), dtype=torch.float32, device=device)
|
| 33 |
+
|
| 34 |
+
def convert_world_to_pixel(
|
| 35 |
+
xy: Float[Tensor, "*batch 2"],
|
| 36 |
+
) -> Float[Tensor, "*batch 2"]:
|
| 37 |
+
return (xy - minima) / (maxima - minima) * wh
|
| 38 |
+
|
| 39 |
+
def convert_pixel_to_world(
|
| 40 |
+
xy: Float[Tensor, "*batch 2"],
|
| 41 |
+
) -> Float[Tensor, "*batch 2"]:
|
| 42 |
+
return xy / wh * (maxima - minima) + minima
|
| 43 |
+
|
| 44 |
+
return convert_world_to_pixel, convert_pixel_to_world
|
src/visualization/drawing/lines.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Literal, Optional
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from einops import einsum, repeat
|
| 5 |
+
from jaxtyping import Float
|
| 6 |
+
from torch import Tensor
|
| 7 |
+
|
| 8 |
+
from .coordinate_conversion import generate_conversions
|
| 9 |
+
from .rendering import render_over_image
|
| 10 |
+
from .types import Pair, Scalar, Vector, sanitize_scalar, sanitize_vector
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def draw_lines(
|
| 14 |
+
image: Float[Tensor, "3 height width"],
|
| 15 |
+
start: Vector,
|
| 16 |
+
end: Vector,
|
| 17 |
+
color: Vector,
|
| 18 |
+
width: Scalar,
|
| 19 |
+
cap: Literal["butt", "round", "square"] = "round",
|
| 20 |
+
num_msaa_passes: int = 1,
|
| 21 |
+
x_range: Optional[Pair] = None,
|
| 22 |
+
y_range: Optional[Pair] = None,
|
| 23 |
+
) -> Float[Tensor, "3 height width"]:
|
| 24 |
+
device = image.device
|
| 25 |
+
start = sanitize_vector(start, 2, device)
|
| 26 |
+
end = sanitize_vector(end, 2, device)
|
| 27 |
+
color = sanitize_vector(color, 3, device)
|
| 28 |
+
width = sanitize_scalar(width, device)
|
| 29 |
+
(num_lines,) = torch.broadcast_shapes(
|
| 30 |
+
start.shape[0],
|
| 31 |
+
end.shape[0],
|
| 32 |
+
color.shape[0],
|
| 33 |
+
width.shape,
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
# Convert world-space points to pixel space.
|
| 37 |
+
_, h, w = image.shape
|
| 38 |
+
world_to_pixel, _ = generate_conversions((h, w), device, x_range, y_range)
|
| 39 |
+
start = world_to_pixel(start)
|
| 40 |
+
end = world_to_pixel(end)
|
| 41 |
+
|
| 42 |
+
def color_function(
|
| 43 |
+
xy: Float[Tensor, "point 2"],
|
| 44 |
+
) -> Float[Tensor, "point 4"]:
|
| 45 |
+
# Define a vector between the start and end points.
|
| 46 |
+
delta = end - start
|
| 47 |
+
delta_norm = delta.norm(dim=-1, keepdim=True)
|
| 48 |
+
u_delta = delta / delta_norm
|
| 49 |
+
|
| 50 |
+
# Define a vector between each sample and the start point.
|
| 51 |
+
indicator = xy - start[:, None]
|
| 52 |
+
|
| 53 |
+
# Determine whether each sample is inside the line in the parallel direction.
|
| 54 |
+
extra = 0.5 * width[:, None] if cap == "square" else 0
|
| 55 |
+
parallel = einsum(u_delta, indicator, "l xy, l s xy -> l s")
|
| 56 |
+
parallel_inside_line = (parallel <= delta_norm + extra) & (parallel > -extra)
|
| 57 |
+
|
| 58 |
+
# Determine whether each sample is inside the line perpendicularly.
|
| 59 |
+
perpendicular = indicator - parallel[..., None] * u_delta[:, None]
|
| 60 |
+
perpendicular_inside_line = perpendicular.norm(dim=-1) < 0.5 * width[:, None]
|
| 61 |
+
|
| 62 |
+
inside_line = parallel_inside_line & perpendicular_inside_line
|
| 63 |
+
|
| 64 |
+
# Compute round caps.
|
| 65 |
+
if cap == "round":
|
| 66 |
+
near_start = indicator.norm(dim=-1) < 0.5 * width[:, None]
|
| 67 |
+
inside_line |= near_start
|
| 68 |
+
end_indicator = indicator = xy - end[:, None]
|
| 69 |
+
near_end = end_indicator.norm(dim=-1) < 0.5 * width[:, None]
|
| 70 |
+
inside_line |= near_end
|
| 71 |
+
|
| 72 |
+
# Determine the sample's color.
|
| 73 |
+
selectable_color = color.broadcast_to((num_lines, 3))
|
| 74 |
+
arrangement = inside_line * torch.arange(num_lines, device=device)[:, None]
|
| 75 |
+
top_color = selectable_color.gather(
|
| 76 |
+
dim=0,
|
| 77 |
+
index=repeat(arrangement.argmax(dim=0), "s -> s c", c=3),
|
| 78 |
+
)
|
| 79 |
+
rgba = torch.cat((top_color, inside_line.any(dim=0).float()[:, None]), dim=-1)
|
| 80 |
+
|
| 81 |
+
return rgba
|
| 82 |
+
|
| 83 |
+
return render_over_image(image, color_function, device, num_passes=num_msaa_passes)
|
src/visualization/drawing/points.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from einops import repeat
|
| 5 |
+
from jaxtyping import Float
|
| 6 |
+
from torch import Tensor
|
| 7 |
+
|
| 8 |
+
from .coordinate_conversion import generate_conversions
|
| 9 |
+
from .rendering import render_over_image
|
| 10 |
+
from .types import Pair, Scalar, Vector, sanitize_scalar, sanitize_vector
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def draw_points(
|
| 14 |
+
image: Float[Tensor, "3 height width"],
|
| 15 |
+
points: Vector,
|
| 16 |
+
color: Vector = [1, 1, 1],
|
| 17 |
+
radius: Scalar = 1,
|
| 18 |
+
inner_radius: Scalar = 0,
|
| 19 |
+
num_msaa_passes: int = 1,
|
| 20 |
+
x_range: Optional[Pair] = None,
|
| 21 |
+
y_range: Optional[Pair] = None,
|
| 22 |
+
) -> Float[Tensor, "3 height width"]:
|
| 23 |
+
device = image.device
|
| 24 |
+
points = sanitize_vector(points, 2, device)
|
| 25 |
+
color = sanitize_vector(color, 3, device)
|
| 26 |
+
radius = sanitize_scalar(radius, device)
|
| 27 |
+
inner_radius = sanitize_scalar(inner_radius, device)
|
| 28 |
+
(num_points,) = torch.broadcast_shapes(
|
| 29 |
+
points.shape[0],
|
| 30 |
+
color.shape[0],
|
| 31 |
+
radius.shape,
|
| 32 |
+
inner_radius.shape,
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
# Convert world-space points to pixel space.
|
| 36 |
+
_, h, w = image.shape
|
| 37 |
+
world_to_pixel, _ = generate_conversions((h, w), device, x_range, y_range)
|
| 38 |
+
points = world_to_pixel(points)
|
| 39 |
+
|
| 40 |
+
def color_function(
|
| 41 |
+
xy: Float[Tensor, "point 2"],
|
| 42 |
+
) -> Float[Tensor, "point 4"]:
|
| 43 |
+
# Define a vector between the start and end points.
|
| 44 |
+
delta = xy[:, None] - points[None]
|
| 45 |
+
delta_norm = delta.norm(dim=-1)
|
| 46 |
+
mask = (delta_norm >= inner_radius[None]) & (delta_norm <= radius[None])
|
| 47 |
+
|
| 48 |
+
# Determine the sample's color.
|
| 49 |
+
selectable_color = color.broadcast_to((num_points, 3))
|
| 50 |
+
arrangement = mask * torch.arange(num_points, device=device)
|
| 51 |
+
top_color = selectable_color.gather(
|
| 52 |
+
dim=0,
|
| 53 |
+
index=repeat(arrangement.argmax(dim=1), "s -> s c", c=3),
|
| 54 |
+
)
|
| 55 |
+
rgba = torch.cat((top_color, mask.any(dim=1).float()[:, None]), dim=-1)
|
| 56 |
+
|
| 57 |
+
return rgba
|
| 58 |
+
|
| 59 |
+
return render_over_image(image, color_function, device, num_passes=num_msaa_passes)
|
src/visualization/drawing/rendering.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Protocol, runtime_checkable
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from einops import rearrange, reduce
|
| 5 |
+
from jaxtyping import Bool, Float
|
| 6 |
+
from torch import Tensor
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
@runtime_checkable
|
| 10 |
+
class ColorFunction(Protocol):
|
| 11 |
+
def __call__(
|
| 12 |
+
self,
|
| 13 |
+
xy: Float[Tensor, "point 2"],
|
| 14 |
+
) -> Float[Tensor, "point 4"]: # RGBA color
|
| 15 |
+
pass
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def generate_sample_grid(
|
| 19 |
+
shape: tuple[int, int],
|
| 20 |
+
device: torch.device,
|
| 21 |
+
) -> Float[Tensor, "height width 2"]:
|
| 22 |
+
h, w = shape
|
| 23 |
+
x = torch.arange(w, device=device) + 0.5
|
| 24 |
+
y = torch.arange(h, device=device) + 0.5
|
| 25 |
+
x, y = torch.meshgrid(x, y, indexing="xy")
|
| 26 |
+
return torch.stack([x, y], dim=-1)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def detect_msaa_pixels(
|
| 30 |
+
image: Float[Tensor, "batch 4 height width"],
|
| 31 |
+
) -> Bool[Tensor, "batch height width"]:
|
| 32 |
+
b, _, h, w = image.shape
|
| 33 |
+
|
| 34 |
+
mask = torch.zeros((b, h, w), dtype=torch.bool, device=image.device)
|
| 35 |
+
|
| 36 |
+
# Detect horizontal differences.
|
| 37 |
+
horizontal = (image[:, :, :, 1:] != image[:, :, :, :-1]).any(dim=1)
|
| 38 |
+
mask[:, :, 1:] |= horizontal
|
| 39 |
+
mask[:, :, :-1] |= horizontal
|
| 40 |
+
|
| 41 |
+
# Detect vertical differences.
|
| 42 |
+
vertical = (image[:, :, 1:, :] != image[:, :, :-1, :]).any(dim=1)
|
| 43 |
+
mask[:, 1:, :] |= vertical
|
| 44 |
+
mask[:, :-1, :] |= vertical
|
| 45 |
+
|
| 46 |
+
# Detect diagonal (top left to bottom right) differences.
|
| 47 |
+
tlbr = (image[:, :, 1:, 1:] != image[:, :, :-1, :-1]).any(dim=1)
|
| 48 |
+
mask[:, 1:, 1:] |= tlbr
|
| 49 |
+
mask[:, :-1, :-1] |= tlbr
|
| 50 |
+
|
| 51 |
+
# Detect diagonal (top right to bottom left) differences.
|
| 52 |
+
trbl = (image[:, :, :-1, 1:] != image[:, :, 1:, :-1]).any(dim=1)
|
| 53 |
+
mask[:, :-1, 1:] |= trbl
|
| 54 |
+
mask[:, 1:, :-1] |= trbl
|
| 55 |
+
|
| 56 |
+
return mask
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def reduce_straight_alpha(
|
| 60 |
+
rgba: Float[Tensor, "batch 4 height width"],
|
| 61 |
+
) -> Float[Tensor, "batch 4"]:
|
| 62 |
+
color, alpha = rgba.split((3, 1), dim=1)
|
| 63 |
+
|
| 64 |
+
# Color becomes a weighted average of color (weighted by alpha).
|
| 65 |
+
weighted_color = reduce(color * alpha, "b c h w -> b c", "sum")
|
| 66 |
+
alpha_sum = reduce(alpha, "b c h w -> b c", "sum")
|
| 67 |
+
color = weighted_color / (alpha_sum + 1e-10)
|
| 68 |
+
|
| 69 |
+
# Alpha becomes mean alpha.
|
| 70 |
+
alpha = reduce(alpha, "b c h w -> b c", "mean")
|
| 71 |
+
|
| 72 |
+
return torch.cat((color, alpha), dim=-1)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
@torch.no_grad()
|
| 76 |
+
def run_msaa_pass(
|
| 77 |
+
xy: Float[Tensor, "batch height width 2"],
|
| 78 |
+
color_function: ColorFunction,
|
| 79 |
+
scale: float,
|
| 80 |
+
subdivision: int,
|
| 81 |
+
remaining_passes: int,
|
| 82 |
+
device: torch.device,
|
| 83 |
+
batch_size: int = int(2**16),
|
| 84 |
+
) -> Float[Tensor, "batch 4 height width"]: # color (RGBA with straight alpha)
|
| 85 |
+
# Sample the color function.
|
| 86 |
+
b, h, w, _ = xy.shape
|
| 87 |
+
color = [
|
| 88 |
+
color_function(batch)
|
| 89 |
+
for batch in rearrange(xy, "b h w xy -> (b h w) xy").split(batch_size)
|
| 90 |
+
]
|
| 91 |
+
color = torch.cat(color, dim=0)
|
| 92 |
+
color = rearrange(color, "(b h w) c -> b c h w", b=b, h=h, w=w)
|
| 93 |
+
|
| 94 |
+
# If any MSAA passes remain, subdivide.
|
| 95 |
+
if remaining_passes > 0:
|
| 96 |
+
mask = detect_msaa_pixels(color)
|
| 97 |
+
batch_index, row_index, col_index = torch.where(mask)
|
| 98 |
+
xy = xy[batch_index, row_index, col_index]
|
| 99 |
+
|
| 100 |
+
offsets = generate_sample_grid((subdivision, subdivision), device)
|
| 101 |
+
offsets = (offsets / subdivision - 0.5) * scale
|
| 102 |
+
|
| 103 |
+
color_fine = run_msaa_pass(
|
| 104 |
+
xy[:, None, None] + offsets,
|
| 105 |
+
color_function,
|
| 106 |
+
scale / subdivision,
|
| 107 |
+
subdivision,
|
| 108 |
+
remaining_passes - 1,
|
| 109 |
+
device,
|
| 110 |
+
batch_size=batch_size,
|
| 111 |
+
)
|
| 112 |
+
color[batch_index, :, row_index, col_index] = reduce_straight_alpha(color_fine)
|
| 113 |
+
|
| 114 |
+
return color
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
@torch.no_grad()
|
| 118 |
+
def render(
|
| 119 |
+
shape: tuple[int, int],
|
| 120 |
+
color_function: ColorFunction,
|
| 121 |
+
device: torch.device,
|
| 122 |
+
subdivision: int = 8,
|
| 123 |
+
num_passes: int = 2,
|
| 124 |
+
) -> Float[Tensor, "4 height width"]: # color (RGBA with straight alpha)
|
| 125 |
+
xy = generate_sample_grid(shape, device)
|
| 126 |
+
return run_msaa_pass(
|
| 127 |
+
xy[None],
|
| 128 |
+
color_function,
|
| 129 |
+
1.0,
|
| 130 |
+
subdivision,
|
| 131 |
+
num_passes,
|
| 132 |
+
device,
|
| 133 |
+
)[0]
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def render_over_image(
|
| 137 |
+
image: Float[Tensor, "3 height width"],
|
| 138 |
+
color_function: ColorFunction,
|
| 139 |
+
device: torch.device,
|
| 140 |
+
subdivision: int = 8,
|
| 141 |
+
num_passes: int = 1,
|
| 142 |
+
) -> Float[Tensor, "3 height width"]:
|
| 143 |
+
_, h, w = image.shape
|
| 144 |
+
overlay = render(
|
| 145 |
+
(h, w),
|
| 146 |
+
color_function,
|
| 147 |
+
device,
|
| 148 |
+
subdivision=subdivision,
|
| 149 |
+
num_passes=num_passes,
|
| 150 |
+
)
|
| 151 |
+
color, alpha = overlay.split((3, 1), dim=0)
|
| 152 |
+
return image * (1 - alpha) + color * alpha
|
src/visualization/drawing/types.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Iterable, Union
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from einops import repeat
|
| 5 |
+
from jaxtyping import Float, Shaped
|
| 6 |
+
from torch import Tensor
|
| 7 |
+
|
| 8 |
+
Real = Union[float, int]
|
| 9 |
+
|
| 10 |
+
Vector = Union[
|
| 11 |
+
Real,
|
| 12 |
+
Iterable[Real],
|
| 13 |
+
Shaped[Tensor, "dim"],
|
| 14 |
+
Shaped[Tensor, "batch dim"],
|
| 15 |
+
]
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def sanitize_vector(
|
| 19 |
+
vector: Vector,
|
| 20 |
+
dim: int,
|
| 21 |
+
device: torch.device,
|
| 22 |
+
) -> Float[Tensor, "*#batch dim"]:
|
| 23 |
+
if isinstance(vector, Tensor):
|
| 24 |
+
vector = vector.type(torch.float32).to(device)
|
| 25 |
+
else:
|
| 26 |
+
vector = torch.tensor(vector, dtype=torch.float32, device=device)
|
| 27 |
+
while vector.ndim < 2:
|
| 28 |
+
vector = vector[None]
|
| 29 |
+
if vector.shape[-1] == 1:
|
| 30 |
+
vector = repeat(vector, "... () -> ... c", c=dim)
|
| 31 |
+
assert vector.shape[-1] == dim
|
| 32 |
+
assert vector.ndim == 2
|
| 33 |
+
return vector
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
Scalar = Union[
|
| 37 |
+
Real,
|
| 38 |
+
Iterable[Real],
|
| 39 |
+
Shaped[Tensor, ""],
|
| 40 |
+
Shaped[Tensor, " batch"],
|
| 41 |
+
]
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def sanitize_scalar(scalar: Scalar, device: torch.device) -> Float[Tensor, "*#batch"]:
|
| 45 |
+
if isinstance(scalar, Tensor):
|
| 46 |
+
scalar = scalar.type(torch.float32).to(device)
|
| 47 |
+
else:
|
| 48 |
+
scalar = torch.tensor(scalar, dtype=torch.float32, device=device)
|
| 49 |
+
while scalar.ndim < 1:
|
| 50 |
+
scalar = scalar[None]
|
| 51 |
+
assert scalar.ndim == 1
|
| 52 |
+
return scalar
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
Pair = Union[
|
| 56 |
+
Iterable[Real],
|
| 57 |
+
Shaped[Tensor, "2"],
|
| 58 |
+
]
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def sanitize_pair(pair: Pair, device: torch.device) -> Float[Tensor, "2"]:
|
| 62 |
+
if isinstance(pair, Tensor):
|
| 63 |
+
pair = pair.type(torch.float32).to(device)
|
| 64 |
+
else:
|
| 65 |
+
pair = torch.tensor(pair, dtype=torch.float32, device=device)
|
| 66 |
+
assert pair.shape == (2,)
|
| 67 |
+
return pair
|
src/visualization/layout.py
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""This file contains useful layout utilities for images. They are:
|
| 2 |
+
|
| 3 |
+
- add_border: Add a border to an image.
|
| 4 |
+
- cat/hcat/vcat: Join images by arranging them in a line. If the images have different
|
| 5 |
+
sizes, they are aligned as specified (start, end, center). Allows you to specify a gap
|
| 6 |
+
between images.
|
| 7 |
+
|
| 8 |
+
Images are assumed to be float32 tensors with shape (channel, height, width).
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from typing import Any, Generator, Iterable, Literal, Optional, Union
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
import torch.nn.functional as F
|
| 15 |
+
from jaxtyping import Float
|
| 16 |
+
from torch import Tensor
|
| 17 |
+
|
| 18 |
+
Alignment = Literal["start", "center", "end"]
|
| 19 |
+
Axis = Literal["horizontal", "vertical"]
|
| 20 |
+
Color = Union[
|
| 21 |
+
int,
|
| 22 |
+
float,
|
| 23 |
+
Iterable[int],
|
| 24 |
+
Iterable[float],
|
| 25 |
+
Float[Tensor, "#channel"],
|
| 26 |
+
Float[Tensor, ""],
|
| 27 |
+
]
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _sanitize_color(color: Color) -> Float[Tensor, "#channel"]:
|
| 31 |
+
# Convert tensor to list (or individual item).
|
| 32 |
+
if isinstance(color, torch.Tensor):
|
| 33 |
+
color = color.tolist()
|
| 34 |
+
|
| 35 |
+
# Turn iterators and individual items into lists.
|
| 36 |
+
if isinstance(color, Iterable):
|
| 37 |
+
color = list(color)
|
| 38 |
+
else:
|
| 39 |
+
color = [color]
|
| 40 |
+
|
| 41 |
+
return torch.tensor(color, dtype=torch.float32)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _intersperse(iterable: Iterable, delimiter: Any) -> Generator[Any, None, None]:
|
| 45 |
+
it = iter(iterable)
|
| 46 |
+
yield next(it)
|
| 47 |
+
for item in it:
|
| 48 |
+
yield delimiter
|
| 49 |
+
yield item
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _get_main_dim(main_axis: Axis) -> int:
|
| 53 |
+
return {
|
| 54 |
+
"horizontal": 2,
|
| 55 |
+
"vertical": 1,
|
| 56 |
+
}[main_axis]
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _get_cross_dim(main_axis: Axis) -> int:
|
| 60 |
+
return {
|
| 61 |
+
"horizontal": 1,
|
| 62 |
+
"vertical": 2,
|
| 63 |
+
}[main_axis]
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _compute_offset(base: int, overlay: int, align: Alignment) -> slice:
|
| 67 |
+
assert base >= overlay
|
| 68 |
+
offset = {
|
| 69 |
+
"start": 0,
|
| 70 |
+
"center": (base - overlay) // 2,
|
| 71 |
+
"end": base - overlay,
|
| 72 |
+
}[align]
|
| 73 |
+
return slice(offset, offset + overlay)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def overlay(
|
| 77 |
+
base: Float[Tensor, "channel base_height base_width"],
|
| 78 |
+
overlay: Float[Tensor, "channel overlay_height overlay_width"],
|
| 79 |
+
main_axis: Axis,
|
| 80 |
+
main_axis_alignment: Alignment,
|
| 81 |
+
cross_axis_alignment: Alignment,
|
| 82 |
+
) -> Float[Tensor, "channel base_height base_width"]:
|
| 83 |
+
# The overlay must be smaller than the base.
|
| 84 |
+
_, base_height, base_width = base.shape
|
| 85 |
+
_, overlay_height, overlay_width = overlay.shape
|
| 86 |
+
assert base_height >= overlay_height and base_width >= overlay_width
|
| 87 |
+
|
| 88 |
+
# Compute spacing on the main dimension.
|
| 89 |
+
main_dim = _get_main_dim(main_axis)
|
| 90 |
+
main_slice = _compute_offset(
|
| 91 |
+
base.shape[main_dim], overlay.shape[main_dim], main_axis_alignment
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
# Compute spacing on the cross dimension.
|
| 95 |
+
cross_dim = _get_cross_dim(main_axis)
|
| 96 |
+
cross_slice = _compute_offset(
|
| 97 |
+
base.shape[cross_dim], overlay.shape[cross_dim], cross_axis_alignment
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
# Combine the slices and paste the overlay onto the base accordingly.
|
| 101 |
+
selector = [..., None, None]
|
| 102 |
+
selector[main_dim] = main_slice
|
| 103 |
+
selector[cross_dim] = cross_slice
|
| 104 |
+
result = base.clone()
|
| 105 |
+
result[selector] = overlay
|
| 106 |
+
return result
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def cat(
|
| 110 |
+
main_axis: Axis,
|
| 111 |
+
*images: Iterable[Float[Tensor, "channel _ _"]],
|
| 112 |
+
align: Alignment = "center",
|
| 113 |
+
gap: int = 8,
|
| 114 |
+
gap_color: Color = 1,
|
| 115 |
+
) -> Float[Tensor, "channel height width"]:
|
| 116 |
+
"""Arrange images in a line. The interface resembles a CSS div with flexbox."""
|
| 117 |
+
device = images[0].device
|
| 118 |
+
gap_color = _sanitize_color(gap_color).to(device)
|
| 119 |
+
|
| 120 |
+
# Find the maximum image side length in the cross axis dimension.
|
| 121 |
+
cross_dim = _get_cross_dim(main_axis)
|
| 122 |
+
cross_axis_length = max(image.shape[cross_dim] for image in images)
|
| 123 |
+
|
| 124 |
+
# Pad the images.
|
| 125 |
+
padded_images = []
|
| 126 |
+
for image in images:
|
| 127 |
+
# Create an empty image with the correct size.
|
| 128 |
+
padded_shape = list(image.shape)
|
| 129 |
+
padded_shape[cross_dim] = cross_axis_length
|
| 130 |
+
base = torch.ones(padded_shape, dtype=torch.float32, device=device)
|
| 131 |
+
base = base * gap_color[:, None, None]
|
| 132 |
+
padded_images.append(overlay(base, image, main_axis, "start", align))
|
| 133 |
+
|
| 134 |
+
# Intersperse separators if necessary.
|
| 135 |
+
if gap > 0:
|
| 136 |
+
# Generate a separator.
|
| 137 |
+
c, _, _ = images[0].shape
|
| 138 |
+
separator_size = [gap, gap]
|
| 139 |
+
separator_size[cross_dim - 1] = cross_axis_length
|
| 140 |
+
separator = torch.ones((c, *separator_size), dtype=torch.float32, device=device)
|
| 141 |
+
separator = separator * gap_color[:, None, None]
|
| 142 |
+
|
| 143 |
+
# Intersperse the separator between the images.
|
| 144 |
+
padded_images = list(_intersperse(padded_images, separator))
|
| 145 |
+
|
| 146 |
+
return torch.cat(padded_images, dim=_get_main_dim(main_axis))
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def hcat(
|
| 150 |
+
*images: Iterable[Float[Tensor, "channel _ _"]],
|
| 151 |
+
align: Literal["start", "center", "end", "top", "bottom"] = "start",
|
| 152 |
+
gap: int = 8,
|
| 153 |
+
gap_color: Color = 1,
|
| 154 |
+
):
|
| 155 |
+
"""Shorthand for a horizontal linear concatenation."""
|
| 156 |
+
return cat(
|
| 157 |
+
"horizontal",
|
| 158 |
+
*images,
|
| 159 |
+
align={
|
| 160 |
+
"start": "start",
|
| 161 |
+
"center": "center",
|
| 162 |
+
"end": "end",
|
| 163 |
+
"top": "start",
|
| 164 |
+
"bottom": "end",
|
| 165 |
+
}[align],
|
| 166 |
+
gap=gap,
|
| 167 |
+
gap_color=gap_color,
|
| 168 |
+
)
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def vcat(
|
| 172 |
+
*images: Iterable[Float[Tensor, "channel _ _"]],
|
| 173 |
+
align: Literal["start", "center", "end", "left", "right"] = "start",
|
| 174 |
+
gap: int = 8,
|
| 175 |
+
gap_color: Color = 1,
|
| 176 |
+
):
|
| 177 |
+
"""Shorthand for a horizontal linear concatenation."""
|
| 178 |
+
return cat(
|
| 179 |
+
"vertical",
|
| 180 |
+
*images,
|
| 181 |
+
align={
|
| 182 |
+
"start": "start",
|
| 183 |
+
"center": "center",
|
| 184 |
+
"end": "end",
|
| 185 |
+
"left": "start",
|
| 186 |
+
"right": "end",
|
| 187 |
+
}[align],
|
| 188 |
+
gap=gap,
|
| 189 |
+
gap_color=gap_color,
|
| 190 |
+
)
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def add_border(
|
| 194 |
+
image: Float[Tensor, "channel height width"],
|
| 195 |
+
border: int = 8,
|
| 196 |
+
color: Color = 1,
|
| 197 |
+
) -> Float[Tensor, "channel new_height new_width"]:
|
| 198 |
+
color = _sanitize_color(color).to(image)
|
| 199 |
+
c, h, w = image.shape
|
| 200 |
+
result = torch.empty(
|
| 201 |
+
(c, h + 2 * border, w + 2 * border), dtype=torch.float32, device=image.device
|
| 202 |
+
)
|
| 203 |
+
result[:] = color[:, None, None]
|
| 204 |
+
result[:, border : h + border, border : w + border] = image
|
| 205 |
+
return result
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def resize(
|
| 209 |
+
image: Float[Tensor, "channel height width"],
|
| 210 |
+
shape: Optional[tuple[int, int]] = None,
|
| 211 |
+
width: Optional[int] = None,
|
| 212 |
+
height: Optional[int] = None,
|
| 213 |
+
) -> Float[Tensor, "channel new_height new_width"]:
|
| 214 |
+
assert (shape is not None) + (width is not None) + (height is not None) == 1
|
| 215 |
+
_, h, w = image.shape
|
| 216 |
+
|
| 217 |
+
if width is not None:
|
| 218 |
+
shape = (int(h * width / w), width)
|
| 219 |
+
elif height is not None:
|
| 220 |
+
shape = (height, int(w * height / h))
|
| 221 |
+
|
| 222 |
+
return F.interpolate(
|
| 223 |
+
image[None],
|
| 224 |
+
shape,
|
| 225 |
+
mode="bilinear",
|
| 226 |
+
align_corners=False,
|
| 227 |
+
antialias="bilinear",
|
| 228 |
+
)[0]
|
src/visualization/validation_in_3d.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from jaxtyping import Float, Shaped
|
| 3 |
+
from torch import Tensor
|
| 4 |
+
|
| 5 |
+
from ..model.decoder.cuda_splatting import render_cuda_orthographic
|
| 6 |
+
from ..model.types import Gaussians
|
| 7 |
+
from ..visualization.annotation import add_label
|
| 8 |
+
from ..visualization.drawing.cameras import draw_cameras
|
| 9 |
+
from .drawing.cameras import compute_equal_aabb_with_margin
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def pad(images: list[Shaped[Tensor, "..."]]) -> list[Shaped[Tensor, "..."]]:
|
| 13 |
+
shapes = torch.stack([torch.tensor(x.shape) for x in images])
|
| 14 |
+
padded_shape = shapes.max(dim=0)[0]
|
| 15 |
+
results = [
|
| 16 |
+
torch.ones(padded_shape.tolist(), dtype=x.dtype, device=x.device)
|
| 17 |
+
for x in images
|
| 18 |
+
]
|
| 19 |
+
for image, result in zip(images, results):
|
| 20 |
+
slices = [slice(0, x) for x in image.shape]
|
| 21 |
+
result[slices] = image[slices]
|
| 22 |
+
return results
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def render_projections(
|
| 26 |
+
gaussians: Gaussians,
|
| 27 |
+
resolution: int,
|
| 28 |
+
margin: float = 0.1,
|
| 29 |
+
draw_label: bool = True,
|
| 30 |
+
extra_label: str = "",
|
| 31 |
+
) -> Float[Tensor, "batch 3 3 height width"]:
|
| 32 |
+
device = gaussians.means.device
|
| 33 |
+
b, _, _ = gaussians.means.shape
|
| 34 |
+
|
| 35 |
+
# Compute the minima and maxima of the scene.
|
| 36 |
+
minima = gaussians.means.min(dim=1).values
|
| 37 |
+
maxima = gaussians.means.max(dim=1).values
|
| 38 |
+
scene_minima, scene_maxima = compute_equal_aabb_with_margin(
|
| 39 |
+
minima, maxima, margin=margin
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
projections = []
|
| 43 |
+
for look_axis in range(3):
|
| 44 |
+
right_axis = (look_axis + 1) % 3
|
| 45 |
+
down_axis = (look_axis + 2) % 3
|
| 46 |
+
|
| 47 |
+
# Define the extrinsics for rendering.
|
| 48 |
+
extrinsics = torch.zeros((b, 4, 4), dtype=torch.float32, device=device)
|
| 49 |
+
extrinsics[:, right_axis, 0] = 1
|
| 50 |
+
extrinsics[:, down_axis, 1] = 1
|
| 51 |
+
extrinsics[:, look_axis, 2] = 1
|
| 52 |
+
extrinsics[:, right_axis, 3] = 0.5 * (
|
| 53 |
+
scene_minima[:, right_axis] + scene_maxima[:, right_axis]
|
| 54 |
+
)
|
| 55 |
+
extrinsics[:, down_axis, 3] = 0.5 * (
|
| 56 |
+
scene_minima[:, down_axis] + scene_maxima[:, down_axis]
|
| 57 |
+
)
|
| 58 |
+
extrinsics[:, look_axis, 3] = scene_minima[:, look_axis]
|
| 59 |
+
extrinsics[:, 3, 3] = 1
|
| 60 |
+
|
| 61 |
+
# Define the intrinsics for rendering.
|
| 62 |
+
extents = scene_maxima - scene_minima
|
| 63 |
+
far = extents[:, look_axis]
|
| 64 |
+
near = torch.zeros_like(far)
|
| 65 |
+
width = extents[:, right_axis]
|
| 66 |
+
height = extents[:, down_axis]
|
| 67 |
+
|
| 68 |
+
projection = render_cuda_orthographic(
|
| 69 |
+
extrinsics,
|
| 70 |
+
width,
|
| 71 |
+
height,
|
| 72 |
+
near,
|
| 73 |
+
far,
|
| 74 |
+
(resolution, resolution),
|
| 75 |
+
torch.zeros((b, 3), dtype=torch.float32, device=device),
|
| 76 |
+
gaussians.means,
|
| 77 |
+
gaussians.covariances,
|
| 78 |
+
gaussians.harmonics,
|
| 79 |
+
gaussians.opacities,
|
| 80 |
+
fov_degrees=10.0,
|
| 81 |
+
)
|
| 82 |
+
if draw_label:
|
| 83 |
+
right_axis_name = "XYZ"[right_axis]
|
| 84 |
+
down_axis_name = "XYZ"[down_axis]
|
| 85 |
+
label = f"{right_axis_name}{down_axis_name} Projection {extra_label}"
|
| 86 |
+
projection = torch.stack([add_label(x, label) for x in projection])
|
| 87 |
+
|
| 88 |
+
projections.append(projection)
|
| 89 |
+
|
| 90 |
+
return torch.stack(pad(projections), dim=1)
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def render_cameras(batch: dict, resolution: int) -> Float[Tensor, "3 3 height width"]:
|
| 94 |
+
# Define colors for context and target views.
|
| 95 |
+
num_context_views = batch["context"]["extrinsics"].shape[1]
|
| 96 |
+
num_target_views = batch["target"]["extrinsics"].shape[1]
|
| 97 |
+
color = torch.ones(
|
| 98 |
+
(num_target_views + num_context_views, 3),
|
| 99 |
+
dtype=torch.float32,
|
| 100 |
+
device=batch["target"]["extrinsics"].device,
|
| 101 |
+
)
|
| 102 |
+
color[num_context_views:, 1:] = 0
|
| 103 |
+
|
| 104 |
+
return draw_cameras(
|
| 105 |
+
resolution,
|
| 106 |
+
torch.cat(
|
| 107 |
+
(batch["context"]["extrinsics"][0], batch["target"]["extrinsics"][0])
|
| 108 |
+
),
|
| 109 |
+
torch.cat(
|
| 110 |
+
(batch["context"]["intrinsics"][0], batch["target"]["intrinsics"][0])
|
| 111 |
+
),
|
| 112 |
+
color,
|
| 113 |
+
torch.cat((batch["context"]["near"][0], batch["target"]["near"][0])),
|
| 114 |
+
torch.cat((batch["context"]["far"][0], batch["target"]["far"][0])),
|
| 115 |
+
)
|
src/visualization/video_render.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Protocol, runtime_checkable
|
| 2 |
+
|
| 3 |
+
import cv2
|
| 4 |
+
import torch
|
| 5 |
+
from einops import repeat, pack
|
| 6 |
+
from jaxtyping import Float
|
| 7 |
+
from torch import Tensor
|
| 8 |
+
|
| 9 |
+
from .camera_trajectory.interpolation import interpolate_extrinsics, interpolate_intrinsics
|
| 10 |
+
from .camera_trajectory.wobble import generate_wobble, generate_wobble_transformation
|
| 11 |
+
from .layout import vcat
|
| 12 |
+
from ..dataset.types import BatchedExample
|
| 13 |
+
from ..misc.image_io import save_video
|
| 14 |
+
from ..misc.utils import vis_depth_map
|
| 15 |
+
from ..model.decoder import Decoder
|
| 16 |
+
from ..model.types import Gaussians
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@runtime_checkable
|
| 20 |
+
class TrajectoryFn(Protocol):
|
| 21 |
+
def __call__(
|
| 22 |
+
self,
|
| 23 |
+
t: Float[Tensor, " t"],
|
| 24 |
+
) -> tuple[
|
| 25 |
+
Float[Tensor, "batch view 4 4"], # extrinsics
|
| 26 |
+
Float[Tensor, "batch view 3 3"], # intrinsics
|
| 27 |
+
]:
|
| 28 |
+
pass
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def render_video_wobble(
|
| 32 |
+
gaussians: Gaussians,
|
| 33 |
+
decoder: Decoder,
|
| 34 |
+
batch: BatchedExample,
|
| 35 |
+
num_frames: int = 60,
|
| 36 |
+
smooth: bool = True,
|
| 37 |
+
loop_reverse: bool = True,
|
| 38 |
+
add_depth: bool = False,
|
| 39 |
+
) -> Tensor:
|
| 40 |
+
# Two views are needed to get the wobble radius,use the first and the last view
|
| 41 |
+
_, v, _, _ = batch["context"]["extrinsics"].shape
|
| 42 |
+
|
| 43 |
+
def trajectory_fn(t):
|
| 44 |
+
origin_a = batch["context"]["extrinsics"][:, 0, :3, 3]
|
| 45 |
+
origin_b = batch["context"]["extrinsics"][:, -1, :3, 3]
|
| 46 |
+
delta = (origin_a - origin_b).norm(dim=-1)
|
| 47 |
+
extrinsics = generate_wobble(
|
| 48 |
+
batch["context"]["extrinsics"][:, 0],
|
| 49 |
+
delta * 0.25,
|
| 50 |
+
t,
|
| 51 |
+
)
|
| 52 |
+
intrinsics = repeat(
|
| 53 |
+
batch["context"]["intrinsics"][:, 0],
|
| 54 |
+
"b i j -> b v i j",
|
| 55 |
+
v=t.shape[0],
|
| 56 |
+
)
|
| 57 |
+
return extrinsics, intrinsics
|
| 58 |
+
|
| 59 |
+
return render_video_generic(gaussians, decoder, batch, trajectory_fn, num_frames, smooth, loop_reverse, add_depth)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def render_video_interpolation(
|
| 63 |
+
gaussians: Gaussians,
|
| 64 |
+
decoder: Decoder,
|
| 65 |
+
batch: BatchedExample,
|
| 66 |
+
num_frames: int = 60,
|
| 67 |
+
smooth: bool = True,
|
| 68 |
+
loop_reverse: bool = True,
|
| 69 |
+
add_depth: bool = False,
|
| 70 |
+
) -> Tensor:
|
| 71 |
+
_, v, _, _ = batch["context"]["extrinsics"].shape
|
| 72 |
+
|
| 73 |
+
def trajectory_fn(t):
|
| 74 |
+
extrinsics = interpolate_extrinsics(
|
| 75 |
+
batch["context"]["extrinsics"][0, 0],
|
| 76 |
+
batch["context"]["extrinsics"][0, -1],
|
| 77 |
+
t,
|
| 78 |
+
)
|
| 79 |
+
intrinsics = interpolate_intrinsics(
|
| 80 |
+
batch["context"]["intrinsics"][0, 0],
|
| 81 |
+
batch["context"]["intrinsics"][0, -1],
|
| 82 |
+
t,
|
| 83 |
+
)
|
| 84 |
+
return extrinsics[None], intrinsics[None]
|
| 85 |
+
|
| 86 |
+
return render_video_generic(gaussians, decoder, batch, trajectory_fn, num_frames, smooth, loop_reverse, add_depth)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def render_video_interpolation_exaggerated(
|
| 90 |
+
gaussians: Gaussians,
|
| 91 |
+
decoder: Decoder,
|
| 92 |
+
batch: BatchedExample,
|
| 93 |
+
num_frames: int = 300,
|
| 94 |
+
smooth: bool = False,
|
| 95 |
+
loop_reverse: bool = False,
|
| 96 |
+
add_depth: bool = False,
|
| 97 |
+
) -> Tensor:
|
| 98 |
+
# Two views are needed to get the wobble radius.
|
| 99 |
+
_, v, _, _ = batch["context"]["extrinsics"].shape
|
| 100 |
+
|
| 101 |
+
def trajectory_fn(t):
|
| 102 |
+
origin_a = batch["context"]["extrinsics"][:, 0, :3, 3]
|
| 103 |
+
origin_b = batch["context"]["extrinsics"][:, -1, :3, 3]
|
| 104 |
+
delta = (origin_a - origin_b).norm(dim=-1)
|
| 105 |
+
tf = generate_wobble_transformation(
|
| 106 |
+
delta * 0.5,
|
| 107 |
+
t,
|
| 108 |
+
5,
|
| 109 |
+
scale_radius_with_t=False,
|
| 110 |
+
)
|
| 111 |
+
extrinsics = interpolate_extrinsics(
|
| 112 |
+
batch["context"]["extrinsics"][0, 0],
|
| 113 |
+
batch["context"]["extrinsics"][0, -1],
|
| 114 |
+
t * 5 - 2,
|
| 115 |
+
)
|
| 116 |
+
intrinsics = interpolate_intrinsics(
|
| 117 |
+
batch["context"]["intrinsics"][0, 0],
|
| 118 |
+
batch["context"]["extrinsics"][0, -1],
|
| 119 |
+
t * 5 - 2,
|
| 120 |
+
)
|
| 121 |
+
return extrinsics @ tf, intrinsics[None]
|
| 122 |
+
|
| 123 |
+
return render_video_generic(gaussians, decoder, batch, trajectory_fn, num_frames, smooth, loop_reverse, add_depth)
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def render_video_generic(
|
| 127 |
+
gaussians: Gaussians,
|
| 128 |
+
decoder: Decoder,
|
| 129 |
+
batch: BatchedExample,
|
| 130 |
+
trajectory_fn: TrajectoryFn,
|
| 131 |
+
num_frames: int = 30,
|
| 132 |
+
smooth: bool = True,
|
| 133 |
+
loop_reverse: bool = True,
|
| 134 |
+
add_depth: bool = False,
|
| 135 |
+
) -> Tensor:
|
| 136 |
+
device = gaussians.means.device
|
| 137 |
+
|
| 138 |
+
t = torch.linspace(0, 1, num_frames, dtype=torch.float32, device=device)
|
| 139 |
+
if smooth:
|
| 140 |
+
t = (torch.cos(torch.pi * (t + 1)) + 1) / 2
|
| 141 |
+
|
| 142 |
+
extrinsics, intrinsics = trajectory_fn(t)
|
| 143 |
+
|
| 144 |
+
_, _, _, h, w = batch["context"]["image"].shape
|
| 145 |
+
|
| 146 |
+
near = repeat(batch["context"]["near"][:, 0], "b -> b v", v=num_frames)
|
| 147 |
+
far = repeat(batch["context"]["far"][:, 0], "b -> b v", v=num_frames)
|
| 148 |
+
output = decoder.forward(
|
| 149 |
+
gaussians, extrinsics, intrinsics, near, far, (h, w), "depth"
|
| 150 |
+
)
|
| 151 |
+
images = [
|
| 152 |
+
vcat(rgb, depth) if add_depth else rgb
|
| 153 |
+
for rgb, depth in zip(output.color[0], vis_depth_map(output.depth[0]))
|
| 154 |
+
]
|
| 155 |
+
|
| 156 |
+
video = torch.stack(images)
|
| 157 |
+
# video = (video.clip(min=0, max=1) * 255).type(torch.uint8).cpu().numpy()
|
| 158 |
+
if loop_reverse:
|
| 159 |
+
# video = pack([video, video[::-1][1:-1]], "* c h w")[0]
|
| 160 |
+
video = pack([video, video.flip(dims=(0,))[1:-1]], "* c h w")[0]
|
| 161 |
+
|
| 162 |
+
return video
|