Aeonicc commited on
Commit
123da8a
·
verified ·
1 Parent(s): 8e540bb

Initial commit for brand new AnySplat ZeroGPU Space (part 5)

Browse files
Files changed (49) hide show
  1. src/model/encoder/visualization/encoder_visualizer_epipolar_cfg.py +10 -0
  2. src/model/encodings/positional_encoding.py +36 -0
  3. src/model/model/__init__.py +21 -0
  4. src/model/model/anysplat.py +126 -0
  5. src/model/model_wrapper.py +847 -0
  6. src/model/ply_export.py +74 -0
  7. src/model/transformer/attention.py +70 -0
  8. src/model/transformer/feed_forward.py +40 -0
  9. src/model/transformer/pre_norm.py +35 -0
  10. src/model/transformer/transformer.py +71 -0
  11. src/model/types.py +15 -0
  12. src/post_opt/datasets/colmap.py +490 -0
  13. src/post_opt/datasets/normalize.py +143 -0
  14. src/post_opt/datasets/traj.py +254 -0
  15. src/post_opt/exporter.py +553 -0
  16. src/post_opt/gsplat_viewer.py +249 -0
  17. src/post_opt/image_fitting.py +189 -0
  18. src/post_opt/lib_bilagrid.py +573 -0
  19. src/post_opt/simple_trainer.py +1433 -0
  20. src/post_opt/simple_viewer.py +275 -0
  21. src/post_opt/utils.py +224 -0
  22. src/utils/__init.py +0 -0
  23. src/utils/ba.py +407 -0
  24. src/utils/cropping.py +163 -0
  25. src/utils/device.py +95 -0
  26. src/utils/geometry.py +428 -0
  27. src/utils/image.py +185 -0
  28. src/utils/misc.py +531 -0
  29. src/utils/point.py +47 -0
  30. src/utils/pose.py +218 -0
  31. src/utils/render.py +284 -0
  32. src/utils/tensor_to_pycolmap.py +320 -0
  33. src/utils/transforms.py +18 -0
  34. src/utils/viz.py +377 -0
  35. src/visualization/annotation.py +49 -0
  36. src/visualization/camera_trajectory/interpolation.py +255 -0
  37. src/visualization/camera_trajectory/spin.py +37 -0
  38. src/visualization/camera_trajectory/wobble.py +32 -0
  39. src/visualization/color_map.py +48 -0
  40. src/visualization/colors.py +32 -0
  41. src/visualization/drawing/cameras.py +195 -0
  42. src/visualization/drawing/coordinate_conversion.py +44 -0
  43. src/visualization/drawing/lines.py +83 -0
  44. src/visualization/drawing/points.py +59 -0
  45. src/visualization/drawing/rendering.py +152 -0
  46. src/visualization/drawing/types.py +67 -0
  47. src/visualization/layout.py +228 -0
  48. src/visualization/validation_in_3d.py +115 -0
  49. src/visualization/video_render.py +162 -0
src/model/encoder/visualization/encoder_visualizer_epipolar_cfg.py ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+
3
+ # This is in a separate file to avoid circular imports.
4
+
5
+
6
+ @dataclass
7
+ class EncoderVisualizerEpipolarCfg:
8
+ num_samples: int
9
+ min_resolution: int
10
+ export_ply: bool
src/model/encodings/positional_encoding.py ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ from einops import einsum, rearrange, repeat
4
+ from jaxtyping import Float
5
+ from torch import Tensor
6
+
7
+
8
+ class PositionalEncoding(nn.Module):
9
+ """For the sake of simplicity, this encodes values in the range [0, 1]."""
10
+
11
+ frequencies: Float[Tensor, "frequency phase"]
12
+ phases: Float[Tensor, "frequency phase"]
13
+
14
+ def __init__(self, num_octaves: int):
15
+ super().__init__()
16
+ octaves = torch.arange(num_octaves).float()
17
+
18
+ # The lowest frequency has a period of 1.
19
+ frequencies = 2 * torch.pi * 2**octaves
20
+ frequencies = repeat(frequencies, "f -> f p", p=2)
21
+ self.register_buffer("frequencies", frequencies, persistent=False)
22
+
23
+ # Choose the phases to match sine and cosine.
24
+ phases = torch.tensor([0, 0.5 * torch.pi], dtype=torch.float32)
25
+ phases = repeat(phases, "p -> f p", f=num_octaves)
26
+ self.register_buffer("phases", phases, persistent=False)
27
+
28
+ def forward(
29
+ self,
30
+ samples: Float[Tensor, "*batch dim"],
31
+ ) -> Float[Tensor, "*batch embedded_dim"]:
32
+ samples = einsum(samples, self.frequencies, "... d, f p -> ... d f p")
33
+ return rearrange(torch.sin(samples + self.phases), "... d f p -> ... (d f p)")
34
+
35
+ def d_out(self, dimensionality: int):
36
+ return self.frequencies.numel() * dimensionality
src/model/model/__init__.py ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional, Union
2
+
3
+ from ..encoder import Encoder
4
+ from ..encoder.visualization.encoder_visualizer import EncoderVisualizer
5
+ from ..encoder.anysplat import EncoderAnySplat, EncoderAnySplatCfg
6
+ from ..decoder.decoder_splatting_cuda import DecoderSplattingCUDACfg
7
+ from torch import nn
8
+ from .anysplat import AnySplat
9
+
10
+ MODELS = {
11
+ "anysplat": AnySplat,
12
+ }
13
+
14
+ EncoderCfg = Union[EncoderAnySplatCfg]
15
+ DecoderCfg = DecoderSplattingCUDACfg
16
+
17
+
18
+ # hard code for now
19
+ def get_model(encoder_cfg: EncoderCfg, decoder_cfg: DecoderCfg) -> nn.Module:
20
+ model = MODELS['anysplat'](encoder_cfg, decoder_cfg)
21
+ return model
src/model/model/anysplat.py ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from copy import deepcopy
3
+ import time
4
+ from typing import Optional
5
+ from einops import rearrange
6
+ import huggingface_hub
7
+ from omegaconf import DictConfig, OmegaConf
8
+ import torch
9
+ import torch.distributed
10
+ import torch.nn as nn
11
+ import numpy as np
12
+ import torch.nn.functional as F
13
+ from dataclasses import dataclass
14
+
15
+ from src.model.encoder.common.gaussian_adapter import GaussianAdapterCfg
16
+ from src.model.decoder.decoder_splatting_cuda import DecoderSplattingCUDA, DecoderSplattingCUDACfg
17
+ from src.model.encoder.anysplat import EncoderAnySplat, EncoderAnySplatCfg, OpacityMappingCfg
18
+
19
+ class AnySplat(nn.Module, huggingface_hub.PyTorchModelHubMixin):
20
+ def __init__(
21
+ self,
22
+ encoder_cfg: EncoderAnySplatCfg,
23
+ decoder_cfg: DecoderSplattingCUDACfg,
24
+ ):
25
+ super(AnySplat, self).__init__()
26
+ self.encoder_cfg = encoder_cfg
27
+ self.decoder_cfg = decoder_cfg
28
+ self.build_encoder(encoder_cfg)
29
+ self.build_decoder(decoder_cfg)
30
+
31
+ def convert_nested_config(self, cfg_dict: dict, target_class: type):
32
+ """Convert nested dictionary config to dataclass instance
33
+
34
+ Args:
35
+ cfg_dict: Configuration dictionary or already converted object
36
+ target_class: Target dataclass type to convert to
37
+
38
+ Returns:
39
+ Instance of target_class
40
+ """
41
+ if isinstance(cfg_dict, dict):
42
+ # Convert dict to dataclass
43
+ return target_class(**cfg_dict)
44
+ elif isinstance(cfg_dict, target_class):
45
+ # Already converted, return as is
46
+ return cfg_dict
47
+ elif cfg_dict is None:
48
+ # Handle None case
49
+ return None
50
+ else:
51
+ raise ValueError(f"Cannot convert {type(cfg_dict)} to {target_class}")
52
+
53
+ def convert_config_recursively(self, cfg_obj, conversion_map: dict):
54
+ """Convert nested configurations recursively using a conversion map
55
+
56
+ Args:
57
+ cfg_obj: Configuration object to convert
58
+ conversion_map: Dict mapping field names to their target classes
59
+ e.g., {'gaussian_adapter': GaussianAdapterCfg}
60
+
61
+ Returns:
62
+ Converted configuration object
63
+ """
64
+ if not hasattr(cfg_obj, '__dict__'):
65
+ return cfg_obj
66
+
67
+ cfg_dict = cfg_obj.__dict__.copy()
68
+
69
+ for field_name, target_class in conversion_map.items():
70
+ if field_name in cfg_dict:
71
+ cfg_dict[field_name] = self.convert_nested_config(
72
+ cfg_dict[field_name],
73
+ target_class
74
+ )
75
+
76
+ # Return new instance of the same type
77
+ return type(cfg_obj)(**cfg_dict)
78
+
79
+ def convert_encoder_config(self, encoder_cfg: EncoderAnySplatCfg) -> EncoderAnySplatCfg:
80
+ """Convert all nested configurations in encoder_cfg"""
81
+ conversion_map = {
82
+ 'gaussian_adapter': GaussianAdapterCfg,
83
+ 'opacity_mapping': OpacityMappingCfg,
84
+ }
85
+
86
+ return self.convert_config_recursively(encoder_cfg, conversion_map)
87
+
88
+ def build_encoder(self, encoder_cfg: EncoderAnySplatCfg):
89
+ # Convert nested configurations using the helper method
90
+ encoder_cfg = self.convert_encoder_config(encoder_cfg)
91
+ self.encoder = EncoderAnySplat(encoder_cfg)
92
+
93
+ def build_decoder(self, decoder_cfg: DecoderSplattingCUDACfg):
94
+ self.decoder = DecoderSplattingCUDA(decoder_cfg)
95
+
96
+ @torch.no_grad()
97
+ def inference(self,
98
+ context_image: torch.Tensor,
99
+ ):
100
+ self.encoder.distill = False
101
+ encoder_output = self.encoder(context_image, global_step=0, visualization_dump=None)
102
+ gaussians, pred_context_pose = encoder_output.gaussians, encoder_output.pred_context_pose
103
+ return gaussians, pred_context_pose
104
+
105
+ def forward(self,
106
+ context_image: torch.Tensor,
107
+ global_step: int = 0,
108
+ visualization_dump: Optional[dict] = None,
109
+ near: float = 0.01,
110
+ far: float = 100.0,
111
+ ):
112
+ b, v, c, h, w = context_image.shape
113
+ device = context_image.device
114
+ encoder_output = self.encoder(context_image, global_step, visualization_dump=visualization_dump)
115
+ gaussians, pred_context_pose = encoder_output.gaussians, encoder_output.pred_context_pose
116
+ output = self.decoder.forward(
117
+ gaussians,
118
+ pred_context_pose['extrinsic'],
119
+ pred_context_pose["intrinsic"],
120
+ torch.ones(1, v, device=device) * near,
121
+ torch.ones(1, v, device=device) * far,
122
+ (h, w),
123
+ "depth",
124
+ )
125
+ return encoder_output, output
126
+
src/model/model_wrapper.py ADDED
@@ -0,0 +1,847 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+ from pathlib import Path
3
+ import gc
4
+ import random
5
+ from typing import Literal, Optional, Protocol, runtime_checkable, Any
6
+
7
+ import moviepy.editor as mpy
8
+ import torch
9
+ import torchvision
10
+ import wandb
11
+ from einops import pack, rearrange, repeat
12
+ from jaxtyping import Float
13
+ from lightning.pytorch import LightningModule
14
+ from lightning.pytorch.loggers.wandb import WandbLogger
15
+ from lightning.pytorch.utilities import rank_zero_only
16
+ from tabulate import tabulate
17
+ from torch import Tensor, nn, optim
18
+ import torch.nn.functional as F
19
+
20
+ from loss.loss_lpips import LossLpips
21
+ from loss.loss_mse import LossMse
22
+ from model.encoder.vggt.utils.pose_enc import pose_encoding_to_extri_intri
23
+
24
+ from ..loss.loss_distill import DistillLoss
25
+ from src.utils.render import generate_path
26
+ from src.utils.point import get_normal_map
27
+
28
+ from ..loss.loss_huber import HuberLoss, extri_intri_to_pose_encoding
29
+
30
+ # from model.types import Gaussians
31
+
32
+ from ..dataset.data_module import get_data_shim
33
+ from ..dataset.types import BatchedExample
34
+ from ..evaluation.metrics import compute_lpips, compute_psnr, compute_ssim, abs_relative_difference, delta1_acc
35
+ from ..global_cfg import get_cfg
36
+ from ..loss import Loss
37
+ from ..loss.loss_point import Regr3D
38
+ from ..loss.loss_ssim import ssim
39
+ from ..misc.benchmarker import Benchmarker
40
+ from ..misc.cam_utils import update_pose, get_pnp_pose, rotation_6d_to_matrix
41
+ from ..misc.image_io import prep_image, save_image, save_video
42
+ from ..misc.LocalLogger import LOG_PATH, LocalLogger
43
+ from ..misc.nn_module_tools import convert_to_buffer
44
+ from ..misc.step_tracker import StepTracker
45
+ from ..misc.utils import inverse_normalize, vis_depth_map, confidence_map, get_overlap_tag
46
+ from ..visualization.annotation import add_label
47
+ from ..visualization.camera_trajectory.interpolation import (
48
+ interpolate_extrinsics,
49
+ interpolate_intrinsics,
50
+ )
51
+ from ..visualization.camera_trajectory.wobble import (
52
+ generate_wobble,
53
+ generate_wobble_transformation,
54
+ )
55
+ from ..visualization.color_map import apply_color_map_to_image
56
+ from ..visualization.layout import add_border, hcat, vcat
57
+ # from ..visualization.validation_in_3d import render_cameras, render_projections
58
+ from .decoder.decoder import Decoder, DepthRenderingMode
59
+ from .encoder import Encoder
60
+ from .encoder.visualization.encoder_visualizer import EncoderVisualizer
61
+ from .ply_export import export_ply
62
+
63
+ @dataclass
64
+ class OptimizerCfg:
65
+ lr: float
66
+ warm_up_steps: int
67
+ backbone_lr_multiplier: float
68
+
69
+
70
+ @dataclass
71
+ class TestCfg:
72
+ output_path: Path
73
+ align_pose: bool
74
+ pose_align_steps: int
75
+ rot_opt_lr: float
76
+ trans_opt_lr: float
77
+ compute_scores: bool
78
+ save_image: bool
79
+ save_video: bool
80
+ save_compare: bool
81
+ generate_video: bool
82
+ mode: Literal["inference", "evaluation"]
83
+ image_folder: str
84
+
85
+
86
+ @dataclass
87
+ class TrainCfg:
88
+ output_path: Path
89
+ depth_mode: DepthRenderingMode | None
90
+ extended_visualization: bool
91
+ print_log_every_n_steps: int
92
+ distiller: str
93
+ distill_max_steps: int
94
+ pose_loss_alpha: float = 1.0
95
+ pose_loss_delta: float = 1.0
96
+ cxt_depth_weight: float = 0.01
97
+ weight_pose: float = 1.0
98
+ weight_depth: float = 1.0
99
+ weight_normal: float = 1.0
100
+ render_ba: bool = False
101
+ render_ba_after_step: int = 0
102
+
103
+
104
+ @runtime_checkable
105
+ class TrajectoryFn(Protocol):
106
+ def __call__(
107
+ self,
108
+ t: Float[Tensor, " t"],
109
+ ) -> tuple[
110
+ Float[Tensor, "batch view 4 4"], # extrinsics
111
+ Float[Tensor, "batch view 3 3"], # intrinsics
112
+ ]:
113
+ pass
114
+
115
+
116
+ class ModelWrapper(LightningModule):
117
+ logger: Optional[WandbLogger]
118
+ model: nn.Module
119
+ losses: nn.ModuleList
120
+ optimizer_cfg: OptimizerCfg
121
+ test_cfg: TestCfg
122
+ train_cfg: TrainCfg
123
+ step_tracker: StepTracker | None
124
+
125
+ def __init__(
126
+ self,
127
+ optimizer_cfg: OptimizerCfg,
128
+ test_cfg: TestCfg,
129
+ train_cfg: TrainCfg,
130
+ model: nn.Module,
131
+ losses: list[Loss],
132
+ step_tracker: StepTracker | None
133
+ ) -> None:
134
+ super().__init__()
135
+ self.optimizer_cfg = optimizer_cfg
136
+ self.test_cfg = test_cfg
137
+ self.train_cfg = train_cfg
138
+ self.step_tracker = step_tracker
139
+
140
+ # Set up the model.
141
+ self.encoder_visualizer = None
142
+ self.model = model
143
+ self.data_shim = get_data_shim(self.model.encoder)
144
+ self.losses = nn.ModuleList(losses)
145
+
146
+ if self.model.encoder.pred_pose:
147
+ self.loss_pose = HuberLoss(alpha=self.train_cfg.pose_loss_alpha, delta=self.train_cfg.pose_loss_delta)
148
+
149
+ if self.model.encoder.distill:
150
+ self.loss_distill = DistillLoss(
151
+ delta=self.train_cfg.pose_loss_delta,
152
+ weight_pose=self.train_cfg.weight_pose,
153
+ weight_depth=self.train_cfg.weight_depth,
154
+ weight_normal=self.train_cfg.weight_normal
155
+ )
156
+
157
+ # This is used for testing.
158
+ self.benchmarker = Benchmarker()
159
+
160
+ def on_train_epoch_start(self) -> None:
161
+ # our custom dataset and sampler has to have epoch set by calling set_epoch
162
+ if hasattr(self.trainer.datamodule.train_loader.dataset, "set_epoch"):
163
+ self.trainer.datamodule.train_loader.dataset.set_epoch(self.current_epoch)
164
+ if hasattr(self.trainer.datamodule.train_loader.sampler, "set_epoch"):
165
+ self.trainer.datamodule.train_loader.sampler.set_epoch(self.current_epoch)
166
+
167
+ def on_validation_epoch_start(self) -> None:
168
+ print(f"Validation epoch start on rank {self.trainer.global_rank}")
169
+ # our custom dataset and sampler has to have epoch set by calling set_epoch
170
+ if hasattr(self.trainer.datamodule.val_loader.dataset, "set_epoch"):
171
+ self.trainer.datamodule.val_loader.dataset.set_epoch(self.current_epoch)
172
+ if hasattr(self.trainer.datamodule.val_loader.sampler, "set_epoch"):
173
+ self.trainer.datamodule.val_loader.sampler.set_epoch(self.current_epoch)
174
+
175
+ def training_step(self, batch, batch_idx):
176
+ # combine batch from different dataloaders
177
+ # torch.cuda.empty_cache()
178
+ if isinstance(batch, list):
179
+ batch_combined = None
180
+ for batch_per_dl in batch:
181
+ if batch_combined is None:
182
+ batch_combined = batch_per_dl
183
+ else:
184
+ for k in batch_combined.keys():
185
+ if isinstance(batch_combined[k], list):
186
+ batch_combined[k] += batch_per_dl[k]
187
+ elif isinstance(batch_combined[k], dict):
188
+ for kk in batch_combined[k].keys():
189
+ batch_combined[k][kk] = torch.cat([batch_combined[k][kk], batch_per_dl[k][kk]], dim=0)
190
+ else:
191
+ raise NotImplementedError
192
+ batch = batch_combined
193
+
194
+ batch: BatchedExample = self.data_shim(batch)
195
+ b, v, c, h, w = batch["context"]["image"].shape
196
+ context_image = (batch["context"]["image"] + 1) / 2
197
+
198
+ # Run the model.
199
+ visualization_dump = None
200
+
201
+ encoder_output, output = self.model(context_image, self.global_step, visualization_dump=visualization_dump)
202
+ gaussians, pred_pose_enc_list, depth_dict = encoder_output.gaussians, encoder_output.pred_pose_enc_list, encoder_output.depth_dict
203
+ pred_context_pose = encoder_output.pred_context_pose
204
+ infos = encoder_output.infos
205
+ distill_infos = encoder_output.distill_infos
206
+
207
+ num_context_views = pred_context_pose['extrinsic'].shape[1]
208
+
209
+ using_index = torch.arange(num_context_views, device=gaussians.means.device)
210
+ batch["using_index"] = using_index
211
+
212
+ target_gt = (batch["context"]["image"] + 1) / 2
213
+ scene_scale = infos["scene_scale"]
214
+ self.log("train/scene_scale", infos["scene_scale"])
215
+ self.log("train/voxelize_ratio", infos["voxelize_ratio"])
216
+
217
+ # Compute metrics.
218
+ psnr_probabilistic = compute_psnr(
219
+ rearrange(target_gt, "b v c h w -> (b v) c h w"),
220
+ rearrange(output.color, "b v c h w -> (b v) c h w"),
221
+ )
222
+ self.log("train/psnr_probabilistic", psnr_probabilistic.mean())
223
+
224
+ consis_absrel = abs_relative_difference(
225
+ rearrange(output.depth, "b v h w -> (b v) h w"),
226
+ rearrange(depth_dict['depth'].squeeze(-1), "b v h w -> (b v) h w"),
227
+ rearrange(distill_infos['conf_mask'], "b v h w -> (b v) h w"),
228
+ )
229
+ self.log("train/consis_absrel", consis_absrel.mean())
230
+
231
+ consis_delta1 = delta1_acc(
232
+ rearrange(output.depth, "b v h w -> (b v) h w"),
233
+ rearrange(depth_dict['depth'].squeeze(-1), "b v h w -> (b v) h w"),
234
+ rearrange(distill_infos['conf_mask'], "b v h w -> (b v) h w"),
235
+ )
236
+ self.log("train/consis_delta1", consis_delta1.mean())
237
+
238
+ # Compute and log loss.
239
+ total_loss = 0
240
+
241
+ depth_dict['distill_infos'] = distill_infos
242
+ with torch.amp.autocast('cuda', enabled=False):
243
+ for loss_fn in self.losses:
244
+ loss = loss_fn.forward(output, batch, gaussians, depth_dict, self.global_step)
245
+ self.log(f"loss/{loss_fn.name}", loss)
246
+ total_loss = total_loss + loss
247
+
248
+ if depth_dict is not None and "depth" in get_cfg()["loss"].keys() and self.train_cfg.cxt_depth_weight > 0:
249
+ depth_loss_idx = list(get_cfg()["loss"].keys()).index("depth")
250
+ depth_loss_fn = self.losses[depth_loss_idx].ctx_depth_loss
251
+ loss_depth = depth_loss_fn(depth_dict["depth_map"], depth_dict["depth_conf"], batch, cxt_depth_weight=self.train_cfg.cxt_depth_weight)
252
+ self.log("loss/ctx_depth", loss_depth)
253
+ total_loss = total_loss + loss_depth
254
+
255
+ if distill_infos is not None:
256
+ # distill ctx pred_pose & depth & normal
257
+ loss_distill_list = self.loss_distill(distill_infos, pred_pose_enc_list, output, batch)
258
+ self.log("loss/distill", loss_distill_list['loss_distill'])
259
+ self.log("loss/distill_pose", loss_distill_list['loss_pose'])
260
+ self.log("loss/distill_depth", loss_distill_list['loss_depth'])
261
+ self.log("loss/distill_normal", loss_distill_list['loss_normal'])
262
+ total_loss = total_loss + loss_distill_list['loss_distill']
263
+
264
+ self.log("loss/total", total_loss)
265
+ print(f"total_loss: {total_loss}")
266
+
267
+ # Skip batch if loss is too high after certain step
268
+ SKIP_AFTER_STEP = 1000
269
+ LOSS_THRESHOLD = 0.2
270
+ if self.global_step > SKIP_AFTER_STEP and total_loss > LOSS_THRESHOLD:
271
+ print(f"Skipping batch with high loss ({total_loss:.6f}) at step {self.global_step} on Rank {self.global_rank}")
272
+ # set to a really small number
273
+ return total_loss * 1e-10
274
+
275
+ if (
276
+ self.global_rank == 0
277
+ and self.global_step % self.train_cfg.print_log_every_n_steps == 0
278
+ ):
279
+ print(
280
+ f"train step {self.global_step}; "
281
+ f"scene = {[x[:20] for x in batch['scene']]}; "
282
+ f"context = {batch['context']['index'].tolist()}; "
283
+ f"loss = {total_loss:.6f}; "
284
+ )
285
+
286
+ self.log("info/global_step", self.global_step) # hack for ckpt monitor
287
+
288
+ # Tell the data loader processes about the current step.
289
+ if self.step_tracker is not None:
290
+ self.step_tracker.set_step(self.global_step)
291
+
292
+ del batch
293
+ if self.global_step % 50 == 0:
294
+ gc.collect()
295
+ torch.cuda.empty_cache()
296
+
297
+ return total_loss
298
+
299
+ def on_after_backward(self):
300
+ total_norm = 0.0
301
+ counter = 0
302
+ for p in self.parameters():
303
+ if p.grad is not None:
304
+ param_norm = p.grad.detach().data.norm(2)
305
+ total_norm += param_norm.item() ** 2
306
+ counter += 1
307
+ total_norm = (total_norm / counter) ** 0.5
308
+ self.log("loss/grad_norm", total_norm)
309
+
310
+ def test_step(self, batch, batch_idx):
311
+ batch: BatchedExample = self.data_shim(batch)
312
+ b, v, _, h, w = batch["target"]["image"].shape
313
+ assert b == 1
314
+ if batch_idx % 100 == 0:
315
+ print(f"Test step {batch_idx:0>6}.")
316
+
317
+ # Render Gaussians.
318
+ with self.benchmarker.time("encoder"):
319
+ gaussians = self.model.encoder(
320
+ (batch["context"]["image"]+1)/2,
321
+ self.global_step,
322
+ )[0]
323
+ # export_ply(gaussians.means[0], gaussians.scales[0], gaussians.rotations[0], gaussians.harmonics[0], gaussians.opacities[0], Path("gaussians.ply"))
324
+ # align the target pose
325
+ if self.test_cfg.align_pose:
326
+ output = self.test_step_align(batch, gaussians)
327
+ else:
328
+ with self.benchmarker.time("decoder", num_calls=v):
329
+ output = self.model.decoder.forward(
330
+ gaussians,
331
+ batch["target"]["extrinsics"],
332
+ batch["target"]["intrinsics"],
333
+ batch["target"]["near"],
334
+ batch["target"]["far"],
335
+ (h, w),
336
+ )
337
+
338
+ # compute scores
339
+ if self.test_cfg.compute_scores:
340
+ overlap = batch["context"]["overlap"][0]
341
+ overlap_tag = get_overlap_tag(overlap)
342
+
343
+ rgb_pred = output.color[0]
344
+ rgb_gt = batch["target"]["image"][0]
345
+ all_metrics = {
346
+ f"lpips_ours": compute_lpips(rgb_gt, rgb_pred).mean(),
347
+ f"ssim_ours": compute_ssim(rgb_gt, rgb_pred).mean(),
348
+ f"psnr_ours": compute_psnr(rgb_gt, rgb_pred).mean(),
349
+ }
350
+ methods = ['ours']
351
+
352
+ self.log_dict(all_metrics)
353
+ self.print_preview_metrics(all_metrics, methods, overlap_tag=overlap_tag)
354
+
355
+ # Save images.
356
+ (scene,) = batch["scene"]
357
+ name = get_cfg()["wandb"]["name"]
358
+ path = self.test_cfg.output_path / name
359
+ if self.test_cfg.save_image:
360
+ for index, color in zip(batch["target"]["index"][0], output.color[0]):
361
+ save_image(color, path / scene / f"color/{index:0>6}.png")
362
+
363
+ if self.test_cfg.save_video:
364
+ frame_str = "_".join([str(x.item()) for x in batch["context"]["index"][0]])
365
+ save_video(
366
+ [a for a in output.color[0]],
367
+ path / "video" / f"{scene}_frame_{frame_str}.mp4",
368
+ )
369
+
370
+ if self.test_cfg.save_compare:
371
+ # Construct comparison image.
372
+ context_img = inverse_normalize(batch["context"]["image"][0])
373
+ comparison = hcat(
374
+ add_label(vcat(*context_img), "Context"),
375
+ add_label(vcat(*rgb_gt), "Target (Ground Truth)"),
376
+ add_label(vcat(*rgb_pred), "Target (Prediction)"),
377
+ )
378
+ save_image(comparison, path / f"{scene}.png")
379
+
380
+ def test_step_align(self, batch, gaussians):
381
+ self.model.encoder.eval()
382
+ # freeze all parameters
383
+ for param in self.model.encoder.parameters():
384
+ param.requires_grad = False
385
+
386
+ b, v, _, h, w = batch["target"]["image"].shape
387
+ output_c2ws = batch["target"]["extrinsics"]
388
+ with torch.set_grad_enabled(True):
389
+ cam_rot_delta = nn.Parameter(torch.zeros([b, v, 6], requires_grad=True, device=output_c2ws.device))
390
+ cam_trans_delta = nn.Parameter(torch.zeros([b, v, 3], requires_grad=True, device=output_c2ws.device))
391
+ opt_params = []
392
+ self.register_buffer("identity", torch.tensor([1.0, 0.0, 0.0, 0.0, 1.0, 0.0]).to(output_c2ws))
393
+ opt_params.append(
394
+ {
395
+ "params": [cam_rot_delta],
396
+ "lr": 0.005,
397
+ }
398
+ )
399
+ opt_params.append(
400
+ {
401
+ "params": [cam_trans_delta],
402
+ "lr": 0.005,
403
+ }
404
+ )
405
+ pose_optimizer = torch.optim.Adam(opt_params)
406
+ extrinsics = output_c2ws.clone()
407
+ with self.benchmarker.time("optimize"):
408
+ for i in range(self.test_cfg.pose_align_steps):
409
+ pose_optimizer.zero_grad()
410
+ dx, drot = cam_trans_delta, cam_rot_delta
411
+ rot = rotation_6d_to_matrix(
412
+ drot + self.identity.expand(b, v, -1)
413
+ ) # (..., 3, 3)
414
+
415
+ transform = torch.eye(4, device=extrinsics.device).repeat((b, v, 1, 1))
416
+ transform[..., :3, :3] = rot
417
+ transform[..., :3, 3] = dx
418
+
419
+ new_extrinsics = torch.matmul(extrinsics, transform)
420
+ output = self.model.decoder.forward(
421
+ gaussians,
422
+ new_extrinsics,
423
+ batch["target"]["intrinsics"],
424
+ batch["target"]["near"],
425
+ batch["target"]["far"],
426
+ (h, w),
427
+ # cam_rot_delta=cam_rot_delta,
428
+ # cam_trans_delta=cam_trans_delta,
429
+ )
430
+
431
+ # Compute and log loss.
432
+ total_loss = 0
433
+ for loss_fn in self.losses:
434
+ loss = loss_fn.forward(output, batch, gaussians, self.global_step)
435
+ total_loss = total_loss + loss
436
+
437
+ total_loss.backward()
438
+ pose_optimizer.step()
439
+
440
+ # Render Gaussians.
441
+ output = self.model.decoder.forward(
442
+ gaussians,
443
+ new_extrinsics,
444
+ batch["target"]["intrinsics"],
445
+ batch["target"]["near"],
446
+ batch["target"]["far"],
447
+ (h, w),
448
+ )
449
+
450
+ return output
451
+
452
+ def on_test_end(self) -> None:
453
+ name = get_cfg()["wandb"]["name"]
454
+ self.benchmarker.dump(self.test_cfg.output_path / name / "benchmark.json")
455
+ self.benchmarker.dump_memory(
456
+ self.test_cfg.output_path / name / "peak_memory.json"
457
+ )
458
+ self.benchmarker.summarize()
459
+
460
+ @rank_zero_only
461
+ def validation_step(self, batch, batch_idx, dataloader_idx=0):
462
+ batch: BatchedExample = self.data_shim(batch)
463
+
464
+ if self.global_rank == 0:
465
+ print(
466
+ f"validation step {self.global_step}; "
467
+ f"scene = {batch['scene']}; "
468
+ f"context = {batch['context']['index'].tolist()}"
469
+ )
470
+
471
+ # Render Gaussians.
472
+ b, v, _, h, w = batch["context"]["image"].shape
473
+ assert b == 1
474
+ visualization_dump = {}
475
+
476
+ encoder_output, output = self.model(batch["context"]["image"], self.global_step, visualization_dump=visualization_dump)
477
+ gaussians, pred_pose_enc_list, depth_dict = encoder_output.gaussians, encoder_output.pred_pose_enc_list, encoder_output.depth_dict
478
+ pred_context_pose, distill_infos = encoder_output.pred_context_pose, encoder_output.distill_infos
479
+ infos = encoder_output.infos
480
+
481
+ GS_num = infos['voxelize_ratio'] * (h*w*v)
482
+ self.log("val/GS_num", GS_num)
483
+
484
+ num_context_views = pred_context_pose['extrinsic'].shape[1]
485
+ num_target_views = batch["target"]["extrinsics"].shape[1]
486
+ rgb_pred = output.color[0].float()
487
+ depth_pred = vis_depth_map(output.depth[0])
488
+
489
+ # direct depth from gaussian means (used for visualization only)
490
+ gaussian_means = visualization_dump["depth"][0].squeeze()
491
+ if gaussian_means.shape[-1] == 3:
492
+ gaussian_means = gaussian_means.mean(dim=-1)
493
+
494
+ # Compute validation metrics.
495
+ rgb_gt = (batch["context"]["image"][0].float() + 1) / 2
496
+ psnr = compute_psnr(rgb_gt, rgb_pred).mean()
497
+ self.log(f"val/psnr", psnr)
498
+ lpips = compute_lpips(rgb_gt, rgb_pred).mean()
499
+ self.log(f"val/lpips", lpips)
500
+ ssim = compute_ssim(rgb_gt, rgb_pred).mean()
501
+ self.log(f"val/ssim", ssim)
502
+
503
+ # depth metrics
504
+ consis_absrel = abs_relative_difference(
505
+ rearrange(output.depth, "b v h w -> (b v) h w"),
506
+ rearrange(depth_dict['depth'].squeeze(-1), "b v h w -> (b v) h w"),
507
+ )
508
+ self.log("val/consis_absrel", consis_absrel.mean())
509
+
510
+ consis_delta1 = delta1_acc(
511
+ rearrange(output.depth, "b v h w -> (b v) h w"),
512
+ rearrange(depth_dict['depth'].squeeze(-1), "b v h w -> (b v) h w"),
513
+ valid_mask=rearrange(torch.ones_like(output.depth, device=output.depth.device, dtype=torch.bool), "b v h w -> (b v) h w"),
514
+ )
515
+ self.log("val/consis_delta1", consis_delta1.mean())
516
+
517
+ diff_map = torch.abs(output.depth - depth_dict['depth'].squeeze(-1))
518
+ self.log("val/consis_mse", diff_map[distill_infos['conf_mask']].mean())
519
+
520
+ # Construct comparison image.
521
+ context_img = inverse_normalize(batch["context"]["image"][0])
522
+ # context_img_depth = vis_depth_map(gaussian_means)
523
+ context = []
524
+ for i in range(context_img.shape[0]):
525
+ context.append(context_img[i])
526
+ # context.append(context_img_depth[i])
527
+
528
+ colored_diff_map = vis_depth_map(diff_map[0], near=torch.tensor(1e-4, device=diff_map.device), far=torch.tensor(1.0, device=diff_map.device))
529
+ model_depth_pred = depth_dict["depth"].squeeze(-1)[0]
530
+ model_depth_pred = vis_depth_map(model_depth_pred)
531
+
532
+ render_normal = (get_normal_map(output.depth.flatten(0, 1), batch["context"]["intrinsics"].flatten(0, 1)).permute(0, 3, 1, 2) + 1) / 2.
533
+ pred_normal = (get_normal_map(depth_dict['depth'].flatten(0, 1).squeeze(-1), batch["context"]["intrinsics"].flatten(0, 1)).permute(0, 3, 1, 2) + 1) / 2.
534
+
535
+ comparison = hcat(
536
+ add_label(vcat(*context), "Context"),
537
+ add_label(vcat(*rgb_gt), "Target (Ground Truth)"),
538
+ add_label(vcat(*rgb_pred), "Target (Prediction)"),
539
+ add_label(vcat(*depth_pred), "Depth (Prediction)"),
540
+ add_label(vcat(*model_depth_pred), "Depth (VGGT Prediction)"),
541
+ add_label(vcat(*render_normal), "Normal (Prediction)"),
542
+ add_label(vcat(*pred_normal), "Normal (VGGT Prediction)"),
543
+ add_label(vcat(*colored_diff_map), "Diff Map"),
544
+ )
545
+
546
+ comparison = torch.nn.functional.interpolate(
547
+ comparison.unsqueeze(0),
548
+ scale_factor=0.5,
549
+ mode='bicubic',
550
+ align_corners=False
551
+ ).squeeze(0)
552
+
553
+ self.logger.log_image(
554
+ "comparison",
555
+ [prep_image(add_border(comparison))],
556
+ step=self.global_step,
557
+ caption=batch["scene"],
558
+ )
559
+
560
+ # self.logger.log_image(
561
+ # key="comparison",
562
+ # images=[wandb.Image(prep_image(add_border(comparison)), caption=batch["scene"], file_type="jpg")],
563
+ # step=self.global_step
564
+ # )
565
+
566
+ # Render projections and construct projection image.
567
+ # These are disabled for now, since RE10k scenes are effectively unbounded.
568
+
569
+ # if isinstance(gaussians, Gaussians):
570
+ # projections = hcat(
571
+ # *render_projections(
572
+ # gaussians,
573
+ # 256,
574
+ # extra_label="",
575
+ # )[0]
576
+ # )
577
+ # self.logger.log_image(
578
+ # "projection",
579
+ # [prep_image(add_border(projections))],
580
+ # step=self.global_step,
581
+ # )
582
+
583
+ # Draw cameras.
584
+ # cameras = hcat(*render_cameras(batch, 256))
585
+ # self.logger.log_image(
586
+ # "cameras", [prep_image(add_border(cameras))], step=self.global_step
587
+ # )
588
+
589
+ if self.encoder_visualizer is not None:
590
+ for k, image in self.encoder_visualizer.visualize(
591
+ batch["context"], self.global_step
592
+ ).items():
593
+ self.logger.log_image(k, [prep_image(image)], step=self.global_step)
594
+
595
+ # Run video validation step.
596
+ self.render_video_interpolation(batch)
597
+ self.render_video_wobble(batch)
598
+ if self.train_cfg.extended_visualization:
599
+ self.render_video_interpolation_exaggerated(batch)
600
+
601
+ @rank_zero_only
602
+ def render_video_wobble(self, batch: BatchedExample) -> None:
603
+ # Two views are needed to get the wobble radius.
604
+ _, v, _, _ = batch["context"]["extrinsics"].shape
605
+ if v != 2:
606
+ return
607
+
608
+ def trajectory_fn(t):
609
+ origin_a = batch["context"]["extrinsics"][:, 0, :3, 3]
610
+ origin_b = batch["context"]["extrinsics"][:, 1, :3, 3]
611
+ delta = (origin_a - origin_b).norm(dim=-1)
612
+ extrinsics = generate_wobble(
613
+ batch["context"]["extrinsics"][:, 0],
614
+ delta * 0.25,
615
+ t,
616
+ )
617
+ intrinsics = repeat(
618
+ batch["context"]["intrinsics"][:, 0],
619
+ "b i j -> b v i j",
620
+ v=t.shape[0],
621
+ )
622
+ return extrinsics, intrinsics
623
+
624
+ return self.render_video_generic(batch, trajectory_fn, "wobble", num_frames=60)
625
+
626
+ @rank_zero_only
627
+ def render_video_interpolation(self, batch: BatchedExample) -> None:
628
+ _, v, _, _ = batch["context"]["extrinsics"].shape
629
+
630
+ def trajectory_fn(t):
631
+ extrinsics = interpolate_extrinsics(
632
+ batch["context"]["extrinsics"][0, 0],
633
+ (
634
+ batch["context"]["extrinsics"][0, 1]
635
+ if v == 2
636
+ else batch["target"]["extrinsics"][0, 0]
637
+ ),
638
+ t,
639
+ )
640
+ intrinsics = interpolate_intrinsics(
641
+ batch["context"]["intrinsics"][0, 0],
642
+ (
643
+ batch["context"]["intrinsics"][0, 1]
644
+ if v == 2
645
+ else batch["target"]["intrinsics"][0, 0]
646
+ ),
647
+ t,
648
+ )
649
+ return extrinsics[None], intrinsics[None]
650
+
651
+ return self.render_video_generic(batch, trajectory_fn, "rgb")
652
+
653
+ @rank_zero_only
654
+ def render_video_interpolation_exaggerated(self, batch: BatchedExample) -> None:
655
+ # Two views are needed to get the wobble radius.
656
+ _, v, _, _ = batch["context"]["extrinsics"].shape
657
+ if v != 2:
658
+ return
659
+
660
+ def trajectory_fn(t):
661
+ origin_a = batch["context"]["extrinsics"][:, 0, :3, 3]
662
+ origin_b = batch["context"]["extrinsics"][:, 1, :3, 3]
663
+ delta = (origin_a - origin_b).norm(dim=-1)
664
+ tf = generate_wobble_transformation(
665
+ delta * 0.5,
666
+ t,
667
+ 5,
668
+ scale_radius_with_t=False,
669
+ )
670
+ extrinsics = interpolate_extrinsics(
671
+ batch["context"]["extrinsics"][0, 0],
672
+ (
673
+ batch["context"]["extrinsics"][0, 1]
674
+ if v == 2
675
+ else batch["target"]["extrinsics"][0, 0]
676
+ ),
677
+ t * 5 - 2,
678
+ )
679
+ intrinsics = interpolate_intrinsics(
680
+ batch["context"]["intrinsics"][0, 0],
681
+ (
682
+ batch["context"]["intrinsics"][0, 1]
683
+ if v == 2
684
+ else batch["target"]["intrinsics"][0, 0]
685
+ ),
686
+ t * 5 - 2,
687
+ )
688
+ return extrinsics @ tf, intrinsics[None]
689
+
690
+ return self.render_video_generic(
691
+ batch,
692
+ trajectory_fn,
693
+ "interpolation_exagerrated",
694
+ num_frames=300,
695
+ smooth=False,
696
+ loop_reverse=False,
697
+ )
698
+
699
+ @rank_zero_only
700
+ def render_video_generic(
701
+ self,
702
+ batch: BatchedExample,
703
+ trajectory_fn: TrajectoryFn,
704
+ name: str,
705
+ num_frames: int = 30,
706
+ smooth: bool = True,
707
+ loop_reverse: bool = True,
708
+ ) -> None:
709
+ # Render probabilistic estimate of scene.
710
+ encoder_output = self.model.encoder((batch["context"]["image"]+1)/2, self.global_step)
711
+ gaussians, pred_pose_enc_list = encoder_output.gaussians, encoder_output.pred_pose_enc_list
712
+
713
+ t = torch.linspace(0, 1, num_frames, dtype=torch.float32, device=self.device)
714
+ if smooth:
715
+ t = (torch.cos(torch.pi * (t + 1)) + 1) / 2
716
+
717
+ extrinsics, intrinsics = trajectory_fn(t)
718
+
719
+ _, _, _, h, w = batch["context"]["image"].shape
720
+
721
+ # TODO: Interpolate near and far planes?
722
+ near = repeat(batch["context"]["near"][:, 0], "b -> b v", v=num_frames)
723
+ far = repeat(batch["context"]["far"][:, 0], "b -> b v", v=num_frames)
724
+ output = self.model.decoder.forward(
725
+ gaussians, extrinsics, intrinsics, near, far, (h, w), "depth"
726
+ )
727
+ images = [
728
+ vcat(rgb, depth)
729
+ for rgb, depth in zip(output.color[0], vis_depth_map(output.depth[0]))
730
+ ]
731
+
732
+ video = torch.stack(images)
733
+ video = (video.clip(min=0, max=1) * 255).type(torch.uint8).cpu().numpy()
734
+ if loop_reverse:
735
+ video = pack([video, video[::-1][1:-1]], "* c h w")[0]
736
+ visualizations = {
737
+ f"video/{name}": wandb.Video(video[None], fps=30, format="mp4")
738
+ }
739
+
740
+ # Since the PyTorch Lightning doesn't support video logging, log to wandb directly.
741
+ try:
742
+ wandb.log(visualizations)
743
+ except Exception:
744
+ assert isinstance(self.logger, LocalLogger)
745
+ for key, value in visualizations.items():
746
+ tensor = value._prepare_video(value.data)
747
+ clip = mpy.ImageSequenceClip(list(tensor), fps=30)
748
+ dir = LOG_PATH / key
749
+ dir.mkdir(exist_ok=True, parents=True)
750
+ clip.write_videofile(
751
+ str(dir / f"{self.global_step:0>6}.mp4"), logger=None
752
+ )
753
+
754
+ def print_preview_metrics(self, metrics: dict[str, float | Tensor], methods: list[str] | None = None, overlap_tag: str | None = None) -> None:
755
+ if getattr(self, "running_metrics", None) is None:
756
+ self.running_metrics = metrics
757
+ self.running_metric_steps = 1
758
+ else:
759
+ s = self.running_metric_steps
760
+ self.running_metrics = {
761
+ k: ((s * v) + metrics[k]) / (s + 1)
762
+ for k, v in self.running_metrics.items()
763
+ }
764
+ self.running_metric_steps += 1
765
+
766
+ if overlap_tag is not None:
767
+ if getattr(self, "running_metrics_sub", None) is None:
768
+ self.running_metrics_sub = {overlap_tag: metrics}
769
+ self.running_metric_steps_sub = {overlap_tag: 1}
770
+ elif overlap_tag not in self.running_metrics_sub:
771
+ self.running_metrics_sub[overlap_tag] = metrics
772
+ self.running_metric_steps_sub[overlap_tag] = 1
773
+ else:
774
+ s = self.running_metric_steps_sub[overlap_tag]
775
+ self.running_metrics_sub[overlap_tag] = {k: ((s * v) + metrics[k]) / (s + 1)
776
+ for k, v in self.running_metrics_sub[overlap_tag].items()}
777
+ self.running_metric_steps_sub[overlap_tag] += 1
778
+
779
+ metric_list = ["psnr", "lpips", "ssim"]
780
+
781
+ def print_metrics(runing_metric, methods=None):
782
+ table = []
783
+ if methods is None:
784
+ methods = ['ours']
785
+
786
+ for method in methods:
787
+ row = [
788
+ f"{runing_metric[f'{metric}_{method}']:.3f}"
789
+ for metric in metric_list
790
+ ]
791
+ table.append((method, *row))
792
+
793
+ headers = ["Method"] + metric_list
794
+ table = tabulate(table, headers)
795
+ print(table)
796
+
797
+ print("All Pairs:")
798
+ print_metrics(self.running_metrics, methods)
799
+ if overlap_tag is not None:
800
+ for k, v in self.running_metrics_sub.items():
801
+ print(f"Overlap: {k}")
802
+ print_metrics(v, methods)
803
+
804
+ def configure_optimizers(self):
805
+ new_params, new_param_names = [], []
806
+ pretrained_params, pretrained_param_names = [], []
807
+ for name, param in self.named_parameters():
808
+ if not param.requires_grad:
809
+ continue
810
+
811
+ if "gaussian_param_head" in name or "interm" in name:
812
+ new_params.append(param)
813
+ new_param_names.append(name)
814
+ else:
815
+ pretrained_params.append(param)
816
+ pretrained_param_names.append(name)
817
+
818
+ param_dicts = [
819
+ {
820
+ "params": new_params,
821
+ "lr": self.optimizer_cfg.lr,
822
+ },
823
+ {
824
+ "params": pretrained_params,
825
+ "lr": self.optimizer_cfg.lr * self.optimizer_cfg.backbone_lr_multiplier,
826
+ },
827
+ ]
828
+ optimizer = torch.optim.AdamW(param_dicts, lr=self.optimizer_cfg.lr, weight_decay=0.05, betas=(0.9, 0.95))
829
+ warm_up_steps = self.optimizer_cfg.warm_up_steps
830
+ warm_up = torch.optim.lr_scheduler.LinearLR(
831
+ optimizer,
832
+ 1 / warm_up_steps,
833
+ 1,
834
+ total_iters=warm_up_steps,
835
+ )
836
+
837
+ lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=get_cfg()["trainer"]["max_steps"], eta_min=self.optimizer_cfg.lr * 0.1)
838
+ lr_scheduler = torch.optim.lr_scheduler.SequentialLR(optimizer, schedulers=[warm_up, lr_scheduler], milestones=[warm_up_steps])
839
+
840
+ return {
841
+ "optimizer": optimizer,
842
+ "lr_scheduler": {
843
+ "scheduler": lr_scheduler,
844
+ "interval": "step",
845
+ "frequency": 1,
846
+ },
847
+ }
src/model/ply_export.py ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pathlib import Path
2
+
3
+ import numpy as np
4
+ import torch
5
+ from einops import einsum, rearrange
6
+ from jaxtyping import Float
7
+ from plyfile import PlyData, PlyElement
8
+ from scipy.spatial.transform import Rotation as R
9
+ from torch import Tensor
10
+
11
+
12
+ def construct_list_of_attributes(num_rest: int) -> list[str]:
13
+ attributes = ["x", "y", "z", "nx", "ny", "nz"]
14
+ for i in range(3):
15
+ attributes.append(f"f_dc_{i}")
16
+ for i in range(num_rest):
17
+ attributes.append(f"f_rest_{i}")
18
+ attributes.append("opacity")
19
+ for i in range(3):
20
+ attributes.append(f"scale_{i}")
21
+ for i in range(4):
22
+ attributes.append(f"rot_{i}")
23
+ return attributes
24
+
25
+
26
+ def export_ply(
27
+ means: Float[Tensor, "gaussian 3"],
28
+ scales: Float[Tensor, "gaussian 3"],
29
+ rotations: Float[Tensor, "gaussian 4"],
30
+ harmonics: Float[Tensor, "gaussian 3 d_sh"],
31
+ opacities: Float[Tensor, " gaussian"],
32
+ path: Path,
33
+ shift_and_scale: bool = False,
34
+ save_sh_dc_only: bool = True,
35
+ ):
36
+ if shift_and_scale:
37
+ # Shift the scene so that the median Gaussian is at the origin.
38
+ means = means - means.median(dim=0).values
39
+
40
+ # Rescale the scene so that most Gaussians are within range [-1, 1].
41
+ scale_factor = means.abs().quantile(0.95, dim=0).max()
42
+ means = means / scale_factor
43
+ scales = scales / scale_factor
44
+
45
+ # Apply the rotation to the Gaussian rotations.
46
+ rotations = R.from_quat(rotations.detach().cpu().numpy()).as_matrix()
47
+ rotations = R.from_matrix(rotations).as_quat()
48
+ x, y, z, w = rearrange(rotations, "g xyzw -> xyzw g")
49
+ rotations = np.stack((w, x, y, z), axis=-1)
50
+
51
+ # Since current model use SH_degree = 4,
52
+ # which require large memory to store, we can only save the DC band to save memory.
53
+ f_dc = harmonics[..., 0]
54
+ f_rest = harmonics[..., 1:].flatten(start_dim=1)
55
+
56
+ dtype_full = [(attribute, "f4") for attribute in construct_list_of_attributes(0 if save_sh_dc_only else f_rest.shape[1])]
57
+ elements = np.empty(means.shape[0], dtype=dtype_full)
58
+ attributes = [
59
+ means.detach().cpu().numpy(),
60
+ torch.zeros_like(means).detach().cpu().numpy(),
61
+ f_dc.detach().cpu().contiguous().numpy(),
62
+ f_rest.detach().cpu().contiguous().numpy(),
63
+ opacities[..., None].detach().cpu().numpy(),
64
+ scales.log().detach().cpu().numpy(),
65
+ rotations,
66
+ ]
67
+ if save_sh_dc_only:
68
+ # remove f_rest from attributes
69
+ attributes.pop(3)
70
+
71
+ attributes = np.concatenate(attributes, axis=1)
72
+ elements[:] = list(map(tuple, attributes))
73
+ path.parent.mkdir(exist_ok=True, parents=True)
74
+ PlyData([PlyElement.describe(elements, "vertex")]).write(path)
src/model/transformer/attention.py ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # MIT License
2
+
3
+ # Copyright (c) 2022 Karl Stelzner
4
+
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+
12
+ # The above copyright notice and this permission notice shall be included in all
13
+ # copies or substantial portions of the Software.
14
+
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ # SOFTWARE.
22
+
23
+ # This file comes from https://github.com/stelzner/srt
24
+
25
+ import torch
26
+ from einops import rearrange
27
+ from torch import nn
28
+
29
+
30
+ class Attention(nn.Module):
31
+ def __init__(
32
+ self, dim, heads=8, dim_head=64, dropout=0.0, selfatt=True, kv_dim=None
33
+ ):
34
+ super().__init__()
35
+ inner_dim = dim_head * heads
36
+ project_out = not (heads == 1 and dim_head == dim)
37
+
38
+ self.heads = heads
39
+ self.scale = dim_head**-0.5
40
+
41
+ self.attend = nn.Softmax(dim=-1)
42
+ if selfatt:
43
+ self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False)
44
+ else:
45
+ self.to_q = nn.Linear(dim, inner_dim, bias=False)
46
+ self.to_kv = nn.Linear(kv_dim, inner_dim * 2, bias=False)
47
+
48
+ self.to_out = (
49
+ nn.Sequential(nn.Linear(inner_dim, dim), nn.Dropout(dropout))
50
+ if project_out
51
+ else nn.Identity()
52
+ )
53
+
54
+ def forward(self, x, z=None):
55
+ if z is None:
56
+ qkv = self.to_qkv(x).chunk(3, dim=-1)
57
+ else:
58
+ q = self.to_q(x)
59
+ k, v = self.to_kv(z).chunk(2, dim=-1)
60
+ qkv = (q, k, v)
61
+
62
+ q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b h n d", h=self.heads), qkv)
63
+
64
+ dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale
65
+
66
+ attn = self.attend(dots)
67
+
68
+ out = torch.matmul(attn, v)
69
+ out = rearrange(out, "b h n d -> b n (h d)")
70
+ return self.to_out(out)
src/model/transformer/feed_forward.py ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # MIT License
2
+
3
+ # Copyright (c) 2022 Karl Stelzner
4
+
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+
12
+ # The above copyright notice and this permission notice shall be included in all
13
+ # copies or substantial portions of the Software.
14
+
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ # SOFTWARE.
22
+
23
+ # This file comes from https://github.com/stelzner/srt
24
+
25
+ from torch import nn
26
+
27
+
28
+ class FeedForward(nn.Module):
29
+ def __init__(self, dim, hidden_dim, dropout=0.0):
30
+ super().__init__()
31
+ self.net = nn.Sequential(
32
+ nn.Linear(dim, hidden_dim),
33
+ nn.GELU(),
34
+ nn.Dropout(dropout),
35
+ nn.Linear(hidden_dim, dim),
36
+ nn.Dropout(dropout),
37
+ )
38
+
39
+ def forward(self, x):
40
+ return self.net(x)
src/model/transformer/pre_norm.py ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # MIT License
2
+
3
+ # Copyright (c) 2022 Karl Stelzner
4
+
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+
12
+ # The above copyright notice and this permission notice shall be included in all
13
+ # copies or substantial portions of the Software.
14
+
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ # SOFTWARE.
22
+
23
+ # This file comes from https://github.com/stelzner/srt
24
+
25
+ from torch import nn
26
+
27
+
28
+ class PreNorm(nn.Module):
29
+ def __init__(self, dim, fn):
30
+ super().__init__()
31
+ self.norm = nn.LayerNorm(dim)
32
+ self.fn = fn
33
+
34
+ def forward(self, x, **kwargs):
35
+ return self.fn(self.norm(x), **kwargs)
src/model/transformer/transformer.py ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # MIT License
2
+
3
+ # Copyright (c) 2022 Karl Stelzner
4
+
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+
12
+ # The above copyright notice and this permission notice shall be included in all
13
+ # copies or substantial portions of the Software.
14
+
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ # SOFTWARE.
22
+
23
+ # This file comes from https://github.com/stelzner/srt
24
+
25
+ from torch import nn
26
+
27
+ from .attention import Attention
28
+ from .feed_forward import FeedForward
29
+ from .pre_norm import PreNorm
30
+
31
+
32
+ class Transformer(nn.Module):
33
+ def __init__(
34
+ self,
35
+ dim,
36
+ depth,
37
+ heads,
38
+ dim_head,
39
+ mlp_dim,
40
+ dropout=0.0,
41
+ selfatt=True,
42
+ kv_dim=None,
43
+ feed_forward_layer=FeedForward,
44
+ ):
45
+ super().__init__()
46
+ self.layers = nn.ModuleList([])
47
+ for _ in range(depth):
48
+ self.layers.append(
49
+ nn.ModuleList(
50
+ [
51
+ PreNorm(
52
+ dim,
53
+ Attention(
54
+ dim,
55
+ heads=heads,
56
+ dim_head=dim_head,
57
+ dropout=dropout,
58
+ selfatt=selfatt,
59
+ kv_dim=kv_dim,
60
+ ),
61
+ ),
62
+ PreNorm(dim, feed_forward_layer(dim, mlp_dim, dropout=dropout)),
63
+ ]
64
+ )
65
+ )
66
+
67
+ def forward(self, x, z=None, **kwargs):
68
+ for attn, ff in self.layers:
69
+ x = attn(x, z=z) + x
70
+ x = ff(x, **kwargs) + x
71
+ return x
src/model/types.py ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+
3
+ from jaxtyping import Float
4
+ from torch import Tensor
5
+
6
+
7
+ @dataclass
8
+ class Gaussians:
9
+ means: Float[Tensor, "batch gaussian dim"]
10
+ covariances: Float[Tensor, "batch gaussian dim dim"]
11
+ harmonics: Float[Tensor, "batch gaussian 3 d_sh"]
12
+ opacities: Float[Tensor, "batch gaussian"]
13
+ scales: Float[Tensor, "batch gaussian 3"]
14
+ rotations: Float[Tensor, "batch gaussian 4"]
15
+ # levels: Float[Tensor, "batch gaussian"]
src/post_opt/datasets/colmap.py ADDED
@@ -0,0 +1,490 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+ from typing import Any, Dict, List, Optional
4
+
5
+ import cv2
6
+ import imageio.v2 as imageio
7
+ import numpy as np
8
+ import torch
9
+ from PIL import Image
10
+ from pycolmap import SceneManager
11
+ from tqdm import tqdm
12
+ from typing_extensions import assert_never
13
+
14
+ import sys
15
+ sys.path.append("/cpfs01/user/jianglihan/projects/gsplat/examples/datasets")
16
+ sys.path.append("/cpfs01/user/jianglihan/projects/gsplat/examples")
17
+ sys.path.append("/cpfs01/user/jianglihan/projects/gsplat")
18
+
19
+ from normalize import (
20
+ align_principal_axes,
21
+ similarity_from_cameras,
22
+ transform_cameras,
23
+ transform_points,
24
+ )
25
+
26
+
27
+ def _get_rel_paths(path_dir: str) -> List[str]:
28
+ """Recursively get relative paths of files in a directory."""
29
+ paths = []
30
+ for dp, dn, fn in os.walk(path_dir):
31
+ for f in fn:
32
+ paths.append(os.path.relpath(os.path.join(dp, f), path_dir))
33
+ return paths
34
+
35
+
36
+ def _resize_image_folder(image_dir: str, resized_dir: str, factor: int) -> str:
37
+ """Resize image folder."""
38
+ print(f"Downscaling images by {factor}x from {image_dir} to {resized_dir}.")
39
+ os.makedirs(resized_dir, exist_ok=True)
40
+
41
+ image_files = _get_rel_paths(image_dir)
42
+ for image_file in tqdm(image_files):
43
+ image_path = os.path.join(image_dir, image_file)
44
+ resized_path = os.path.join(
45
+ resized_dir, os.path.splitext(image_file)[0] + ".png"
46
+ )
47
+ if os.path.isfile(resized_path):
48
+ continue
49
+ image = imageio.imread(image_path)[..., :3]
50
+ resized_size = (
51
+ int(round(image.shape[1] / factor)),
52
+ int(round(image.shape[0] / factor)),
53
+ )
54
+ resized_image = np.array(
55
+ Image.fromarray(image).resize(resized_size, Image.BICUBIC)
56
+ )
57
+ imageio.imwrite(resized_path, resized_image)
58
+ return resized_dir
59
+
60
+
61
+ class Parser:
62
+ """COLMAP parser."""
63
+
64
+ def __init__(
65
+ self,
66
+ data_dir: str,
67
+ factor: int = 1,
68
+ normalize: bool = False,
69
+ test_every: int = 8,
70
+ ):
71
+ self.data_dir = data_dir
72
+ self.factor = factor
73
+ self.normalize = normalize
74
+ self.test_every = test_every
75
+
76
+
77
+
78
+ colmap_dir = os.path.join(data_dir, "sparse/0/")
79
+ if not os.path.exists(colmap_dir):
80
+ colmap_dir = os.path.join(data_dir, "sparse")
81
+ assert os.path.exists(
82
+ colmap_dir
83
+ ), f"COLMAP directory {colmap_dir} does not exist."
84
+
85
+ manager = SceneManager(colmap_dir)
86
+ manager.load_cameras()
87
+ manager.load_images()
88
+ manager.load_points3D()
89
+
90
+ # Extract extrinsic matrices in world-to-camera format.
91
+ imdata = manager.images
92
+ w2c_mats = []
93
+ camera_ids = []
94
+ Ks_dict = dict()
95
+ params_dict = dict()
96
+ imsize_dict = dict() # width, height
97
+ mask_dict = dict()
98
+ bottom = np.array([0, 0, 0, 1]).reshape(1, 4)
99
+ for k in imdata:
100
+ im = imdata[k]
101
+ rot = im.R()
102
+ trans = im.tvec.reshape(3, 1)
103
+ w2c = np.concatenate([np.concatenate([rot, trans], 1), bottom], axis=0)
104
+ w2c_mats.append(w2c)
105
+
106
+ # support different camera intrinsics
107
+ camera_id = im.camera_id
108
+ camera_ids.append(camera_id)
109
+
110
+ # camera intrinsics
111
+ cam = manager.cameras[camera_id]
112
+ fx, fy, cx, cy = cam.fx, cam.fy, cam.cx, cam.cy
113
+ K = np.array([[fx, 0, cx], [0, fy, cy], [0, 0, 1]])
114
+ K[:2, :] /= factor
115
+ Ks_dict[camera_id] = K
116
+
117
+ # Get distortion parameters.
118
+ type_ = cam.camera_type
119
+ if type_ == 0 or type_ == "SIMPLE_PINHOLE":
120
+ params = np.empty(0, dtype=np.float32)
121
+ camtype = "perspective"
122
+ elif type_ == 1 or type_ == "PINHOLE":
123
+ params = np.empty(0, dtype=np.float32)
124
+ camtype = "perspective"
125
+ if type_ == 2 or type_ == "SIMPLE_RADIAL":
126
+ params = np.array([cam.k1, 0.0, 0.0, 0.0], dtype=np.float32)
127
+ camtype = "perspective"
128
+ elif type_ == 3 or type_ == "RADIAL":
129
+ params = np.array([cam.k1, cam.k2, 0.0, 0.0], dtype=np.float32)
130
+ camtype = "perspective"
131
+ elif type_ == 4 or type_ == "OPENCV":
132
+ params = np.array([cam.k1, cam.k2, cam.p1, cam.p2], dtype=np.float32)
133
+ camtype = "perspective"
134
+ elif type_ == 5 or type_ == "OPENCV_FISHEYE":
135
+ params = np.array([cam.k1, cam.k2, cam.k3, cam.k4], dtype=np.float32)
136
+ camtype = "fisheye"
137
+ assert (
138
+ camtype == "perspective" or camtype == "fisheye"
139
+ ), f"Only perspective and fisheye cameras are supported, got {type_}"
140
+
141
+ params_dict[camera_id] = params
142
+ imsize_dict[camera_id] = (cam.width // factor, cam.height // factor)
143
+ mask_dict[camera_id] = None
144
+ print(
145
+ f"[Parser] {len(imdata)} images, taken by {len(set(camera_ids))} cameras."
146
+ )
147
+
148
+ if len(imdata) == 0:
149
+ raise ValueError("No images found in COLMAP.")
150
+ if not (type_ == 0 or type_ == 1):
151
+ print("Warning: COLMAP Camera is not PINHOLE. Images have distortion.")
152
+
153
+ w2c_mats = np.stack(w2c_mats, axis=0)
154
+
155
+ # Convert extrinsics to camera-to-world.
156
+ camtoworlds = np.linalg.inv(w2c_mats)
157
+
158
+ # Image names from COLMAP. No need for permuting the poses according to
159
+ # image names anymore.
160
+ image_names = [imdata[k].name for k in imdata]
161
+
162
+ # Previous Nerf results were generated with images sorted by filename,
163
+ # ensure metrics are reported on the same test set.
164
+ inds = np.argsort(image_names)
165
+ image_names = [image_names[i] for i in inds]
166
+ camtoworlds = camtoworlds[inds]
167
+ camera_ids = [camera_ids[i] for i in inds]
168
+
169
+ # Load extended metadata. Used by Bilarf dataset.
170
+ self.extconf = {
171
+ "spiral_radius_scale": 1.0,
172
+ "no_factor_suffix": False,
173
+ }
174
+ extconf_file = os.path.join(data_dir, "ext_metadata.json")
175
+ if os.path.exists(extconf_file):
176
+ with open(extconf_file) as f:
177
+ self.extconf.update(json.load(f))
178
+
179
+ # Load bounds if possible (only used in forward facing scenes).
180
+ self.bounds = np.array([0.01, 1.0])
181
+ posefile = os.path.join(data_dir, "poses_bounds.npy")
182
+ if os.path.exists(posefile):
183
+ self.bounds = np.load(posefile)[:, -2:]
184
+
185
+ # Load images.
186
+ if factor > 1 and not self.extconf["no_factor_suffix"]:
187
+ image_dir_suffix = f"_{factor}"
188
+ else:
189
+ image_dir_suffix = ""
190
+ colmap_image_dir = os.path.join(data_dir, "images")
191
+ image_dir = os.path.join(data_dir, "images" + image_dir_suffix)
192
+ for d in [image_dir, colmap_image_dir]:
193
+ if not os.path.exists(d):
194
+ raise ValueError(f"Image folder {d} does not exist.")
195
+
196
+ # Downsampled images may have different names vs images used for COLMAP,
197
+ # so we need to map between the two sorted lists of files.
198
+ colmap_files = sorted(_get_rel_paths(colmap_image_dir))
199
+ image_files = sorted(_get_rel_paths(image_dir))
200
+ if factor > 1 and os.path.splitext(image_files[0])[1].lower() == ".jpg":
201
+ image_dir = _resize_image_folder(
202
+ colmap_image_dir, image_dir + "_png", factor=factor
203
+ )
204
+ image_files = sorted(_get_rel_paths(image_dir))
205
+ colmap_to_image = dict(zip(colmap_files, image_files))
206
+ image_paths = [os.path.join(image_dir, colmap_to_image[f]) for f in image_names]
207
+
208
+ # 3D points and {image_name -> [point_idx]}
209
+ points = manager.points3D.astype(np.float32)
210
+ points_err = manager.point3D_errors.astype(np.float32)
211
+ points_rgb = manager.point3D_colors.astype(np.uint8)
212
+ point_indices = dict()
213
+
214
+ image_id_to_name = {v: k for k, v in manager.name_to_image_id.items()}
215
+ for point_id, data in manager.point3D_id_to_images.items():
216
+ for image_id, _ in data:
217
+ image_name = image_id_to_name[image_id]
218
+ point_idx = manager.point3D_id_to_point3D_idx[point_id]
219
+ point_indices.setdefault(image_name, []).append(point_idx)
220
+ point_indices = {
221
+ k: np.array(v).astype(np.int32) for k, v in point_indices.items()
222
+ }
223
+
224
+ # Normalize the world space.
225
+ if normalize:
226
+ T1 = similarity_from_cameras(camtoworlds)
227
+ camtoworlds = transform_cameras(T1, camtoworlds)
228
+ points = transform_points(T1, points)
229
+
230
+ T2 = align_principal_axes(points)
231
+ camtoworlds = transform_cameras(T2, camtoworlds)
232
+ points = transform_points(T2, points)
233
+
234
+ transform = T2 @ T1
235
+
236
+ # Fix for up side down. We assume more points towards
237
+ # the bottom of the scene which is true when ground floor is
238
+ # present in the images.
239
+ if np.median(points[:, 2]) > np.mean(points[:, 2]):
240
+ # rotate 180 degrees around x axis such that z is flipped
241
+ T3 = np.array(
242
+ [
243
+ [1.0, 0.0, 0.0, 0.0],
244
+ [0.0, -1.0, 0.0, 0.0],
245
+ [0.0, 0.0, -1.0, 0.0],
246
+ [0.0, 0.0, 0.0, 1.0],
247
+ ]
248
+ )
249
+ camtoworlds = transform_cameras(T3, camtoworlds)
250
+ points = transform_points(T3, points)
251
+ transform = T3 @ transform
252
+ else:
253
+ transform = np.eye(4)
254
+
255
+ self.image_names = image_names # List[str], (num_images,)
256
+ self.image_paths = image_paths # List[str], (num_images,)
257
+ self.camtoworlds = camtoworlds # np.ndarray, (num_images, 4, 4)
258
+ self.camera_ids = camera_ids # List[int], (num_images,)
259
+ self.Ks_dict = Ks_dict # Dict of camera_id -> K
260
+ self.params_dict = params_dict # Dict of camera_id -> params
261
+ self.imsize_dict = imsize_dict # Dict of camera_id -> (width, height)
262
+ self.mask_dict = mask_dict # Dict of camera_id -> mask
263
+ self.points = points # np.ndarray, (num_points, 3)
264
+ self.points_err = points_err # np.ndarray, (num_points,)
265
+ self.points_rgb = points_rgb # np.ndarray, (num_points, 3)
266
+ self.point_indices = point_indices # Dict[str, np.ndarray], image_name -> [M,]
267
+ self.transform = transform # np.ndarray, (4, 4)
268
+
269
+ # load one image to check the size. In the case of tanksandtemples dataset, the
270
+ # intrinsics stored in COLMAP corresponds to 2x upsampled images.
271
+ actual_image = imageio.imread(self.image_paths[0])[..., :3]
272
+ actual_height, actual_width = actual_image.shape[:2]
273
+ colmap_width, colmap_height = self.imsize_dict[self.camera_ids[0]]
274
+ s_height, s_width = actual_height / colmap_height, actual_width / colmap_width
275
+ for camera_id, K in self.Ks_dict.items():
276
+ K[0, :] *= s_width
277
+ K[1, :] *= s_height
278
+ self.Ks_dict[camera_id] = K
279
+ width, height = self.imsize_dict[camera_id]
280
+ self.imsize_dict[camera_id] = (int(width * s_width), int(height * s_height))
281
+
282
+ # undistortion
283
+ self.mapx_dict = dict()
284
+ self.mapy_dict = dict()
285
+ self.roi_undist_dict = dict()
286
+ for camera_id in self.params_dict.keys():
287
+ params = self.params_dict[camera_id]
288
+ if len(params) == 0:
289
+ continue # no distortion
290
+ assert camera_id in self.Ks_dict, f"Missing K for camera {camera_id}"
291
+ assert (
292
+ camera_id in self.params_dict
293
+ ), f"Missing params for camera {camera_id}"
294
+ K = self.Ks_dict[camera_id]
295
+ width, height = self.imsize_dict[camera_id]
296
+
297
+ if camtype == "perspective":
298
+ K_undist, roi_undist = cv2.getOptimalNewCameraMatrix(
299
+ K, params, (width, height), 0
300
+ )
301
+ mapx, mapy = cv2.initUndistortRectifyMap(
302
+ K, params, None, K_undist, (width, height), cv2.CV_32FC1
303
+ )
304
+ mask = None
305
+ elif camtype == "fisheye":
306
+ fx = K[0, 0]
307
+ fy = K[1, 1]
308
+ cx = K[0, 2]
309
+ cy = K[1, 2]
310
+ grid_x, grid_y = np.meshgrid(
311
+ np.arange(width, dtype=np.float32),
312
+ np.arange(height, dtype=np.float32),
313
+ indexing="xy",
314
+ )
315
+ x1 = (grid_x - cx) / fx
316
+ y1 = (grid_y - cy) / fy
317
+ theta = np.sqrt(x1**2 + y1**2)
318
+ r = (
319
+ 1.0
320
+ + params[0] * theta**2
321
+ + params[1] * theta**4
322
+ + params[2] * theta**6
323
+ + params[3] * theta**8
324
+ )
325
+ mapx = (fx * x1 * r + width // 2).astype(np.float32)
326
+ mapy = (fy * y1 * r + height // 2).astype(np.float32)
327
+
328
+ # Use mask to define ROI
329
+ mask = np.logical_and(
330
+ np.logical_and(mapx > 0, mapy > 0),
331
+ np.logical_and(mapx < width - 1, mapy < height - 1),
332
+ )
333
+ y_indices, x_indices = np.nonzero(mask)
334
+ y_min, y_max = y_indices.min(), y_indices.max() + 1
335
+ x_min, x_max = x_indices.min(), x_indices.max() + 1
336
+ mask = mask[y_min:y_max, x_min:x_max]
337
+ K_undist = K.copy()
338
+ K_undist[0, 2] -= x_min
339
+ K_undist[1, 2] -= y_min
340
+ roi_undist = [x_min, y_min, x_max - x_min, y_max - y_min]
341
+ else:
342
+ assert_never(camtype)
343
+
344
+ self.mapx_dict[camera_id] = mapx
345
+ self.mapy_dict[camera_id] = mapy
346
+ self.Ks_dict[camera_id] = K_undist
347
+ self.roi_undist_dict[camera_id] = roi_undist
348
+ self.imsize_dict[camera_id] = (roi_undist[2], roi_undist[3])
349
+ self.mask_dict[camera_id] = mask
350
+
351
+ # size of the scene measured by cameras
352
+ camera_locations = camtoworlds[:, :3, 3]
353
+ scene_center = np.mean(camera_locations, axis=0)
354
+ dists = np.linalg.norm(camera_locations - scene_center, axis=1)
355
+ self.scene_scale = np.max(dists)
356
+
357
+
358
+ class Dataset:
359
+ """A simple dataset class."""
360
+
361
+ def __init__(
362
+ self,
363
+ # parser: Parser,
364
+ images: np.ndarray,
365
+ camtoworlds: np.ndarray,
366
+ Ks: np.ndarray,
367
+ split: str = "train",
368
+ patch_size: Optional[int] = None,
369
+ load_depths: bool = False,
370
+ ):
371
+ # self.parser = parser
372
+ self.split = split
373
+ self.patch_size = patch_size
374
+ self.load_depths = load_depths
375
+ self.images = images
376
+ self.camtoworlds = camtoworlds
377
+ self.Ks = Ks
378
+ H, W = self.images.shape[-2:]
379
+ self.Ks[:, 0, :] *= W
380
+ self.Ks[:, 1, :] *= H
381
+ self.indices = np.arange(len(images))
382
+ # indices = np.arange(len(self.parser.image_names))
383
+ # if split == "train":
384
+ # self.indices = indices[indices % self.parser.test_every != 0]
385
+ # else:
386
+ # self.indices = indices[indices % self.parser.test_every == 0]
387
+
388
+ # if split == "train":
389
+ # self.images = np.load(os.path.join(self.parser.true_data_dir, "context_image.npy"))
390
+ # self.camtoworlds = np.load(os.path.join(self.parser.true_data_dir, "context_extrinsic.npy"))
391
+ # self.Ks = np.load(os.path.join(self.parser.true_data_dir, "context_intrinsic.npy"))
392
+ # H, W = self.images.shape[-2:]
393
+ # self.Ks[:, 0, :] *= W
394
+ # self.Ks[:, 1, :] *= H
395
+ # self.indices = np.arange(len(self.images))
396
+ # else:
397
+ # self.images = np.load(os.path.join(self.parser.true_data_dir, "target_image.npy"))
398
+ # self.camtoworlds = np.load(os.path.join(self.parser.true_data_dir, "target_extrinsic.npy"))
399
+ # self.Ks = np.load(os.path.join(self.parser.true_data_dir, "target_intrinsic.npy"))
400
+ # H, W = self.images.shape[-2:]
401
+ # self.Ks[:, 0, :] *= W
402
+ # self.Ks[:, 1, :] *= H
403
+ # self.indices = np.arange(len(self.images))
404
+
405
+ def __len__(self):
406
+ return len(self.indices)
407
+
408
+ def __getitem__(self, item: int) -> Dict[str, Any]:
409
+ index = self.indices[item]
410
+ image = (self.images[index]*255.0).transpose(1, 2, 0).astype(np.uint8) # (H, W, 3)
411
+ K = self.Ks[index].copy() # undistorted K
412
+ params = None
413
+ camtoworlds = self.camtoworlds[index]
414
+ mask = None
415
+
416
+ if self.patch_size is not None:
417
+ # Random crop.
418
+ h, w = image.shape[:2]
419
+ x = np.random.randint(0, max(w - self.patch_size, 1))
420
+ y = np.random.randint(0, max(h - self.patch_size, 1))
421
+ image = image[y : y + self.patch_size, x : x + self.patch_size]
422
+ K[0, 2] -= x
423
+ K[1, 2] -= y
424
+
425
+ data = {
426
+ "K": torch.from_numpy(K).float(),
427
+ "camtoworld": torch.from_numpy(camtoworlds).float(),
428
+ "image": torch.from_numpy(image).float(),
429
+ "image_id": item, # the index of the image in the dataset
430
+ }
431
+ if mask is not None:
432
+ data["mask"] = torch.from_numpy(mask).bool()
433
+
434
+ if self.load_depths and False:
435
+ # projected points to image plane to get depths
436
+ worldtocams = np.linalg.inv(camtoworlds)
437
+ image_name = self.parser.image_names[index]
438
+ point_indices = self.parser.point_indices[image_name]
439
+ points_world = self.parser.points[point_indices]
440
+ points_cam = (worldtocams[:3, :3] @ points_world.T + worldtocams[:3, 3:4]).T
441
+ points_proj = (K @ points_cam.T).T
442
+ points = points_proj[:, :2] / points_proj[:, 2:3] # (M, 2)
443
+ depths = points_cam[:, 2] # (M,)
444
+ # filter out points outside the image
445
+ selector = (
446
+ (points[:, 0] >= 0)
447
+ & (points[:, 0] < image.shape[1])
448
+ & (points[:, 1] >= 0)
449
+ & (points[:, 1] < image.shape[0])
450
+ & (depths > 0)
451
+ )
452
+ points = points[selector]
453
+ depths = depths[selector]
454
+ data["points"] = torch.from_numpy(points).float()
455
+ data["depths"] = torch.from_numpy(depths).float()
456
+
457
+ return data
458
+
459
+
460
+ if __name__ == "__main__":
461
+ import argparse
462
+
463
+ import imageio.v2 as imageio
464
+
465
+ parser = argparse.ArgumentParser()
466
+ parser.add_argument("--data_dir", type=str, default="data/mipnerf360/garden")
467
+ parser.add_argument("--true_data_dir", type=str, default="/cpfs01/user/jianglihan/projects/anysplat_baselines/demo_data/infer_output/3F_100view/room5")
468
+ parser.add_argument("--factor", type=int, default=4)
469
+ args = parser.parse_args()
470
+
471
+ # Parse COLMAP data.
472
+ parser = Parser(
473
+ data_dir=args.data_dir,
474
+ true_data_dir=args.true_data_dir,
475
+ factor=args.factor,
476
+ normalize=True,
477
+ test_every=8
478
+ )
479
+ dataset = Dataset(parser, split="train", load_depths=True)
480
+ print(f"Dataset: {len(dataset)} images.")
481
+
482
+ writer = imageio.get_writer("results/points.mp4", fps=30)
483
+ for data in tqdm(dataset, desc="Plotting points"):
484
+ image = data["image"].numpy().astype(np.uint8)
485
+ points = data["points"].numpy()
486
+ depths = data["depths"].numpy()
487
+ for x, y in points:
488
+ cv2.circle(image, (int(x), int(y)), 2, (255, 0, 0), -1)
489
+ writer.append_data(image)
490
+ writer.close()
src/post_opt/datasets/normalize.py ADDED
@@ -0,0 +1,143 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+
3
+
4
+ def similarity_from_cameras(c2w, strict_scaling=False, center_method="focus"):
5
+ """
6
+ reference: nerf-factory
7
+ Get a similarity transform to normalize dataset
8
+ from c2w (OpenCV convention) cameras
9
+ :param c2w: (N, 4)
10
+ :return T (4,4) , scale (float)
11
+ """
12
+ t = c2w[:, :3, 3]
13
+ R = c2w[:, :3, :3]
14
+
15
+ # (1) Rotate the world so that z+ is the up axis
16
+ # we estimate the up axis by averaging the camera up axes
17
+ ups = np.sum(R * np.array([0, -1.0, 0]), axis=-1)
18
+ world_up = np.mean(ups, axis=0)
19
+ world_up /= np.linalg.norm(world_up)
20
+
21
+ up_camspace = np.array([0.0, -1.0, 0.0])
22
+ c = (up_camspace * world_up).sum()
23
+ cross = np.cross(world_up, up_camspace)
24
+ skew = np.array(
25
+ [
26
+ [0.0, -cross[2], cross[1]],
27
+ [cross[2], 0.0, -cross[0]],
28
+ [-cross[1], cross[0], 0.0],
29
+ ]
30
+ )
31
+ if c > -1:
32
+ R_align = np.eye(3) + skew + (skew @ skew) * 1 / (1 + c)
33
+ else:
34
+ # In the unlikely case the original data has y+ up axis,
35
+ # rotate 180-deg about x axis
36
+ R_align = np.array([[-1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]])
37
+
38
+ # R_align = np.eye(3) # DEBUG
39
+ R = R_align @ R
40
+ fwds = np.sum(R * np.array([0, 0.0, 1.0]), axis=-1)
41
+ t = (R_align @ t[..., None])[..., 0]
42
+
43
+ # (2) Recenter the scene.
44
+ if center_method == "focus":
45
+ # find the closest point to the origin for each camera's center ray
46
+ nearest = t + (fwds * -t).sum(-1)[:, None] * fwds
47
+ translate = -np.median(nearest, axis=0)
48
+ elif center_method == "poses":
49
+ # use center of the camera positions
50
+ translate = -np.median(t, axis=0)
51
+ else:
52
+ raise ValueError(f"Unknown center_method {center_method}")
53
+
54
+ transform = np.eye(4)
55
+ transform[:3, 3] = translate
56
+ transform[:3, :3] = R_align
57
+
58
+ # (3) Rescale the scene using camera distances
59
+ scale_fn = np.max if strict_scaling else np.median
60
+ scale = 1.0 / scale_fn(np.linalg.norm(t + translate, axis=-1))
61
+ transform[:3, :] *= scale
62
+
63
+ return transform
64
+
65
+
66
+ def align_principal_axes(point_cloud):
67
+ # Compute centroid
68
+ centroid = np.median(point_cloud, axis=0)
69
+
70
+ # Translate point cloud to centroid
71
+ translated_point_cloud = point_cloud - centroid
72
+
73
+ # Compute covariance matrix
74
+ covariance_matrix = np.cov(translated_point_cloud, rowvar=False)
75
+
76
+ # Compute eigenvectors and eigenvalues
77
+ eigenvalues, eigenvectors = np.linalg.eigh(covariance_matrix)
78
+
79
+ # Sort eigenvectors by eigenvalues (descending order) so that the z-axis
80
+ # is the principal axis with the smallest eigenvalue.
81
+ sort_indices = eigenvalues.argsort()[::-1]
82
+ eigenvectors = eigenvectors[:, sort_indices]
83
+
84
+ # Check orientation of eigenvectors. If the determinant of the eigenvectors is
85
+ # negative, then we need to flip the sign of one of the eigenvectors.
86
+ if np.linalg.det(eigenvectors) < 0:
87
+ eigenvectors[:, 0] *= -1
88
+
89
+ # Create rotation matrix
90
+ rotation_matrix = eigenvectors.T
91
+
92
+ # Create SE(3) matrix (4x4 transformation matrix)
93
+ transform = np.eye(4)
94
+ transform[:3, :3] = rotation_matrix
95
+ transform[:3, 3] = -rotation_matrix @ centroid
96
+
97
+ return transform
98
+
99
+
100
+ def transform_points(matrix, points):
101
+ """Transform points using an SE(3) matrix.
102
+
103
+ Args:
104
+ matrix: 4x4 SE(3) matrix
105
+ points: Nx3 array of points
106
+
107
+ Returns:
108
+ Nx3 array of transformed points
109
+ """
110
+ assert matrix.shape == (4, 4)
111
+ assert len(points.shape) == 2 and points.shape[1] == 3
112
+ return points @ matrix[:3, :3].T + matrix[:3, 3]
113
+
114
+
115
+ def transform_cameras(matrix, camtoworlds):
116
+ """Transform cameras using an SE(3) matrix.
117
+
118
+ Args:
119
+ matrix: 4x4 SE(3) matrix
120
+ camtoworlds: Nx4x4 array of camera-to-world matrices
121
+
122
+ Returns:
123
+ Nx4x4 array of transformed camera-to-world matrices
124
+ """
125
+ assert matrix.shape == (4, 4)
126
+ assert len(camtoworlds.shape) == 3 and camtoworlds.shape[1:] == (4, 4)
127
+ camtoworlds = np.einsum("nij, ki -> nkj", camtoworlds, matrix)
128
+ scaling = np.linalg.norm(camtoworlds[:, 0, :3], axis=1)
129
+ camtoworlds[:, :3, :3] = camtoworlds[:, :3, :3] / scaling[:, None, None]
130
+ return camtoworlds
131
+
132
+
133
+ def normalize(camtoworlds, points=None):
134
+ T1 = similarity_from_cameras(camtoworlds)
135
+ camtoworlds = transform_cameras(T1, camtoworlds)
136
+ if points is not None:
137
+ points = transform_points(T1, points)
138
+ T2 = align_principal_axes(points)
139
+ camtoworlds = transform_cameras(T2, camtoworlds)
140
+ points = transform_points(T2, points)
141
+ return camtoworlds, points, T2 @ T1
142
+ else:
143
+ return camtoworlds, T1
src/post_opt/datasets/traj.py ADDED
@@ -0,0 +1,254 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Code borrowed from
3
+
4
+ https://github.com/google-research/multinerf/blob/5b4d4f64608ec8077222c52fdf814d40acc10bc1/internal/camera_utils.py
5
+ """
6
+
7
+ import numpy as np
8
+ import scipy
9
+
10
+
11
+ def normalize(x: np.ndarray) -> np.ndarray:
12
+ """Normalization helper function."""
13
+ return x / np.linalg.norm(x)
14
+
15
+
16
+ def viewmatrix(lookdir: np.ndarray, up: np.ndarray, position: np.ndarray) -> np.ndarray:
17
+ """Construct lookat view matrix."""
18
+ vec2 = normalize(lookdir)
19
+ vec0 = normalize(np.cross(up, vec2))
20
+ vec1 = normalize(np.cross(vec2, vec0))
21
+ m = np.stack([vec0, vec1, vec2, position], axis=1)
22
+ return m
23
+
24
+
25
+ def focus_point_fn(poses: np.ndarray) -> np.ndarray:
26
+ """Calculate nearest point to all focal axes in poses."""
27
+ directions, origins = poses[:, :3, 2:3], poses[:, :3, 3:4]
28
+ m = np.eye(3) - directions * np.transpose(directions, [0, 2, 1])
29
+ mt_m = np.transpose(m, [0, 2, 1]) @ m
30
+ focus_pt = np.linalg.inv(mt_m.mean(0)) @ (mt_m @ origins).mean(0)[:, 0]
31
+ return focus_pt
32
+
33
+
34
+ def average_pose(poses: np.ndarray) -> np.ndarray:
35
+ """New pose using average position, z-axis, and up vector of input poses."""
36
+ position = poses[:, :3, 3].mean(0)
37
+ z_axis = poses[:, :3, 2].mean(0)
38
+ up = poses[:, :3, 1].mean(0)
39
+ cam2world = viewmatrix(z_axis, up, position)
40
+ return cam2world
41
+
42
+
43
+ def generate_spiral_path(
44
+ poses,
45
+ bounds,
46
+ n_frames=120,
47
+ n_rots=2,
48
+ zrate=0.5,
49
+ spiral_scale_f=1.0,
50
+ spiral_scale_r=1.0,
51
+ focus_distance=0.75,
52
+ ):
53
+ """Calculates a forward facing spiral path for rendering."""
54
+ # Find a reasonable 'focus depth' for this dataset as a weighted average
55
+ # of conservative near and far bounds in disparity space.
56
+ near_bound = bounds.min()
57
+ far_bound = bounds.max()
58
+ # All cameras will point towards the world space point (0, 0, -focal).
59
+ focal = 1 / (((1 - focus_distance) / near_bound + focus_distance / far_bound))
60
+ focal = focal * spiral_scale_f
61
+
62
+ # Get radii for spiral path using 90th percentile of camera positions.
63
+ positions = poses[:, :3, 3]
64
+ radii = np.percentile(np.abs(positions), 90, 0)
65
+ radii = radii * spiral_scale_r
66
+ radii = np.concatenate([radii, [1.0]])
67
+
68
+ # Generate poses for spiral path.
69
+ render_poses = []
70
+ cam2world = average_pose(poses)
71
+ up = poses[:, :3, 1].mean(0)
72
+ for theta in np.linspace(0.0, 2.0 * np.pi * n_rots, n_frames, endpoint=False):
73
+ t = radii * [np.cos(theta), -np.sin(theta), -np.sin(theta * zrate), 1.0]
74
+ position = cam2world @ t
75
+ lookat = cam2world @ [0, 0, -focal, 1.0]
76
+ z_axis = position - lookat
77
+ render_poses.append(viewmatrix(z_axis, up, position))
78
+ render_poses = np.stack(render_poses, axis=0)
79
+ return render_poses
80
+
81
+
82
+ def generate_ellipse_path_z(
83
+ poses: np.ndarray,
84
+ n_frames: int = 120,
85
+ # const_speed: bool = True,
86
+ variation: float = 0.0,
87
+ phase: float = 0.0,
88
+ height: float = 0.0,
89
+ ) -> np.ndarray:
90
+ """Generate an elliptical render path based on the given poses."""
91
+ # Calculate the focal point for the path (cameras point toward this).
92
+ center = focus_point_fn(poses)
93
+ # Path height sits at z=height (in middle of zero-mean capture pattern).
94
+ offset = np.array([center[0], center[1], height])
95
+
96
+ # Calculate scaling for ellipse axes based on input camera positions.
97
+ sc = np.percentile(np.abs(poses[:, :3, 3] - offset), 90, axis=0)
98
+ # Use ellipse that is symmetric about the focal point in xy.
99
+ low = -sc + offset
100
+ high = sc + offset
101
+ # Optional height variation need not be symmetric
102
+ z_low = np.percentile((poses[:, :3, 3]), 10, axis=0)
103
+ z_high = np.percentile((poses[:, :3, 3]), 90, axis=0)
104
+
105
+ def get_positions(theta):
106
+ # Interpolate between bounds with trig functions to get ellipse in x-y.
107
+ # Optionally also interpolate in z to change camera height along path.
108
+ return np.stack(
109
+ [
110
+ low[0] + (high - low)[0] * (np.cos(theta) * 0.5 + 0.5),
111
+ low[1] + (high - low)[1] * (np.sin(theta) * 0.5 + 0.5),
112
+ variation
113
+ * (
114
+ z_low[2]
115
+ + (z_high - z_low)[2]
116
+ * (np.cos(theta + 2 * np.pi * phase) * 0.5 + 0.5)
117
+ )
118
+ + height,
119
+ ],
120
+ -1,
121
+ )
122
+
123
+ theta = np.linspace(0, 2.0 * np.pi, n_frames + 1, endpoint=True)
124
+ positions = get_positions(theta)
125
+
126
+ # if const_speed:
127
+ # # Resample theta angles so that the velocity is closer to constant.
128
+ # lengths = np.linalg.norm(positions[1:] - positions[:-1], axis=-1)
129
+ # theta = stepfun.sample(None, theta, np.log(lengths), n_frames + 1)
130
+ # positions = get_positions(theta)
131
+
132
+ # Throw away duplicated last position.
133
+ positions = positions[:-1]
134
+
135
+ # Set path's up vector to axis closest to average of input pose up vectors.
136
+ avg_up = poses[:, :3, 1].mean(0)
137
+ avg_up = avg_up / np.linalg.norm(avg_up)
138
+ ind_up = np.argmax(np.abs(avg_up))
139
+ up = np.eye(3)[ind_up] * np.sign(avg_up[ind_up])
140
+
141
+ return np.stack([viewmatrix(center - p, up, p) for p in positions])
142
+
143
+
144
+ def generate_ellipse_path_y(
145
+ poses: np.ndarray,
146
+ n_frames: int = 120,
147
+ # const_speed: bool = True,
148
+ variation: float = 0.0,
149
+ phase: float = 0.0,
150
+ height: float = 0.0,
151
+ ) -> np.ndarray:
152
+ """Generate an elliptical render path based on the given poses."""
153
+ # Calculate the focal point for the path (cameras point toward this).
154
+ center = focus_point_fn(poses)
155
+ # Path height sits at y=height (in middle of zero-mean capture pattern).
156
+ offset = np.array([center[0], height, center[2]])
157
+
158
+ # Calculate scaling for ellipse axes based on input camera positions.
159
+ sc = np.percentile(np.abs(poses[:, :3, 3] - offset), 90, axis=0)
160
+ # Use ellipse that is symmetric about the focal point in xy.
161
+ low = -sc + offset
162
+ high = sc + offset
163
+ # Optional height variation need not be symmetric
164
+ y_low = np.percentile((poses[:, :3, 3]), 10, axis=0)
165
+ y_high = np.percentile((poses[:, :3, 3]), 90, axis=0)
166
+
167
+ def get_positions(theta):
168
+ # Interpolate between bounds with trig functions to get ellipse in x-z.
169
+ # Optionally also interpolate in y to change camera height along path.
170
+ return np.stack(
171
+ [
172
+ low[0] + (high - low)[0] * (np.cos(theta) * 0.5 + 0.5),
173
+ variation
174
+ * (
175
+ y_low[1]
176
+ + (y_high - y_low)[1]
177
+ * (np.cos(theta + 2 * np.pi * phase) * 0.5 + 0.5)
178
+ )
179
+ + height,
180
+ low[2] + (high - low)[2] * (np.sin(theta) * 0.5 + 0.5),
181
+ ],
182
+ -1,
183
+ )
184
+
185
+ theta = np.linspace(0, 2.0 * np.pi, n_frames + 1, endpoint=True)
186
+ positions = get_positions(theta)
187
+
188
+ # if const_speed:
189
+ # # Resample theta angles so that the velocity is closer to constant.
190
+ # lengths = np.linalg.norm(positions[1:] - positions[:-1], axis=-1)
191
+ # theta = stepfun.sample(None, theta, np.log(lengths), n_frames + 1)
192
+ # positions = get_positions(theta)
193
+
194
+ # Throw away duplicated last position.
195
+ positions = positions[:-1]
196
+
197
+ # Set path's up vector to axis closest to average of input pose up vectors.
198
+ avg_up = poses[:, :3, 1].mean(0)
199
+ avg_up = avg_up / np.linalg.norm(avg_up)
200
+ ind_up = np.argmax(np.abs(avg_up))
201
+ up = np.eye(3)[ind_up] * np.sign(avg_up[ind_up])
202
+
203
+ return np.stack([viewmatrix(p - center, up, p) for p in positions])
204
+
205
+
206
+ def generate_interpolated_path(
207
+ poses: np.ndarray,
208
+ n_interp: int,
209
+ spline_degree: int = 5,
210
+ smoothness: float = 0.03,
211
+ rot_weight: float = 0.1,
212
+ ):
213
+ """Creates a smooth spline path between input keyframe camera poses.
214
+
215
+ Spline is calculated with poses in format (position, lookat-point, up-point).
216
+
217
+ Args:
218
+ poses: (n, 3, 4) array of input pose keyframes.
219
+ n_interp: returned path will have n_interp * (n - 1) total poses.
220
+ spline_degree: polynomial degree of B-spline.
221
+ smoothness: parameter for spline smoothing, 0 forces exact interpolation.
222
+ rot_weight: relative weighting of rotation/translation in spline solve.
223
+
224
+ Returns:
225
+ Array of new camera poses with shape (n_interp * (n - 1), 3, 4).
226
+ """
227
+
228
+ def poses_to_points(poses, dist):
229
+ """Converts from pose matrices to (position, lookat, up) format."""
230
+ pos = poses[:, :3, -1]
231
+ lookat = poses[:, :3, -1] - dist * poses[:, :3, 2]
232
+ up = poses[:, :3, -1] + dist * poses[:, :3, 1]
233
+ return np.stack([pos, lookat, up], 1)
234
+
235
+ def points_to_poses(points):
236
+ """Converts from (position, lookat, up) format to pose matrices."""
237
+ return np.array([viewmatrix(p - l, u - p, p) for p, l, u in points])
238
+
239
+ def interp(points, n, k, s):
240
+ """Runs multidimensional B-spline interpolation on the input points."""
241
+ sh = points.shape
242
+ pts = np.reshape(points, (sh[0], -1))
243
+ k = min(k, sh[0] - 1)
244
+ tck, _ = scipy.interpolate.splprep(pts.T, k=k, s=s)
245
+ u = np.linspace(0, 1, n, endpoint=False)
246
+ new_points = np.array(scipy.interpolate.splev(u, tck))
247
+ new_points = np.reshape(new_points.T, (n, sh[1], sh[2]))
248
+ return new_points
249
+
250
+ points = poses_to_points(poses, dist=rot_weight)
251
+ new_points = interp(
252
+ points, n_interp * (points.shape[0] - 1), k=spline_degree, s=smoothness
253
+ )
254
+ return points_to_poses(new_points)
src/post_opt/exporter.py ADDED
@@ -0,0 +1,553 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ import struct
3
+ from io import BytesIO
4
+ from typing import Literal, Optional
5
+
6
+ import numpy as np
7
+ import torch
8
+
9
+
10
+ def sh2rgb(sh: torch.Tensor) -> torch.Tensor:
11
+ """Convert Sphere Harmonics to RGB
12
+
13
+ Args:
14
+ sh (torch.Tensor): SH tensor
15
+
16
+ Returns:
17
+ torch.Tensor: RGB tensor
18
+ """
19
+ C0 = 0.28209479177387814
20
+ return sh * C0 + 0.5
21
+
22
+
23
+ def part1by2_vec(x: torch.Tensor) -> torch.Tensor:
24
+ """Interleave bits of x with 0s
25
+
26
+ Args:
27
+ x (torch.Tensor): Input tensor. Shape (N,)
28
+
29
+ Returns:
30
+ torch.Tensor: Output tensor. Shape (N,)
31
+ """
32
+
33
+ x = x & 0x000003FF
34
+ x = (x ^ (x << 16)) & 0xFF0000FF
35
+ x = (x ^ (x << 8)) & 0x0300F00F
36
+ x = (x ^ (x << 4)) & 0x030C30C3
37
+ x = (x ^ (x << 2)) & 0x09249249
38
+ return x
39
+
40
+
41
+ def encode_morton3_vec(
42
+ x: torch.Tensor, y: torch.Tensor, z: torch.Tensor
43
+ ) -> torch.Tensor:
44
+ """Compute Morton codes for 3D coordinates
45
+
46
+ Args:
47
+ x (torch.Tensor): X coordinates. Shape (N,)
48
+ y (torch.Tensor): Y coordinates. Shape (N,)
49
+ z (torch.Tensor): Z coordinates. Shape (N,)
50
+ Returns:
51
+ torch.Tensor: Morton codes. Shape (N,)
52
+ """
53
+ return (part1by2_vec(z) << 2) + (part1by2_vec(y) << 1) + part1by2_vec(x)
54
+
55
+
56
+ def sort_centers(centers: torch.Tensor, indices: torch.Tensor) -> torch.Tensor:
57
+ """Sort centers based on Morton codes
58
+
59
+ Args:
60
+ centers (torch.Tensor): Centers. Shape (N, 3)
61
+ indices (torch.Tensor): Indices. Shape (N,)
62
+ Returns:
63
+ torch.Tensor: Sorted indices. Shape (N,)
64
+ """
65
+ # Compute min and max values in a single operation
66
+ min_vals, _ = torch.min(centers, dim=0)
67
+ max_vals, _ = torch.max(centers, dim=0)
68
+
69
+ # Compute the scaling factors
70
+ lengths = max_vals - min_vals
71
+ lengths[lengths == 0] = 1 # Prevent division by zero
72
+
73
+ # Normalize and scale to 10-bit integer range (0-1024)
74
+ scaled_centers = ((centers - min_vals) / lengths * 1024).floor().to(torch.int32)
75
+
76
+ # Extract x, y, z coordinates
77
+ x, y, z = scaled_centers[:, 0], scaled_centers[:, 1], scaled_centers[:, 2]
78
+
79
+ # Compute Morton codes using vectorized operations
80
+ morton = encode_morton3_vec(x, y, z)
81
+
82
+ # Sort indices based on Morton codes
83
+ sorted_indices = indices[torch.argsort(morton).to(indices.device)]
84
+
85
+ return sorted_indices
86
+
87
+
88
+ def pack_unorm(value: torch.Tensor, bits: int) -> torch.Tensor:
89
+ """Pack a floating point value into an unsigned integer with a given number of bits.
90
+
91
+ Args:
92
+ value (torch.Tensor): Floating point value to pack. Shape (N,)
93
+ bits (int): Number of bits to pack into.
94
+
95
+ Returns:
96
+ torch.Tensor: Packed value. Shape (N,)
97
+ """
98
+
99
+ t = (1 << bits) - 1
100
+ packed = torch.clamp((value * t + 0.5).floor(), min=0, max=t)
101
+ # Convert to integer type
102
+ return packed.to(torch.int64)
103
+
104
+
105
+ def pack_111011(x: torch.Tensor, y: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
106
+ """Pack three floating point values into a 32-bit integer with 11, 10, and 11 bits.
107
+
108
+ Args:
109
+ x (torch.Tensor): X component. Shape (N,)
110
+ y (torch.Tensor): Y component. Shape (N,)
111
+ z (torch.Tensor): Z component. Shape (N,)
112
+ Returns:
113
+ torch.Tensor: Packed values. Shape (N,)
114
+ """
115
+ # Pack each component using pack_unorm
116
+ packed_x = pack_unorm(x, 11) << 21
117
+ packed_y = pack_unorm(y, 10) << 11
118
+ packed_z = pack_unorm(z, 11)
119
+
120
+ # Combine the packed values using bitwise OR
121
+ return packed_x | packed_y | packed_z
122
+
123
+
124
+ def pack_8888(
125
+ x: torch.Tensor, y: torch.Tensor, z: torch.Tensor, w: torch.Tensor
126
+ ) -> torch.Tensor:
127
+ """Pack four floating point values into a 32-bit integer with 8 bits each.
128
+
129
+ Args:
130
+ x (torch.Tensor): X component. Shape (N,)
131
+ y (torch.Tensor): Y component. Shape (N,)
132
+ z (torch.Tensor): Z component. Shape (N,)
133
+ w (torch.Tensor): W component. Shape (N,)
134
+ Returns:
135
+ torch.Tensor: Packed values. Shape (N,)
136
+ """
137
+ # Pack each component using pack_unorm
138
+ packed_x = pack_unorm(x, 8) << 24
139
+ packed_y = pack_unorm(y, 8) << 16
140
+ packed_z = pack_unorm(z, 8) << 8
141
+ packed_w = pack_unorm(w, 8)
142
+
143
+ # Combine the packed values using bitwise OR
144
+ return packed_x | packed_y | packed_z | packed_w
145
+
146
+
147
+ def pack_rotation(q: torch.Tensor) -> torch.Tensor:
148
+ """Pack a quaternion into a 32-bit integer.
149
+
150
+ Args:
151
+ q (torch.Tensor): Quaternions. Shape (N, 4)
152
+
153
+ Returns:
154
+ torch.Tensor: Packed values. Shape (N,)
155
+ """
156
+
157
+ # Normalize each quaternion
158
+ norms = torch.linalg.norm(q, dim=-1, keepdim=True)
159
+ q = q / norms
160
+
161
+ # Find the largest component index for each quaternion
162
+ largest_components = torch.argmax(torch.abs(q), dim=-1)
163
+
164
+ # Flip quaternions where the largest component is negative
165
+ batch_indices = torch.arange(q.size(0), device=q.device)
166
+ largest_values = q[batch_indices, largest_components]
167
+ flip_mask = largest_values < 0
168
+ q[flip_mask] *= -1
169
+
170
+ # Precomputed indices for the components to pack (excluding largest)
171
+ precomputed_indices = torch.tensor(
172
+ [[1, 2, 3], [0, 2, 3], [0, 1, 3], [0, 1, 2]], dtype=torch.long, device=q.device
173
+ )
174
+
175
+ # Gather components to pack for each quaternion
176
+ pack_indices = precomputed_indices[largest_components]
177
+ components_to_pack = q[batch_indices[:, None], pack_indices]
178
+
179
+ # Scale and pack each component into 10-bit integers
180
+ norm = math.sqrt(2) * 0.5
181
+ scaled = components_to_pack * norm + 0.5
182
+ packed = pack_unorm(scaled, 10) # Assuming pack_unorm is vectorized
183
+
184
+ # Combine into the final 32-bit integer
185
+ largest_packed = largest_components.to(torch.int64) << 30
186
+ c0_packed = packed[:, 0] << 20
187
+ c1_packed = packed[:, 1] << 10
188
+ c2_packed = packed[:, 2]
189
+
190
+ result = largest_packed | c0_packed | c1_packed | c2_packed
191
+ return result
192
+
193
+
194
+ def splat2ply_bytes_compressed(
195
+ means: torch.Tensor,
196
+ scales: torch.Tensor,
197
+ quats: torch.Tensor,
198
+ opacities: torch.Tensor,
199
+ sh0: torch.Tensor,
200
+ shN: torch.Tensor,
201
+ chunk_max_size: int = 256,
202
+ opacity_threshold: float = 1 / 255,
203
+ ) -> bytes:
204
+ """Return the binary compressed Ply file. Used by Supersplat viewer.
205
+
206
+ Args:
207
+ means (torch.Tensor): Splat means. Shape (N, 3)
208
+ scales (torch.Tensor): Splat scales. Shape (N, 3)
209
+ quats (torch.Tensor): Splat quaternions. Shape (N, 4)
210
+ opacities (torch.Tensor): Splat opacities. Shape (N,)
211
+ sh0 (torch.Tensor): Spherical harmonics. Shape (N, 3)
212
+ shN (torch.Tensor): Spherical harmonics. Shape (N, K*3)
213
+ chunk_max_size (int): Maximum number of splats per chunk. Default: 256
214
+ opacity_threshold (float): Opacity threshold. Default: 1 / 255
215
+
216
+ Returns:
217
+ bytes: Binary compressed Ply file representing the model.
218
+ """
219
+
220
+ # Filter the splats with too low opacity
221
+ mask = torch.sigmoid(opacities) > opacity_threshold
222
+ means = means[mask]
223
+ scales = scales[mask]
224
+ sh0_colors = sh2rgb(sh0)
225
+ sh0_colors = sh0_colors[mask]
226
+ shN = shN[mask]
227
+ quats = quats[mask]
228
+ opacities = opacities[mask]
229
+
230
+ num_splats = means.shape[0]
231
+ n_chunks = num_splats // chunk_max_size + (num_splats % chunk_max_size != 0)
232
+ indices = torch.arange(num_splats)
233
+ indices = sort_centers(means, indices)
234
+
235
+ float_properties = [
236
+ "min_x",
237
+ "min_y",
238
+ "min_z",
239
+ "max_x",
240
+ "max_y",
241
+ "max_z",
242
+ "min_scale_x",
243
+ "min_scale_y",
244
+ "min_scale_z",
245
+ "max_scale_x",
246
+ "max_scale_y",
247
+ "max_scale_z",
248
+ "min_r",
249
+ "min_g",
250
+ "min_b",
251
+ "max_r",
252
+ "max_g",
253
+ "max_b",
254
+ ]
255
+ uint_properties = [
256
+ "packed_position",
257
+ "packed_rotation",
258
+ "packed_scale",
259
+ "packed_color",
260
+ ]
261
+ buffer = BytesIO()
262
+
263
+ # Write PLY header
264
+ buffer.write(b"ply\n")
265
+ buffer.write(b"format binary_little_endian 1.0\n")
266
+ buffer.write(f"element chunk {n_chunks}\n".encode())
267
+ for prop in float_properties:
268
+ buffer.write(f"property float {prop}\n".encode())
269
+ buffer.write(f"element vertex {num_splats}\n".encode())
270
+ for prop in uint_properties:
271
+ buffer.write(f"property uint {prop}\n".encode())
272
+ buffer.write(f"element sh {num_splats}\n".encode())
273
+ for j in range(shN.shape[1]):
274
+ buffer.write(f"property uchar f_rest_{j}\n".encode())
275
+ buffer.write(b"end_header\n")
276
+
277
+ chunk_data = []
278
+ splat_data = []
279
+ sh_data = []
280
+ for chunk_idx in range(n_chunks):
281
+ chunk_end_idx = min((chunk_idx + 1) * chunk_max_size, num_splats)
282
+ chunk_start_idx = chunk_idx * chunk_max_size
283
+ splat_idxs = indices[chunk_start_idx:chunk_end_idx]
284
+
285
+ # Bounds
286
+ # Means
287
+ chunk_means = means[splat_idxs]
288
+ min_means = torch.min(chunk_means, dim=0).values
289
+ max_means = torch.max(chunk_means, dim=0).values
290
+ mean_bounds = torch.cat([min_means, max_means])
291
+ # Scales
292
+ chunk_scales = scales[splat_idxs]
293
+ min_scales = torch.min(chunk_scales, dim=0).values
294
+ max_scales = torch.max(chunk_scales, dim=0).values
295
+ min_scales = torch.clamp(min_scales, -20, 20)
296
+ max_scales = torch.clamp(max_scales, -20, 20)
297
+ scale_bounds = torch.cat([min_scales, max_scales])
298
+ # Colors
299
+ chunk_colors = sh0_colors[splat_idxs]
300
+ min_colors = torch.min(chunk_colors, dim=0).values
301
+ max_colors = torch.max(chunk_colors, dim=0).values
302
+ color_bounds = torch.cat([min_colors, max_colors])
303
+ chunk_data.extend([mean_bounds, scale_bounds, color_bounds])
304
+
305
+ # Quantized properties:
306
+ # Means
307
+ normalized_means = (chunk_means - min_means) / (max_means - min_means)
308
+ means_i = pack_111011(
309
+ normalized_means[:, 0],
310
+ normalized_means[:, 1],
311
+ normalized_means[:, 2],
312
+ )
313
+ # Quaternions
314
+ chunk_quats = quats[splat_idxs]
315
+ quat_i = pack_rotation(chunk_quats)
316
+ # Scales
317
+ normalized_scales = (chunk_scales - min_scales) / (max_scales - min_scales)
318
+ scales_i = pack_111011(
319
+ normalized_scales[:, 0],
320
+ normalized_scales[:, 1],
321
+ normalized_scales[:, 2],
322
+ )
323
+ # Colors
324
+ normalized_colors = (chunk_colors - min_colors) / (max_colors - min_colors)
325
+ chunk_opacities = opacities[splat_idxs]
326
+ chunk_opacities = 1 / (1 + torch.exp(-chunk_opacities))
327
+ chunk_opacities = chunk_opacities.unsqueeze(-1)
328
+ normalized_colors_i = torch.cat([normalized_colors, chunk_opacities], dim=-1)
329
+ color_i = pack_8888(
330
+ normalized_colors_i[:, 0],
331
+ normalized_colors_i[:, 1],
332
+ normalized_colors_i[:, 2],
333
+ normalized_colors_i[:, 3],
334
+ )
335
+ splat_data_chunk = torch.stack([means_i, quat_i, scales_i, color_i], dim=1)
336
+ splat_data_chunk = splat_data_chunk.ravel().to(torch.int64)
337
+ splat_data.extend([splat_data_chunk])
338
+
339
+ # Quantized spherical harmonics
340
+ shN_chunk = shN[splat_idxs]
341
+ shN_chunk_quantized = (shN_chunk / 8 + 0.5) * 256
342
+ shN_chunk_quantized = torch.clamp(torch.trunc(shN_chunk_quantized), 0, 255)
343
+ shN_chunk_quantized = shN_chunk_quantized.to(torch.uint8)
344
+ sh_data.extend([shN_chunk_quantized.ravel()])
345
+
346
+ float_dtype = np.dtype(np.float32).newbyteorder("<")
347
+ uint32_dtype = np.dtype(np.uint32).newbyteorder("<")
348
+ uint8_dtype = np.dtype(np.uint8)
349
+
350
+ buffer.write(
351
+ torch.cat(chunk_data).detach().cpu().numpy().astype(float_dtype).tobytes()
352
+ )
353
+ buffer.write(
354
+ torch.cat(splat_data).detach().cpu().numpy().astype(uint32_dtype).tobytes()
355
+ )
356
+ buffer.write(
357
+ torch.cat(sh_data).detach().cpu().numpy().astype(uint8_dtype).tobytes()
358
+ )
359
+
360
+ return buffer.getvalue()
361
+
362
+
363
+ def splat2ply_bytes(
364
+ means: torch.Tensor,
365
+ scales: torch.Tensor,
366
+ quats: torch.Tensor,
367
+ opacities: torch.Tensor,
368
+ sh0: torch.Tensor,
369
+ shN: torch.Tensor,
370
+ ) -> bytes:
371
+ """Return the binary Ply file. Supported by almost all viewers.
372
+
373
+ Args:
374
+ means (torch.Tensor): Splat means. Shape (N, 3)
375
+ scales (torch.Tensor): Splat scales. Shape (N, 3)
376
+ quats (torch.Tensor): Splat quaternions. Shape (N, 4)
377
+ opacities (torch.Tensor): Splat opacities. Shape (N,)
378
+ sh0 (torch.Tensor): Spherical harmonics. Shape (N, 3)
379
+ shN (torch.Tensor): Spherical harmonics. Shape (N, K*3)
380
+
381
+ Returns:
382
+ bytes: Binary Ply file representing the model.
383
+ """
384
+
385
+ num_splats = means.shape[0]
386
+ buffer = BytesIO()
387
+
388
+ # Write PLY header
389
+ buffer.write(b"ply\n")
390
+ buffer.write(b"format binary_little_endian 1.0\n")
391
+ buffer.write(f"element vertex {num_splats}\n".encode())
392
+ buffer.write(b"property float x\n")
393
+ buffer.write(b"property float y\n")
394
+ buffer.write(b"property float z\n")
395
+ for i, data in enumerate([sh0, shN]):
396
+ prefix = "f_dc" if i == 0 else "f_rest"
397
+ for j in range(data.shape[1]):
398
+ buffer.write(f"property float {prefix}_{j}\n".encode())
399
+ buffer.write(b"property float opacity\n")
400
+ for i in range(scales.shape[1]):
401
+ buffer.write(f"property float scale_{i}\n".encode())
402
+ for i in range(quats.shape[1]):
403
+ buffer.write(f"property float rot_{i}\n".encode())
404
+ buffer.write(b"end_header\n")
405
+
406
+ # Concatenate all tensors in the correct order
407
+ splat_data = torch.cat(
408
+ [means, sh0, shN, opacities.unsqueeze(1), scales, quats], dim=1
409
+ )
410
+ # Ensure correct dtype
411
+ splat_data = splat_data.to(torch.float32)
412
+
413
+ # Write binary data
414
+ float_dtype = np.dtype(np.float32).newbyteorder("<")
415
+ buffer.write(splat_data.detach().cpu().numpy().astype(float_dtype).tobytes())
416
+
417
+ return buffer.getvalue()
418
+
419
+
420
+ def splat2splat_bytes(
421
+ means: torch.Tensor,
422
+ scales: torch.Tensor,
423
+ quats: torch.Tensor,
424
+ opacities: torch.Tensor,
425
+ sh0: torch.Tensor,
426
+ ) -> bytes:
427
+ """Return the binary Splat file. Supported by antimatter15 viewer.
428
+
429
+ Args:
430
+ means (torch.Tensor): Splat means. Shape (N, 3)
431
+ scales (torch.Tensor): Splat scales. Shape (N, 3)
432
+ quats (torch.Tensor): Splat quaternions. Shape (N, 4)
433
+ opacities (torch.Tensor): Splat opacities. Shape (N,)
434
+ sh0 (torch.Tensor): Spherical harmonics. Shape (N, 3)
435
+
436
+ Returns:
437
+ bytes: Binary Splat file representing the model.
438
+ """
439
+
440
+ # Preprocess
441
+ scales = torch.exp(scales)
442
+ sh0_color = sh2rgb(sh0)
443
+ colors = torch.cat([sh0_color, torch.sigmoid(opacities).unsqueeze(-1)], dim=1)
444
+ colors = (colors * 255).clamp(0, 255).to(torch.uint8)
445
+
446
+ rots = (quats / torch.linalg.norm(quats, dim=1, keepdim=True)) * 128 + 128
447
+ rots = rots.clamp(0, 255).to(torch.uint8)
448
+
449
+ # Sort splats
450
+ num_splats = means.shape[0]
451
+ indices = sort_centers(means, torch.arange(num_splats))
452
+
453
+ # Reorder everything
454
+ means = means[indices]
455
+ scales = scales[indices]
456
+ colors = colors[indices]
457
+ rots = rots[indices]
458
+
459
+ float_dtype = np.dtype(np.float32).newbyteorder("<")
460
+ means_np = means.detach().cpu().numpy().astype(float_dtype)
461
+ scales_np = scales.detach().cpu().numpy().astype(float_dtype)
462
+ colors_np = colors.detach().cpu().numpy().astype(np.uint8)
463
+ rots_np = rots.detach().cpu().numpy().astype(np.uint8)
464
+
465
+ buffer = BytesIO()
466
+ for i in range(num_splats):
467
+ buffer.write(means_np[i].tobytes())
468
+ buffer.write(scales_np[i].tobytes())
469
+ buffer.write(colors_np[i].tobytes())
470
+ buffer.write(rots_np[i].tobytes())
471
+
472
+ return buffer.getvalue()
473
+
474
+
475
+ def export_splats(
476
+ means: torch.Tensor,
477
+ scales: torch.Tensor,
478
+ quats: torch.Tensor,
479
+ opacities: torch.Tensor,
480
+ sh0: torch.Tensor,
481
+ shN: torch.Tensor,
482
+ format: Literal["ply", "splat", "ply_compressed"] = "ply",
483
+ save_to: Optional[str] = None,
484
+ ) -> bytes:
485
+ """Export a Gaussian Splats model to bytes.
486
+ The three supported formats are:
487
+ - ply: A standard PLY file format. Supported by most viewers.
488
+ - splat: A custom Splat file format. Supported by antimatter15 viewer.
489
+ - ply_compressed: A compressed PLY file format. Used by Supersplat viewer.
490
+
491
+ Args:
492
+ means (torch.Tensor): Splat means. Shape (N, 3)
493
+ scales (torch.Tensor): Splat scales. Shape (N, 3)
494
+ quats (torch.Tensor): Splat quaternions. Shape (N, 4)
495
+ opacities (torch.Tensor): Splat opacities. Shape (N,)
496
+ sh0 (torch.Tensor): Spherical harmonics. Shape (N, 1, 3)
497
+ shN (torch.Tensor): Spherical harmonics. Shape (N, K, 3)
498
+ format (str): Export format. Options: "ply", "splat", "ply_compressed". Default: "ply"
499
+ save_to (str): Output file path. If provided, the bytes will be written to file.
500
+ """
501
+ total_splats = means.shape[0]
502
+ assert means.shape == (total_splats, 3), "Means must be of shape (N, 3)"
503
+ assert scales.shape == (total_splats, 3), "Scales must be of shape (N, 3)"
504
+ assert quats.shape == (total_splats, 4), "Quaternions must be of shape (N, 4)"
505
+ assert opacities.shape == (total_splats,), "Opacities must be of shape (N,)"
506
+ assert sh0.shape == (total_splats, 1, 3), "sh0 must be of shape (N, 1, 3)"
507
+ assert (
508
+ shN.ndim == 3 and shN.shape[0] == total_splats and shN.shape[2] == 3
509
+ ), f"shN must be of shape (N, K, 3), got {shN.shape}"
510
+
511
+ # Reshape spherical harmonics
512
+ sh0 = sh0.squeeze(1) # Shape (N, 3)
513
+ shN = shN.permute(0, 2, 1).reshape(means.shape[0], -1) # Shape (N, K * 3)
514
+
515
+ # Check for NaN or Inf values
516
+ invalid_mask = (
517
+ torch.isnan(means).any(dim=1)
518
+ | torch.isinf(means).any(dim=1)
519
+ | torch.isnan(scales).any(dim=1)
520
+ | torch.isinf(scales).any(dim=1)
521
+ | torch.isnan(quats).any(dim=1)
522
+ | torch.isinf(quats).any(dim=1)
523
+ | torch.isnan(opacities).any(dim=0)
524
+ | torch.isinf(opacities).any(dim=0)
525
+ | torch.isnan(sh0).any(dim=1)
526
+ | torch.isinf(sh0).any(dim=1)
527
+ | torch.isnan(shN).any(dim=1)
528
+ | torch.isinf(shN).any(dim=1)
529
+ )
530
+
531
+ # Filter out invalid entries
532
+ valid_mask = ~invalid_mask
533
+ means = means[valid_mask]
534
+ scales = scales[valid_mask]
535
+ quats = quats[valid_mask]
536
+ opacities = opacities[valid_mask]
537
+ sh0 = sh0[valid_mask]
538
+ shN = shN[valid_mask]
539
+
540
+ if format == "ply":
541
+ data = splat2ply_bytes(means, scales, quats, opacities, sh0, shN)
542
+ elif format == "splat":
543
+ data = splat2splat_bytes(means, scales, quats, opacities, sh0)
544
+ elif format == "ply_compressed":
545
+ data = splat2ply_bytes_compressed(means, scales, quats, opacities, sh0, shN)
546
+ else:
547
+ raise ValueError(f"Unsupported format: {format}")
548
+
549
+ if save_to:
550
+ with open(save_to, "wb") as binary_file:
551
+ binary_file.write(data)
552
+
553
+ return data
src/post_opt/gsplat_viewer.py ADDED
@@ -0,0 +1,249 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import viser
2
+ from pathlib import Path
3
+ from typing import Literal
4
+ from typing import Tuple, Callable
5
+ from nerfview import Viewer, RenderTabState
6
+
7
+
8
+ class GsplatRenderTabState(RenderTabState):
9
+ # non-controlable parameters
10
+ total_gs_count: int = 0
11
+ rendered_gs_count: int = 0
12
+
13
+ # controlable parameters
14
+ max_sh_degree: int = 5
15
+ near_plane: float = 1e-2
16
+ far_plane: float = 1e2
17
+ radius_clip: float = 0.0
18
+ eps2d: float = 0.3
19
+ backgrounds: Tuple[float, float, float] = (0.0, 0.0, 0.0)
20
+ render_mode: Literal[
21
+ "rgb", "depth(accumulated)", "depth(expected)", "alpha"
22
+ ] = "rgb"
23
+ normalize_nearfar: bool = False
24
+ inverse: bool = True
25
+ colormap: Literal[
26
+ "turbo", "viridis", "magma", "inferno", "cividis", "gray"
27
+ ] = "turbo"
28
+ rasterize_mode: Literal["classic", "antialiased"] = "classic"
29
+ camera_model: Literal["pinhole", "ortho", "fisheye"] = "pinhole"
30
+
31
+
32
+ class GsplatViewer(Viewer):
33
+ """
34
+ Viewer for gsplat.
35
+ """
36
+
37
+ def __init__(
38
+ self,
39
+ server: viser.ViserServer,
40
+ render_fn: Callable,
41
+ output_dir: Path,
42
+ mode: Literal["rendering", "training"] = "rendering",
43
+ ):
44
+ super().__init__(server, render_fn, output_dir, mode)
45
+ server.gui.set_panel_label("gsplat viewer")
46
+
47
+ def _init_rendering_tab(self):
48
+ self.render_tab_state = GsplatRenderTabState()
49
+ self._rendering_tab_handles = {}
50
+ self._rendering_folder = self.server.gui.add_folder("Rendering")
51
+
52
+ def _populate_rendering_tab(self):
53
+ server = self.server
54
+ with self._rendering_folder:
55
+ with server.gui.add_folder("Gsplat"):
56
+ total_gs_count_number = server.gui.add_number(
57
+ "Total",
58
+ initial_value=self.render_tab_state.total_gs_count,
59
+ disabled=True,
60
+ hint="Total number of splats in the scene.",
61
+ )
62
+ rendered_gs_count_number = server.gui.add_number(
63
+ "Rendered",
64
+ initial_value=self.render_tab_state.rendered_gs_count,
65
+ disabled=True,
66
+ hint="Number of splats rendered.",
67
+ )
68
+
69
+ max_sh_degree_number = server.gui.add_number(
70
+ "Max SH",
71
+ initial_value=self.render_tab_state.max_sh_degree,
72
+ min=0,
73
+ max=5,
74
+ step=1,
75
+ hint="Maximum SH degree used",
76
+ )
77
+
78
+ @max_sh_degree_number.on_update
79
+ def _(_) -> None:
80
+ self.render_tab_state.max_sh_degree = int(
81
+ max_sh_degree_number.value
82
+ )
83
+ self.rerender(_)
84
+
85
+ near_far_plane_vec2 = server.gui.add_vector2(
86
+ "Near/Far",
87
+ initial_value=(
88
+ self.render_tab_state.near_plane,
89
+ self.render_tab_state.far_plane,
90
+ ),
91
+ min=(1e-3, 1e1),
92
+ max=(1e1, 1e2),
93
+ step=1e-3,
94
+ hint="Near and far plane for rendering.",
95
+ )
96
+
97
+ @near_far_plane_vec2.on_update
98
+ def _(_) -> None:
99
+ self.render_tab_state.near_plane = near_far_plane_vec2.value[0]
100
+ self.render_tab_state.far_plane = near_far_plane_vec2.value[1]
101
+ self.rerender(_)
102
+
103
+ radius_clip_slider = server.gui.add_number(
104
+ "Radius Clip",
105
+ initial_value=self.render_tab_state.radius_clip,
106
+ min=0.0,
107
+ max=100.0,
108
+ step=1.0,
109
+ hint="2D radius clip for rendering.",
110
+ )
111
+
112
+ @radius_clip_slider.on_update
113
+ def _(_) -> None:
114
+ self.render_tab_state.radius_clip = radius_clip_slider.value
115
+ self.rerender(_)
116
+
117
+ eps2d_slider = server.gui.add_number(
118
+ "2D Epsilon",
119
+ initial_value=self.render_tab_state.eps2d,
120
+ min=0.0,
121
+ max=1.0,
122
+ step=0.01,
123
+ hint="Epsilon added to the egienvalues of projected 2D covariance matrices.",
124
+ )
125
+
126
+ @eps2d_slider.on_update
127
+ def _(_) -> None:
128
+ self.render_tab_state.eps2d = eps2d_slider.value
129
+ self.rerender(_)
130
+
131
+ backgrounds_slider = server.gui.add_rgb(
132
+ "Background",
133
+ initial_value=self.render_tab_state.backgrounds,
134
+ hint="Background color for rendering.",
135
+ )
136
+
137
+ @backgrounds_slider.on_update
138
+ def _(_) -> None:
139
+ self.render_tab_state.backgrounds = backgrounds_slider.value
140
+ self.rerender(_)
141
+
142
+ render_mode_dropdown = server.gui.add_dropdown(
143
+ "Render Mode",
144
+ ("rgb", "depth(accumulated)", "depth(expected)", "alpha"),
145
+ initial_value=self.render_tab_state.render_mode,
146
+ hint="Render mode to use.",
147
+ )
148
+
149
+ @render_mode_dropdown.on_update
150
+ def _(_) -> None:
151
+ if "depth" in render_mode_dropdown.value:
152
+ normalize_nearfar_checkbox.disabled = False
153
+ else:
154
+ normalize_nearfar_checkbox.disabled = True
155
+ if render_mode_dropdown.value == "rgb":
156
+ inverse_checkbox.disabled = True
157
+ else:
158
+ inverse_checkbox.disabled = False
159
+ self.render_tab_state.render_mode = render_mode_dropdown.value
160
+ self.rerender(_)
161
+
162
+ normalize_nearfar_checkbox = server.gui.add_checkbox(
163
+ "Normalize Near/Far",
164
+ initial_value=self.render_tab_state.normalize_nearfar,
165
+ disabled=True,
166
+ hint="Normalize depth with near/far plane.",
167
+ )
168
+
169
+ @normalize_nearfar_checkbox.on_update
170
+ def _(_) -> None:
171
+ self.render_tab_state.normalize_nearfar = (
172
+ normalize_nearfar_checkbox.value
173
+ )
174
+ self.rerender(_)
175
+
176
+ inverse_checkbox = server.gui.add_checkbox(
177
+ "Inverse",
178
+ initial_value=self.render_tab_state.inverse,
179
+ disabled=True,
180
+ hint="Inverse the depth.",
181
+ )
182
+
183
+ @inverse_checkbox.on_update
184
+ def _(_) -> None:
185
+ self.render_tab_state.inverse = inverse_checkbox.value
186
+ self.rerender(_)
187
+
188
+ colormap_dropdown = server.gui.add_dropdown(
189
+ "Colormap",
190
+ ("turbo", "viridis", "magma", "inferno", "cividis", "gray"),
191
+ initial_value=self.render_tab_state.colormap,
192
+ hint="Colormap used for rendering depth/alpha.",
193
+ )
194
+
195
+ @colormap_dropdown.on_update
196
+ def _(_) -> None:
197
+ self.render_tab_state.colormap = colormap_dropdown.value
198
+ self.rerender(_)
199
+
200
+ rasterize_mode_dropdown = server.gui.add_dropdown(
201
+ "Anti-Aliasing",
202
+ ("classic", "antialiased"),
203
+ initial_value=self.render_tab_state.rasterize_mode,
204
+ hint="Whether to use classic or antialiased rasterization.",
205
+ )
206
+
207
+ @rasterize_mode_dropdown.on_update
208
+ def _(_) -> None:
209
+ self.render_tab_state.rasterize_mode = rasterize_mode_dropdown.value
210
+ self.rerender(_)
211
+
212
+ camera_model_dropdown = server.gui.add_dropdown(
213
+ "Camera",
214
+ ("pinhole", "ortho", "fisheye"),
215
+ initial_value=self.render_tab_state.camera_model,
216
+ hint="Camera model used for rendering.",
217
+ )
218
+
219
+ @camera_model_dropdown.on_update
220
+ def _(_) -> None:
221
+ self.render_tab_state.camera_model = camera_model_dropdown.value
222
+ self.rerender(_)
223
+
224
+ self._rendering_tab_handles.update(
225
+ {
226
+ "total_gs_count_number": total_gs_count_number,
227
+ "rendered_gs_count_number": rendered_gs_count_number,
228
+ "near_far_plane_vec2": near_far_plane_vec2,
229
+ "radius_clip_slider": radius_clip_slider,
230
+ "eps2d_slider": eps2d_slider,
231
+ "backgrounds_slider": backgrounds_slider,
232
+ "render_mode_dropdown": render_mode_dropdown,
233
+ "normalize_nearfar_checkbox": normalize_nearfar_checkbox,
234
+ "inverse_checkbox": inverse_checkbox,
235
+ "colormap_dropdown": colormap_dropdown,
236
+ "rasterize_mode_dropdown": rasterize_mode_dropdown,
237
+ "camera_model_dropdown": camera_model_dropdown,
238
+ }
239
+ )
240
+ super()._populate_rendering_tab()
241
+
242
+ def _after_render(self):
243
+ # Update the GUI elements with current values
244
+ self._rendering_tab_handles[
245
+ "total_gs_count_number"
246
+ ].value = self.render_tab_state.total_gs_count
247
+ self._rendering_tab_handles[
248
+ "rendered_gs_count_number"
249
+ ].value = self.render_tab_state.rendered_gs_count
src/post_opt/image_fitting.py ADDED
@@ -0,0 +1,189 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+ import os
3
+ import time
4
+ from pathlib import Path
5
+ from typing import Literal, Optional
6
+
7
+ import numpy as np
8
+ import torch
9
+ import tyro
10
+ from PIL import Image
11
+ from torch import Tensor, optim
12
+
13
+ from gsplat import rasterization, rasterization_2dgs
14
+
15
+
16
+ class SimpleTrainer:
17
+ """Trains random gaussians to fit an image."""
18
+
19
+ def __init__(
20
+ self,
21
+ gt_image: Tensor,
22
+ num_points: int = 2000,
23
+ ):
24
+ self.device = torch.device("cuda:0")
25
+ self.gt_image = gt_image.to(device=self.device)
26
+ self.num_points = num_points
27
+
28
+ fov_x = math.pi / 2.0
29
+ self.H, self.W = gt_image.shape[0], gt_image.shape[1]
30
+ self.focal = 0.5 * float(self.W) / math.tan(0.5 * fov_x)
31
+ self.img_size = torch.tensor([self.W, self.H, 1], device=self.device)
32
+
33
+ self._init_gaussians()
34
+
35
+ def _init_gaussians(self):
36
+ """Random gaussians"""
37
+ bd = 2
38
+
39
+ self.means = bd * (torch.rand(self.num_points, 3, device=self.device) - 0.5)
40
+ self.scales = torch.rand(self.num_points, 3, device=self.device)
41
+ d = 3
42
+ self.rgbs = torch.rand(self.num_points, d, device=self.device)
43
+
44
+ u = torch.rand(self.num_points, 1, device=self.device)
45
+ v = torch.rand(self.num_points, 1, device=self.device)
46
+ w = torch.rand(self.num_points, 1, device=self.device)
47
+
48
+ self.quats = torch.cat(
49
+ [
50
+ torch.sqrt(1.0 - u) * torch.sin(2.0 * math.pi * v),
51
+ torch.sqrt(1.0 - u) * torch.cos(2.0 * math.pi * v),
52
+ torch.sqrt(u) * torch.sin(2.0 * math.pi * w),
53
+ torch.sqrt(u) * torch.cos(2.0 * math.pi * w),
54
+ ],
55
+ -1,
56
+ )
57
+ self.opacities = torch.ones((self.num_points), device=self.device)
58
+
59
+ self.viewmat = torch.tensor(
60
+ [
61
+ [1.0, 0.0, 0.0, 0.0],
62
+ [0.0, 1.0, 0.0, 0.0],
63
+ [0.0, 0.0, 1.0, 8.0],
64
+ [0.0, 0.0, 0.0, 1.0],
65
+ ],
66
+ device=self.device,
67
+ )
68
+ self.background = torch.zeros(d, device=self.device)
69
+
70
+ self.means.requires_grad = True
71
+ self.scales.requires_grad = True
72
+ self.quats.requires_grad = True
73
+ self.rgbs.requires_grad = True
74
+ self.opacities.requires_grad = True
75
+ self.viewmat.requires_grad = False
76
+
77
+ def train(
78
+ self,
79
+ iterations: int = 1000,
80
+ lr: float = 0.01,
81
+ save_imgs: bool = False,
82
+ model_type: Literal["3dgs", "2dgs"] = "3dgs",
83
+ ):
84
+ optimizer = optim.Adam(
85
+ [self.rgbs, self.means, self.scales, self.opacities, self.quats], lr
86
+ )
87
+ mse_loss = torch.nn.MSELoss()
88
+ frames = []
89
+ times = [0] * 2 # rasterization, backward
90
+ K = torch.tensor(
91
+ [
92
+ [self.focal, 0, self.W / 2],
93
+ [0, self.focal, self.H / 2],
94
+ [0, 0, 1],
95
+ ],
96
+ device=self.device,
97
+ )
98
+
99
+ if model_type == "3dgs":
100
+ rasterize_fnc = rasterization
101
+ elif model_type == "2dgs":
102
+ rasterize_fnc = rasterization_2dgs
103
+
104
+ for iter in range(iterations):
105
+ start = time.time()
106
+
107
+ renders = rasterize_fnc(
108
+ self.means,
109
+ self.quats / self.quats.norm(dim=-1, keepdim=True),
110
+ self.scales,
111
+ torch.sigmoid(self.opacities),
112
+ torch.sigmoid(self.rgbs),
113
+ self.viewmat[None],
114
+ K[None],
115
+ self.W,
116
+ self.H,
117
+ packed=False,
118
+ )[0]
119
+ out_img = renders[0]
120
+ torch.cuda.synchronize()
121
+ times[0] += time.time() - start
122
+ loss = mse_loss(out_img, self.gt_image)
123
+ optimizer.zero_grad()
124
+ start = time.time()
125
+ loss.backward()
126
+ torch.cuda.synchronize()
127
+ times[1] += time.time() - start
128
+ optimizer.step()
129
+ print(f"Iteration {iter + 1}/{iterations}, Loss: {loss.item()}")
130
+
131
+ if save_imgs and iter % 5 == 0:
132
+ frames.append((out_img.detach().cpu().numpy() * 255).astype(np.uint8))
133
+ if save_imgs:
134
+ # save them as a gif with PIL
135
+ frames = [Image.fromarray(frame) for frame in frames]
136
+ out_dir = os.path.join(os.getcwd(), "results")
137
+ os.makedirs(out_dir, exist_ok=True)
138
+ frames[0].save(
139
+ f"{out_dir}/training.gif",
140
+ save_all=True,
141
+ append_images=frames[1:],
142
+ optimize=False,
143
+ duration=5,
144
+ loop=0,
145
+ )
146
+ print(f"Total(s):\nRasterization: {times[0]:.3f}, Backward: {times[1]:.3f}")
147
+ print(
148
+ f"Per step(s):\nRasterization: {times[0]/iterations:.5f}, Backward: {times[1]/iterations:.5f}"
149
+ )
150
+
151
+
152
+ def image_path_to_tensor(image_path: Path):
153
+ import torchvision.transforms as transforms
154
+
155
+ img = Image.open(image_path)
156
+ transform = transforms.ToTensor()
157
+ img_tensor = transform(img).permute(1, 2, 0)[..., :3]
158
+ return img_tensor
159
+
160
+
161
+ def main(
162
+ height: int = 256,
163
+ width: int = 256,
164
+ num_points: int = 100000,
165
+ save_imgs: bool = True,
166
+ img_path: Optional[Path] = None,
167
+ iterations: int = 1000,
168
+ lr: float = 0.01,
169
+ model_type: Literal["3dgs", "2dgs"] = "3dgs",
170
+ ) -> None:
171
+ if img_path:
172
+ gt_image = image_path_to_tensor(img_path)
173
+ else:
174
+ gt_image = torch.ones((height, width, 3)) * 1.0
175
+ # make top left and bottom right red, blue
176
+ gt_image[: height // 2, : width // 2, :] = torch.tensor([1.0, 0.0, 0.0])
177
+ gt_image[height // 2 :, width // 2 :, :] = torch.tensor([0.0, 0.0, 1.0])
178
+
179
+ trainer = SimpleTrainer(gt_image=gt_image, num_points=num_points)
180
+ trainer.train(
181
+ iterations=iterations,
182
+ lr=lr,
183
+ save_imgs=save_imgs,
184
+ model_type=model_type,
185
+ )
186
+
187
+
188
+ if __name__ == "__main__":
189
+ tyro.cli(main)
src/post_opt/lib_bilagrid.py ADDED
@@ -0,0 +1,573 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # # Copyright 2024 Yuehao Wang (https://github.com/yuehaowang). This part of code is borrowed form ["Bilateral Guided Radiance Field Processing"](https://bilarfpro.github.io/).
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """
16
+ This is a standalone PyTorch implementation of 3D bilateral grid and CP-decomposed 4D bilateral grid.
17
+ To use this module, you can download the "lib_bilagrid.py" file and simply put it in your project directory.
18
+
19
+ For the details, please check our research project: ["Bilateral Guided Radiance Field Processing"](https://bilarfpro.github.io/).
20
+
21
+ #### Dependencies
22
+
23
+ In addition to PyTorch and Numpy, please install [tensorly](https://github.com/tensorly/tensorly).
24
+ We have tested this module on Python 3.9.18, PyTorch 2.0.1 (CUDA 11), tensorly 0.8.1, and Numpy 1.25.2.
25
+
26
+ #### Overview
27
+
28
+ - For bilateral guided training, you need to construct a `BilateralGrid` instance, which can hold multiple bilateral grids
29
+ for input views. Then, use `slice` function to obtain transformed RGB output and the corresponding affine transformations.
30
+
31
+ - For bilateral guided finishing, you need to instantiate a `BilateralGridCP4D` object and use `slice4d`.
32
+
33
+ #### Examples
34
+
35
+ - Bilateral grid for approximating ISP:
36
+ <a target="_blank" href="https://colab.research.google.com/drive/1tx2qKtsHH9deDDnParMWrChcsa9i7Prr?usp=sharing">
37
+ <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>
38
+
39
+ - Low-rank 4D bilateral grid for MR enhancement:
40
+ <a target="_blank" href="https://colab.research.google.com/drive/17YOjQqgWFT3QI1vysOIH494rMYtt_mHL?usp=sharing">
41
+ <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>
42
+
43
+
44
+ Below is the API reference.
45
+
46
+ """
47
+
48
+ import tensorly as tl
49
+ import torch
50
+ import torch.nn.functional as F
51
+ from torch import nn
52
+
53
+ tl.set_backend("pytorch")
54
+
55
+
56
+ def color_correct(
57
+ img: torch.Tensor, ref: torch.Tensor, num_iters: int = 5, eps: float = 0.5 / 255
58
+ ) -> torch.Tensor:
59
+ """
60
+ Warp `img` to match the colors in `ref_img` using iterative color matching.
61
+
62
+ This function performs color correction by warping the colors of the input image
63
+ to match those of a reference image. It uses a least squares method to find a
64
+ transformation that maps the input image's colors to the reference image's colors.
65
+
66
+ The algorithm iteratively solves a system of linear equations, updating the set of
67
+ unsaturated pixels in each iteration. This approach helps handle non-linear color
68
+ transformations and reduces the impact of clipping.
69
+
70
+ Args:
71
+ img (torch.Tensor): Input image to be color corrected. Shape: [..., num_channels]
72
+ ref (torch.Tensor): Reference image to match colors. Shape: [..., num_channels]
73
+ num_iters (int, optional): Number of iterations for the color matching process.
74
+ Default is 5.
75
+ eps (float, optional): Small value to determine the range of unclipped pixels.
76
+ Default is 0.5 / 255.
77
+
78
+ Returns:
79
+ torch.Tensor: Color corrected image with the same shape as the input image.
80
+
81
+ Note:
82
+ - Both input and reference images should be in the range [0, 1].
83
+ - The function works with any number of channels, but typically used with 3 (RGB).
84
+ """
85
+ if img.shape[-1] != ref.shape[-1]:
86
+ raise ValueError(
87
+ f"img's {img.shape[-1]} and ref's {ref.shape[-1]} channels must match"
88
+ )
89
+ num_channels = img.shape[-1]
90
+ img_mat = img.reshape([-1, num_channels])
91
+ ref_mat = ref.reshape([-1, num_channels])
92
+
93
+ def is_unclipped(z):
94
+ return (z >= eps) & (z <= 1 - eps) # z \in [eps, 1-eps].
95
+
96
+ mask0 = is_unclipped(img_mat)
97
+ # Because the set of saturated pixels may change after solving for a
98
+ # transformation, we repeatedly solve a system `num_iters` times and update
99
+ # our estimate of which pixels are saturated.
100
+ for _ in range(num_iters):
101
+ # Construct the left hand side of a linear system that contains a quadratic
102
+ # expansion of each pixel of `img`.
103
+ a_mat = []
104
+ for c in range(num_channels):
105
+ a_mat.append(img_mat[:, c : (c + 1)] * img_mat[:, c:]) # Quadratic term.
106
+ a_mat.append(img_mat) # Linear term.
107
+ a_mat.append(torch.ones_like(img_mat[:, :1])) # Bias term.
108
+ a_mat = torch.cat(a_mat, dim=-1)
109
+ warp = []
110
+ for c in range(num_channels):
111
+ # Construct the right hand side of a linear system containing each color
112
+ # of `ref`.
113
+ b = ref_mat[:, c]
114
+ # Ignore rows of the linear system that were saturated in the input or are
115
+ # saturated in the current corrected color estimate.
116
+ mask = mask0[:, c] & is_unclipped(img_mat[:, c]) & is_unclipped(b)
117
+ ma_mat = torch.where(mask[:, None], a_mat, torch.zeros_like(a_mat))
118
+ mb = torch.where(mask, b, torch.zeros_like(b))
119
+ w = torch.linalg.lstsq(ma_mat, mb, rcond=-1)[0]
120
+ assert torch.all(torch.isfinite(w))
121
+ warp.append(w)
122
+ warp = torch.stack(warp, dim=-1)
123
+ # Apply the warp to update img_mat.
124
+ img_mat = torch.clip(torch.matmul(a_mat, warp), 0, 1)
125
+ corrected_img = torch.reshape(img_mat, img.shape)
126
+ return corrected_img
127
+
128
+
129
+ def bilateral_grid_tv_loss(model, config):
130
+ """Computes total variations of bilateral grids."""
131
+ total_loss = 0.0
132
+
133
+ for bil_grids in model.bil_grids:
134
+ total_loss += config.bilgrid_tv_loss_mult * total_variation_loss(
135
+ bil_grids.grids
136
+ )
137
+
138
+ return total_loss
139
+
140
+
141
+ def color_affine_transform(affine_mats, rgb):
142
+ """Applies color affine transformations.
143
+
144
+ Args:
145
+ affine_mats (torch.Tensor): Affine transformation matrices. Supported shape: $(..., 3, 4)$.
146
+ rgb (torch.Tensor): Input RGB values. Supported shape: $(..., 3)$.
147
+
148
+ Returns:
149
+ Output transformed colors of shape $(..., 3)$.
150
+ """
151
+ return (
152
+ torch.matmul(affine_mats[..., :3], rgb.unsqueeze(-1)).squeeze(-1)
153
+ + affine_mats[..., 3]
154
+ )
155
+
156
+
157
+ def _num_tensor_elems(t):
158
+ return max(torch.prod(torch.tensor(t.size()[1:]).float()).item(), 1.0)
159
+
160
+
161
+ def total_variation_loss(x): # noqa: F811
162
+ """Returns total variation on multi-dimensional tensors.
163
+
164
+ Args:
165
+ x (torch.Tensor): The input tensor with shape $(B, C, ...)$, where $B$ is the batch size and $C$ is the channel size.
166
+ """
167
+ batch_size = x.shape[0]
168
+ tv = 0
169
+ for i in range(2, len(x.shape)):
170
+ n_res = x.shape[i]
171
+ idx1 = torch.arange(1, n_res, device=x.device)
172
+ idx2 = torch.arange(0, n_res - 1, device=x.device)
173
+ x1 = x.index_select(i, idx1)
174
+ x2 = x.index_select(i, idx2)
175
+ count = _num_tensor_elems(x1)
176
+ tv += torch.pow((x1 - x2), 2).sum() / count
177
+ return tv / batch_size
178
+
179
+
180
+ def slice(bil_grids, xy, rgb, grid_idx):
181
+ """Slices a batch of 3D bilateral grids by pixel coordinates `xy` and gray-scale guidances of pixel colors `rgb`.
182
+
183
+ Supports 2-D, 3-D, and 4-D input shapes. The first dimension of the input is the batch size
184
+ and the last dimension is 2 for `xy`, 3 for `rgb`, and 1 for `grid_idx`.
185
+
186
+ The return value is a dictionary containing the affine transformations `affine_mats` sliced from bilateral grids and
187
+ the output color `rgb_out` after applying the afffine transformations.
188
+
189
+ In the 2-D input case, `xy` is a $(N, 2)$ tensor, `rgb` is a $(N, 3)$ tensor, and `grid_idx` is a $(N, 1)$ tensor.
190
+ Then `affine_mats[i]` can be obtained via slicing the bilateral grid indexed at `grid_idx[i]` by `xy[i, :]` and `rgb2gray(rgb[i, :])`.
191
+ For 3-D and 4-D input cases, the behavior of indexing bilateral grids and coordinates is the same with the 2-D case.
192
+
193
+ .. note::
194
+ This function can be regarded as a wrapper of `color_affine_transform` and `BilateralGrid` with a slight performance improvement.
195
+ When `grid_idx` contains a unique index, only a single bilateral grid will used during the slicing. In this case, this function will not
196
+ perform tensor indexing to avoid data copy and extra memory
197
+ (see [this](https://discuss.pytorch.org/t/does-indexing-a-tensor-return-a-copy-of-it/164905)).
198
+
199
+ Args:
200
+ bil_grids (`BilateralGrid`): An instance of $N$ bilateral grids.
201
+ xy (torch.Tensor): The x-y coordinates of shape $(..., 2)$ in the range of $[0,1]$.
202
+ rgb (torch.Tensor): The RGB values of shape $(..., 3)$ for computing the guidance coordinates, ranging in $[0,1]$.
203
+ grid_idx (torch.Tensor): The indices of bilateral grids for each slicing. Shape: $(..., 1)$.
204
+
205
+ Returns:
206
+ A dictionary with keys and values as follows:
207
+ ```
208
+ {
209
+ "rgb": Transformed RGB colors. Shape: (..., 3),
210
+ "rgb_affine_mats": The sliced affine transformation matrices from bilateral grids. Shape: (..., 3, 4)
211
+ }
212
+ ```
213
+ """
214
+
215
+ sh_ = rgb.shape
216
+
217
+ grid_idx_unique = torch.unique(grid_idx)
218
+ if len(grid_idx_unique) == 1:
219
+ # All pixels are from a single view.
220
+ grid_idx = grid_idx_unique # (1,)
221
+ xy = xy.unsqueeze(0) # (1, ..., 2)
222
+ rgb = rgb.unsqueeze(0) # (1, ..., 3)
223
+ else:
224
+ # Pixels are randomly sampled from different views.
225
+ if len(grid_idx.shape) == 4:
226
+ grid_idx = grid_idx[:, 0, 0, 0] # (chunk_size,)
227
+ elif len(grid_idx.shape) == 3:
228
+ grid_idx = grid_idx[:, 0, 0] # (chunk_size,)
229
+ elif len(grid_idx.shape) == 2:
230
+ grid_idx = grid_idx[:, 0] # (chunk_size,)
231
+ else:
232
+ raise ValueError(
233
+ "The input to bilateral grid slicing is not supported yet."
234
+ )
235
+
236
+ affine_mats = bil_grids(xy, rgb, grid_idx)
237
+ rgb = color_affine_transform(affine_mats, rgb)
238
+
239
+ return {
240
+ "rgb": rgb.reshape(*sh_),
241
+ "rgb_affine_mats": affine_mats.reshape(
242
+ *sh_[:-1], affine_mats.shape[-2], affine_mats.shape[-1]
243
+ ),
244
+ }
245
+
246
+
247
+ class BilateralGrid(nn.Module):
248
+ """Class for 3D bilateral grids.
249
+
250
+ Holds one or more than one bilateral grids.
251
+ """
252
+
253
+ def __init__(self, num, grid_X=16, grid_Y=16, grid_W=8):
254
+ """
255
+ Args:
256
+ num (int): The number of bilateral grids (i.e., the number of views).
257
+ grid_X (int): Defines grid width $W$.
258
+ grid_Y (int): Defines grid height $H$.
259
+ grid_W (int): Defines grid guidance dimension $L$.
260
+ """
261
+ super(BilateralGrid, self).__init__()
262
+
263
+ self.grid_width = grid_X
264
+ """Grid width. Type: int."""
265
+ self.grid_height = grid_Y
266
+ """Grid height. Type: int."""
267
+ self.grid_guidance = grid_W
268
+ """Grid guidance dimension. Type: int."""
269
+
270
+ # Initialize grids.
271
+ grid = self._init_identity_grid()
272
+ self.grids = nn.Parameter(grid.tile(num, 1, 1, 1, 1)) # (N, 12, L, H, W)
273
+ """ A 5-D tensor of shape $(N, 12, L, H, W)$."""
274
+
275
+ # Weights of BT601 RGB-to-gray.
276
+ self.register_buffer("rgb2gray_weight", torch.Tensor([[0.299, 0.587, 0.114]]))
277
+ self.rgb2gray = lambda rgb: (rgb @ self.rgb2gray_weight.T) * 2.0 - 1.0
278
+ """ A function that converts RGB to gray-scale guidance in $[-1, 1]$."""
279
+
280
+ def _init_identity_grid(self):
281
+ grid = torch.tensor(
282
+ [
283
+ 1.0,
284
+ 0,
285
+ 0,
286
+ 0,
287
+ 0,
288
+ 1.0,
289
+ 0,
290
+ 0,
291
+ 0,
292
+ 0,
293
+ 1.0,
294
+ 0,
295
+ ]
296
+ ).float()
297
+ grid = grid.repeat(
298
+ [self.grid_guidance * self.grid_height * self.grid_width, 1]
299
+ ) # (L * H * W, 12)
300
+ grid = grid.reshape(
301
+ 1, self.grid_guidance, self.grid_height, self.grid_width, -1
302
+ ) # (1, L, H, W, 12)
303
+ grid = grid.permute(0, 4, 1, 2, 3) # (1, 12, L, H, W)
304
+ return grid
305
+
306
+ def tv_loss(self):
307
+ """Computes and returns total variation loss on the bilateral grids."""
308
+ return total_variation_loss(self.grids)
309
+
310
+ def forward(self, grid_xy, rgb, idx=None):
311
+ """Bilateral grid slicing. Supports 2-D, 3-D, 4-D, and 5-D input.
312
+ For the 2-D, 3-D, and 4-D cases, please refer to `slice`.
313
+ For the 5-D cases, `idx` will be unused and the first dimension of `xy` should be
314
+ equal to the number of bilateral grids. Then this function becomes PyTorch's
315
+ [`F.grid_sample`](https://pytorch.org/docs/stable/generated/torch.nn.functional.grid_sample.html).
316
+
317
+ Args:
318
+ grid_xy (torch.Tensor): The x-y coordinates in the range of $[0,1]$.
319
+ rgb (torch.Tensor): The RGB values in the range of $[0,1]$.
320
+ idx (torch.Tensor): The bilateral grid indices.
321
+
322
+ Returns:
323
+ Sliced affine matrices of shape $(..., 3, 4)$.
324
+ """
325
+
326
+ grids = self.grids
327
+ input_ndims = len(grid_xy.shape)
328
+ assert len(rgb.shape) == input_ndims
329
+
330
+ if input_ndims > 1 and input_ndims < 5:
331
+ # Convert input into 5D
332
+ for i in range(5 - input_ndims):
333
+ grid_xy = grid_xy.unsqueeze(1)
334
+ rgb = rgb.unsqueeze(1)
335
+ assert idx is not None
336
+ elif input_ndims != 5:
337
+ raise ValueError(
338
+ "Bilateral grid slicing only takes either 2D, 3D, 4D and 5D inputs"
339
+ )
340
+
341
+ grids = self.grids
342
+ if idx is not None:
343
+ grids = grids[idx]
344
+ assert grids.shape[0] == grid_xy.shape[0]
345
+
346
+ # Generate slicing coordinates.
347
+ grid_xy = (grid_xy - 0.5) * 2 # Rescale to [-1, 1].
348
+ grid_z = self.rgb2gray(rgb)
349
+
350
+ # print(grid_xy.shape, grid_z.shape)
351
+ # exit()
352
+ grid_xyz = torch.cat([grid_xy, grid_z], dim=-1) # (N, m, h, w, 3)
353
+
354
+ affine_mats = F.grid_sample(
355
+ grids, grid_xyz, mode="bilinear", align_corners=True, padding_mode="border"
356
+ ) # (N, 12, m, h, w)
357
+ affine_mats = affine_mats.permute(0, 2, 3, 4, 1) # (N, m, h, w, 12)
358
+ affine_mats = affine_mats.reshape(
359
+ *affine_mats.shape[:-1], 3, 4
360
+ ) # (N, m, h, w, 3, 4)
361
+
362
+ for _ in range(5 - input_ndims):
363
+ affine_mats = affine_mats.squeeze(1)
364
+
365
+ return affine_mats
366
+
367
+
368
+ def slice4d(bil_grid4d, xyz, rgb):
369
+ """Slices a 4D bilateral grid by point coordinates `xyz` and gray-scale guidances of radiance colors `rgb`.
370
+
371
+ Args:
372
+ bil_grid4d (`BilateralGridCP4D`): The input 4D bilateral grid.
373
+ xyz (torch.Tensor): The xyz coordinates with shape $(..., 3)$.
374
+ rgb (torch.Tensor): The RGB values with shape $(..., 3)$.
375
+
376
+ Returns:
377
+ A dictionary with keys and values as follows:
378
+ ```
379
+ {
380
+ "rgb": Transformed radiance RGB colors. Shape: (..., 3),
381
+ "rgb_affine_mats": The sliced affine transformation matrices from the 4D bilateral grid. Shape: (..., 3, 4)
382
+ }
383
+ ```
384
+ """
385
+
386
+ affine_mats = bil_grid4d(xyz, rgb)
387
+ rgb = color_affine_transform(affine_mats, rgb)
388
+
389
+ return {"rgb": rgb, "rgb_affine_mats": affine_mats}
390
+
391
+
392
+ class _ScaledTanh(nn.Module):
393
+ def __init__(self, s=2.0):
394
+ super().__init__()
395
+ self.scaler = s
396
+
397
+ def forward(self, x):
398
+ return torch.tanh(self.scaler * x)
399
+
400
+
401
+ class BilateralGridCP4D(nn.Module):
402
+ """Class for low-rank 4D bilateral grids."""
403
+
404
+ def __init__(
405
+ self,
406
+ grid_X=16,
407
+ grid_Y=16,
408
+ grid_Z=16,
409
+ grid_W=8,
410
+ rank=5,
411
+ learn_gray=True,
412
+ gray_mlp_width=8,
413
+ gray_mlp_depth=2,
414
+ init_noise_scale=1e-6,
415
+ bound=2.0,
416
+ ):
417
+ """
418
+ Args:
419
+ grid_X (int): Defines grid width.
420
+ grid_Y (int): Defines grid height.
421
+ grid_Z (int): Defines grid depth.
422
+ grid_W (int): Defines grid guidance dimension.
423
+ rank (int): Rank of the 4D bilateral grid.
424
+ learn_gray (bool): If True, an MLP will be learned to convert RGB colors to gray-scale guidances.
425
+ gray_mlp_width (int): The MLP width for learnable guidance.
426
+ gray_mlp_depth (int): The number of MLP layers for learnable guidance.
427
+ init_noise_scale (float): The noise scale of the initialized factors.
428
+ bound (float): The bound of the xyz coordinates.
429
+ """
430
+ super(BilateralGridCP4D, self).__init__()
431
+
432
+ self.grid_X = grid_X
433
+ """Grid width. Type: int."""
434
+ self.grid_Y = grid_Y
435
+ """Grid height. Type: int."""
436
+ self.grid_Z = grid_Z
437
+ """Grid depth. Type: int."""
438
+ self.grid_W = grid_W
439
+ """Grid guidance dimension. Type: int."""
440
+ self.rank = rank
441
+ """Rank of the 4D bilateral grid. Type: int."""
442
+ self.learn_gray = learn_gray
443
+ """Flags of learnable guidance is used. Type: bool."""
444
+ self.gray_mlp_width = gray_mlp_width
445
+ """The MLP width for learnable guidance. Type: int."""
446
+ self.gray_mlp_depth = gray_mlp_depth
447
+ """The MLP depth for learnable guidance. Type: int."""
448
+ self.init_noise_scale = init_noise_scale
449
+ """The noise scale of the initialized factors. Type: float."""
450
+ self.bound = bound
451
+ """The bound of the xyz coordinates. Type: float."""
452
+
453
+ self._init_cp_factors_parafac()
454
+
455
+ self.rgb2gray = None
456
+ """ A function that converts RGB to gray-scale guidances in $[-1, 1]$.
457
+ If `learn_gray` is True, this will be an MLP network."""
458
+
459
+ if self.learn_gray:
460
+
461
+ def rgb2gray_mlp_linear(layer):
462
+ return nn.Linear(
463
+ self.gray_mlp_width,
464
+ self.gray_mlp_width if layer < self.gray_mlp_depth - 1 else 1,
465
+ )
466
+
467
+ def rgb2gray_mlp_actfn(_):
468
+ return nn.ReLU(inplace=True)
469
+
470
+ self.rgb2gray = nn.Sequential(
471
+ *(
472
+ [nn.Linear(3, self.gray_mlp_width)]
473
+ + [
474
+ nn_module(layer)
475
+ for layer in range(1, self.gray_mlp_depth)
476
+ for nn_module in [rgb2gray_mlp_actfn, rgb2gray_mlp_linear]
477
+ ]
478
+ + [_ScaledTanh(2.0)]
479
+ )
480
+ )
481
+ else:
482
+ # Weights of BT601/BT470 RGB-to-gray.
483
+ self.register_buffer(
484
+ "rgb2gray_weight", torch.Tensor([[0.299, 0.587, 0.114]])
485
+ )
486
+ self.rgb2gray = lambda rgb: (rgb @ self.rgb2gray_weight.T) * 2.0 - 1.0
487
+
488
+ def _init_identity_grid(self):
489
+ grid = torch.tensor(
490
+ [
491
+ 1.0,
492
+ 0,
493
+ 0,
494
+ 0,
495
+ 0,
496
+ 1.0,
497
+ 0,
498
+ 0,
499
+ 0,
500
+ 0,
501
+ 1.0,
502
+ 0,
503
+ ]
504
+ ).float()
505
+ grid = grid.repeat([self.grid_W * self.grid_Z * self.grid_Y * self.grid_X, 1])
506
+ grid = grid.reshape(self.grid_W, self.grid_Z, self.grid_Y, self.grid_X, -1)
507
+ grid = grid.permute(4, 0, 1, 2, 3) # (12, grid_W, grid_Z, grid_Y, grid_X)
508
+ return grid
509
+
510
+ def _init_cp_factors_parafac(self):
511
+ # Initialize identity grids.
512
+ init_grids = self._init_identity_grid()
513
+ # Random noises are added to avoid singularity.
514
+ init_grids = torch.randn_like(init_grids) * self.init_noise_scale + init_grids
515
+ from tensorly.decomposition import parafac
516
+
517
+ # Initialize grid CP factors
518
+ _, facs = parafac(init_grids.clone().detach(), rank=self.rank)
519
+
520
+ self.num_facs = len(facs)
521
+
522
+ self.fac_0 = nn.Linear(facs[0].shape[0], facs[0].shape[1], bias=False)
523
+ self.fac_0.weight = nn.Parameter(facs[0]) # (12, rank)
524
+
525
+ for i in range(1, self.num_facs):
526
+ fac = facs[i].T # (rank, grid_size)
527
+ fac = fac.view(1, fac.shape[0], fac.shape[1], 1) # (1, rank, grid_size, 1)
528
+ self.register_buffer(f"fac_{i}_init", fac)
529
+
530
+ fac_resid = torch.zeros_like(fac)
531
+ self.register_parameter(f"fac_{i}", nn.Parameter(fac_resid))
532
+
533
+ def tv_loss(self):
534
+ """Computes and returns total variation loss on the factors of the low-rank 4D bilateral grids."""
535
+
536
+ total_loss = 0
537
+ for i in range(1, self.num_facs):
538
+ fac = self.get_parameter(f"fac_{i}")
539
+ total_loss += total_variation_loss(fac)
540
+
541
+ return total_loss
542
+
543
+ def forward(self, xyz, rgb):
544
+ """Low-rank 4D bilateral grid slicing.
545
+
546
+ Args:
547
+ xyz (torch.Tensor): The xyz coordinates with shape $(..., 3)$.
548
+ rgb (torch.Tensor): The corresponding RGB values with shape $(..., 3)$.
549
+
550
+ Returns:
551
+ Sliced affine matrices with shape $(..., 3, 4)$.
552
+ """
553
+ sh_ = xyz.shape
554
+ xyz = xyz.reshape(-1, 3) # flatten (N, 3)
555
+ rgb = rgb.reshape(-1, 3) # flatten (N, 3)
556
+
557
+ xyz = xyz / self.bound
558
+ assert self.rgb2gray is not None
559
+ gray = self.rgb2gray(rgb)
560
+ xyzw = torch.cat([xyz, gray], dim=-1) # (N, 4)
561
+ xyzw = xyzw.transpose(0, 1) # (4, N)
562
+ coords = torch.stack([torch.zeros_like(xyzw), xyzw], dim=-1) # (4, N, 2)
563
+ coords = coords.unsqueeze(1) # (4, 1, N, 2)
564
+
565
+ coef = 1.0
566
+ for i in range(1, self.num_facs):
567
+ fac = self.get_parameter(f"fac_{i}") + self.get_buffer(f"fac_{i}_init")
568
+ coef = coef * F.grid_sample(
569
+ fac, coords[[i - 1]], align_corners=True, padding_mode="border"
570
+ ) # [1, rank, 1, N]
571
+ coef = coef.squeeze([0, 2]).transpose(0, 1) # (N, rank) #type: ignore
572
+ mat = self.fac_0(coef)
573
+ return mat.reshape(*sh_[:-1], 3, 4)
src/post_opt/simple_trainer.py ADDED
@@ -0,0 +1,1433 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import math
3
+ import os
4
+ import time
5
+ from collections import defaultdict
6
+ from dataclasses import dataclass, field
7
+ from pathlib import Path
8
+ from typing import Dict, List, Optional, Tuple, Union
9
+
10
+ import imageio
11
+ import matplotlib
12
+ import torchvision
13
+ import numpy as np
14
+ import torch
15
+ import torch.nn.functional as F
16
+ import tqdm
17
+ import tyro
18
+ import viser
19
+ import yaml
20
+ import torchvision
21
+ import sys
22
+ from plyfile import PlyData, PlyElement
23
+
24
+ sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
25
+
26
+ from src.model.encoder.vggt.utils.pose_enc import pose_encoding_to_extri_intri
27
+ from src.model.types import Gaussians
28
+ from src.post_opt.datasets.colmap import Dataset, Parser
29
+ from src.post_opt.datasets.traj import (
30
+ generate_ellipse_path_z,
31
+ generate_interpolated_path,
32
+ generate_spiral_path,
33
+ )
34
+ from fused_ssim import fused_ssim
35
+
36
+ from src.utils.image import process_image
37
+ from src.post_opt.exporter import export_splats
38
+ from src.post_opt.lib_bilagrid import BilateralGrid, color_correct, slice, total_variation_loss
39
+ from torch import Tensor
40
+ from torch.nn.parallel import DistributedDataParallel as DDP
41
+ from torch.utils.tensorboard import SummaryWriter
42
+ from torchmetrics.image import PeakSignalNoiseRatio, StructuralSimilarityIndexMeasure
43
+ from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
44
+ from typing_extensions import Literal, assert_never
45
+ from src.post_opt.utils import AppearanceOptModule, CameraOptModule, knn, rgb_to_sh, set_random_seed
46
+
47
+ # from gsplat import export_splats
48
+ from gsplat.compression import PngCompression
49
+ from gsplat.distributed import cli
50
+ # from gsplat.optimizers import SelectiveAdam
51
+ # from gsplat.rendering import rasterization
52
+ from gsplat import rasterization
53
+ from gsplat.strategy import DefaultStrategy, MCMCStrategy
54
+ from src.post_opt.gsplat_viewer import GsplatViewer, GsplatRenderTabState
55
+ from nerfview import CameraState, RenderTabState, apply_float_colormap
56
+
57
+ import torch
58
+ from einops import rearrange
59
+ from jaxtyping import Float
60
+ from torch import Tensor
61
+ from scipy.spatial.transform import Rotation as R
62
+
63
+ from src.model.model.anysplat import AnySplat
64
+
65
+
66
+ # pytorch3d/pytorch3d/transforms/rotation_conversions.py at main · facebookresearch/pytorch3d
67
+ def quaternion_to_matrix(
68
+ quaternions: Float[Tensor, "*batch 4"],
69
+ eps: float = 1e-8,
70
+ ) -> Float[Tensor, "*batch 3 3"]:
71
+ # Order changed to match scipy format!
72
+ i, j, k, r = torch.unbind(quaternions, dim=-1)
73
+ two_s = 2 / ((quaternions * quaternions).sum(dim=-1) + eps)
74
+
75
+ o = torch.stack(
76
+ (
77
+ 1 - two_s * (j * j + k * k),
78
+ two_s * (i * j - k * r),
79
+ two_s * (i * k + j * r),
80
+ two_s * (i * j + k * r),
81
+ 1 - two_s * (i * i + k * k),
82
+ two_s * (j * k - i * r),
83
+ two_s * (i * k - j * r),
84
+ two_s * (j * k + i * r),
85
+ 1 - two_s * (i * i + j * j),
86
+ ),
87
+ -1,
88
+ )
89
+ return rearrange(o, "... (i j) -> ... i j", i=3, j=3)
90
+
91
+ def construct_list_of_attributes(num_rest: int) -> list[str]:
92
+ attributes = ["x", "y", "z", "nx", "ny", "nz"]
93
+ for i in range(3):
94
+ attributes.append(f"f_dc_{i}")
95
+ for i in range(num_rest):
96
+ attributes.append(f"f_rest_{i}")
97
+ attributes.append("opacity")
98
+ for i in range(3):
99
+ attributes.append(f"scale_{i}")
100
+ for i in range(4):
101
+ attributes.append(f"rot_{i}")
102
+ return attributes
103
+
104
+ def export_ply(
105
+ means: Float[Tensor, "gaussian 3"],
106
+ scales: Float[Tensor, "gaussian 3"],
107
+ rotations: Float[Tensor, "gaussian 4"],
108
+ harmonics: Float[Tensor, "gaussian 3 d_sh"],
109
+ opacities: Float[Tensor, " gaussian"],
110
+ path: Path,
111
+ shift_and_scale: bool = False,
112
+ save_sh_dc_only: bool = True,
113
+ ):
114
+ if shift_and_scale:
115
+ # Shift the scene so that the median Gaussian is at the origin.
116
+ means = means - means.median(dim=0).values
117
+
118
+ # Rescale the scene so that most Gaussians are within range [-1, 1].
119
+ scale_factor = means.abs().quantile(0.95, dim=0).max()
120
+ means = means / scale_factor
121
+ scales = scales / scale_factor
122
+
123
+ # Apply the rotation to the Gaussian rotations.
124
+ rotations = R.from_quat(rotations.detach().cpu().numpy()).as_matrix()
125
+ rotations = R.from_matrix(rotations).as_quat()
126
+ x, y, z, w = rearrange(rotations, "g xyzw -> xyzw g")
127
+ rotations = np.stack((w, x, y, z), axis=-1)
128
+
129
+ # Since current model use SH_degree = 4,
130
+ # which require large memory to store, we can only save the DC band to save memory.
131
+ f_dc = harmonics[..., 0]
132
+ f_rest = harmonics[..., 1:].flatten(start_dim=1)
133
+
134
+ dtype_full = [(attribute, "f4") for attribute in construct_list_of_attributes(0 if save_sh_dc_only else f_rest.shape[1])]
135
+ elements = np.empty(means.shape[0], dtype=dtype_full)
136
+ attributes = [
137
+ means.detach().cpu().numpy(),
138
+ torch.zeros_like(means).detach().cpu().numpy(),
139
+ f_dc.detach().cpu().contiguous().numpy(),
140
+ f_rest.detach().cpu().contiguous().numpy(),
141
+ opacities[..., None].detach().cpu().numpy(),
142
+ scales.detach().cpu().numpy(),
143
+ rotations,
144
+ ]
145
+ if save_sh_dc_only:
146
+ # remove f_rest from attributes
147
+ attributes.pop(3)
148
+
149
+ attributes = np.concatenate(attributes, axis=1)
150
+ elements[:] = list(map(tuple, attributes))
151
+ path.parent.mkdir(exist_ok=True, parents=True)
152
+ PlyData([PlyElement.describe(elements, "vertex")]).write(path)
153
+
154
+ def colorize_depth_maps(depth_map, min_depth=0.0, max_depth=1.0, cmap="Spectral", valid_mask=None):
155
+ """
156
+ Colorize depth maps.
157
+ """
158
+ assert len(depth_map.shape) >= 2, "Invalid dimension"
159
+
160
+ if isinstance(depth_map, torch.Tensor):
161
+ depth = depth_map.detach().clone().squeeze().numpy()
162
+ elif isinstance(depth_map, np.ndarray):
163
+ depth = depth_map.copy().squeeze()
164
+ # reshape to [ (B,) H, W ]
165
+ if depth.ndim < 3:
166
+ depth = depth[np.newaxis, :, :]
167
+
168
+ # colorize
169
+ cm = matplotlib.colormaps[cmap]
170
+ # depth = ((depth - min_depth) / (max_depth - min_depth)).clip(0, 1)
171
+ depth = ((depth - depth.min()) / (depth.max() - depth.min())).clip(0, 1)
172
+ img_colored_np = cm(depth, bytes=False)[:, :, :, 0:3] # value from 0 to 1
173
+ img_colored_np = np.rollaxis(img_colored_np, 3, 1)
174
+
175
+ if valid_mask is not None:
176
+ if isinstance(depth_map, torch.Tensor):
177
+ valid_mask = valid_mask.detach().numpy()
178
+ valid_mask = valid_mask.squeeze() # [H, W] or [B, H, W]
179
+ if valid_mask.ndim < 3:
180
+ valid_mask = valid_mask[np.newaxis, np.newaxis, :, :]
181
+ else:
182
+ valid_mask = valid_mask[:, np.newaxis, :, :]
183
+ valid_mask = np.repeat(valid_mask, 3, axis=1)
184
+ img_colored_np[~valid_mask] = 0
185
+
186
+ if isinstance(depth_map, torch.Tensor):
187
+ img_colored = torch.from_numpy(img_colored_np).float()
188
+ elif isinstance(depth_map, np.ndarray):
189
+ img_colored = img_colored_np
190
+
191
+ return img_colored
192
+
193
+ def build_covariance(
194
+ scale: Float[Tensor, "*#batch 3"],
195
+ rotation_xyzw: Float[Tensor, "*#batch 4"],
196
+ ) -> Float[Tensor, "*batch 3 3"]:
197
+ scale = scale.diag_embed()
198
+ rotation = quaternion_to_matrix(rotation_xyzw)
199
+ return (
200
+ rotation
201
+ @ scale
202
+ @ rearrange(scale, "... i j -> ... j i")
203
+ @ rearrange(rotation, "... i j -> ... j i")
204
+ )
205
+
206
+
207
+ @dataclass
208
+ class Config:
209
+ # Disable viewer
210
+ disable_viewer: bool = True
211
+ # Path to the .pt files. If provide, it will skip training and run evaluation only.
212
+ ckpt: Optional[List[str]] = None
213
+ # Name of compression strategy to use
214
+ compression: Optional[Literal["png"]] = None
215
+ # Render trajectory path
216
+ render_traj_path: str = "interp"
217
+
218
+ data_dir: str = "data/360_v2/garden"
219
+ # Downsample factor for the dataset
220
+ data_factor: int = 4
221
+ # Directory to save results
222
+ result_dir: str = "results/garden"
223
+ # Every N images there is a test image
224
+ test_every: int = 8
225
+ # Random crop size for training (experimental)
226
+ patch_size: Optional[int] = None
227
+ # A global scaler that applies to the scene size related parameters
228
+ global_scale: float = 1.0
229
+ # Normalize the world space
230
+ normalize_world_space: bool = True
231
+ # Camera model
232
+ camera_model: Literal["pinhole", "ortho", "fisheye"] = "pinhole"
233
+
234
+ # Port for the viewer server
235
+ port: int = 8080
236
+
237
+ # Batch size for training. Learning rates are scaled automatically
238
+ batch_size: int = 1
239
+ # A global factor to scale the number of training steps
240
+ steps_scaler: float = 1.0
241
+
242
+ # Number of training steps
243
+ max_steps: int = 3_000
244
+ # Steps to evaluate the model
245
+ eval_steps: List[int] = field(default_factory=lambda: [1, 1_000, 3_000, 7_000, 10_000])
246
+ # Steps to save the model
247
+ save_steps: List[int] = field(default_factory=lambda: [1, 1_000, 3_000, 7_000, 10_000])
248
+ # Whether to save ply file (storage size can be large)
249
+ save_ply: bool = False
250
+ # Steps to save the model as ply
251
+ ply_steps: List[int] = field(default_factory=lambda: [1, 1_000, 3_000, 7_000, 10_000])
252
+ # Whether to disable video generation during training and evaluation
253
+ disable_video: bool = False
254
+
255
+ # Initialization strategy
256
+ init_type: str = "sfm"
257
+ # Initial number of GSs. Ignored if using sfm
258
+ init_num_pts: int = 100_000
259
+ # Initial extent of GSs as a multiple of the camera extent. Ignored if using sfm
260
+ init_extent: float = 3.0
261
+ # Degree of spherical harmonics
262
+ sh_degree: int = 4
263
+ # Turn on another SH degree every this steps
264
+ sh_degree_interval: int = 1000
265
+ # Initial opacity of GS
266
+ init_opa: float = 0.1
267
+ # Initial scale of GS
268
+ init_scale: float = 1.0
269
+ # Weight for SSIM loss
270
+ ssim_lambda: float = 0.2
271
+
272
+ # Near plane clipping distance
273
+ near_plane: float = 1e-10
274
+ # Far plane clipping distance
275
+ far_plane: float = 1e10
276
+
277
+ # Strategy for GS densification
278
+ strategy: Union[DefaultStrategy, MCMCStrategy] = field(
279
+ default_factory=DefaultStrategy
280
+ )
281
+ # Use packed mode for rasterization, this leads to less memory usage but slightly slower.
282
+ packed: bool = False
283
+ # Use sparse gradients for optimization. (experimental)
284
+ sparse_grad: bool = False
285
+ # Use visible adam from Taming 3DGS. (experimental)
286
+ visible_adam: bool = False
287
+ # Anti-aliasing in rasterization. Might slightly hurt quantitative metrics.
288
+ antialiased: bool = False
289
+
290
+ # Use random background for training to discourage transparency
291
+ random_bkgd: bool = False
292
+
293
+ # Opacity regularization
294
+ opacity_reg: float = 0.0
295
+ # Scale regularization
296
+ scale_reg: float = 0.0
297
+
298
+ # Enable camera optimization.
299
+ pose_opt: bool = True
300
+ # Learning rate for camera optimization
301
+ pose_opt_lr: float = 1e-5
302
+ # Regularization for camera optimization as weight decay
303
+ pose_opt_reg: float = 1e-6
304
+ # Add noise to camera extrinsics. This is only to test the camera pose optimization.
305
+ pose_noise: float = 0.0
306
+
307
+ # Enable appearance optimization. (experimental)
308
+ app_opt: bool = False
309
+ # Appearance embedding dimension
310
+ app_embed_dim: int = 16
311
+ # Learning rate for appearance optimization
312
+ app_opt_lr: float = 1e-3
313
+ # Regularization for appearance optimization as weight decay
314
+ app_opt_reg: float = 1e-6
315
+
316
+ # Enable bilateral grid. (experimental)
317
+ use_bilateral_grid: bool = False
318
+ # Shape of the bilateral grid (X, Y, W)
319
+ bilateral_grid_shape: Tuple[int, int, int] = (16, 16, 8)
320
+
321
+ # Enable depth loss. (experimental)
322
+ depth_loss: bool = False
323
+ # Weight for depth loss
324
+ depth_lambda: float = 1e-2
325
+
326
+ # Dump information to tensorboard every this steps
327
+ tb_every: int = 100
328
+ # Save training images to tensorboard
329
+ tb_save_image: bool = False
330
+
331
+ lpips_net: Literal["vgg", "alex"] = "vgg"
332
+
333
+ lr_means: float = 1.6e-4
334
+ lr_scales: float = 5e-3
335
+ lr_quats: float = 1e-3
336
+ lr_opacities: float = 5e-2
337
+ lr_sh: float = 2.5e-3
338
+
339
+ def adjust_steps(self, factor: float):
340
+ self.eval_steps = [int(i * factor) for i in self.eval_steps]
341
+ self.save_steps = [int(i * factor) for i in self.save_steps]
342
+ self.ply_steps = [int(i * factor) for i in self.ply_steps]
343
+ self.max_steps = int(self.max_steps * factor)
344
+ self.sh_degree_interval = int(self.sh_degree_interval * factor)
345
+
346
+ strategy = self.strategy
347
+ if isinstance(strategy, DefaultStrategy):
348
+ # strategy.refine_start_iter = int(strategy.refine_start_iter * factor)
349
+ # strategy.refine_stop_iter = int(strategy.refine_stop_iter * factor)
350
+ # strategy.reset_every = int(strategy.reset_every * factor)
351
+ # strategy.refine_every = int(strategy.refine_every * factor)
352
+
353
+ strategy.refine_start_iter = 30000
354
+ strategy.refine_stop_iter = 0
355
+ strategy.reset_every = 30000
356
+ strategy.refine_every = 30000
357
+
358
+ elif isinstance(strategy, MCMCStrategy):
359
+ strategy.refine_start_iter = int(strategy.refine_start_iter * factor)
360
+ strategy.refine_stop_iter = int(strategy.refine_stop_iter * factor)
361
+ strategy.refine_every = int(strategy.refine_every * factor)
362
+ else:
363
+ assert_never(strategy)
364
+
365
+
366
+ def create_splats_with_optimizers(
367
+ gaussians: Gaussians,
368
+ init_num_pts: int = 100_000,
369
+ init_extent: float = 3.0,
370
+ init_opacity: float = 0.1,
371
+ init_scale: float = 1.0,
372
+ sh_degree: int = 3,
373
+ sparse_grad: bool = False,
374
+ visible_adam: bool = False,
375
+ batch_size: int = 1,
376
+ feature_dim: Optional[int] = None,
377
+ device: str = "cuda",
378
+ world_rank: int = 0,
379
+ world_size: int = 1,
380
+ cfg: Config = None,
381
+ ) -> Tuple[torch.nn.ParameterDict, Dict[str, torch.optim.Optimizer]]:
382
+
383
+ points = gaussians.means[0].detach().float()
384
+ scales = torch.log(gaussians.scales[0].detach().float())
385
+ quats = gaussians.rotations[0].detach().float()
386
+ opacities = torch.logit(gaussians.opacities[0].detach().float())
387
+ harmonics = gaussians.harmonics[0].detach().float().permute(0, 2, 1).contiguous()
388
+
389
+ N = points.shape[0]
390
+
391
+ scene_scale = 1.0
392
+ masks = opacities.sigmoid() > 0.01
393
+ harmonics = harmonics[masks]
394
+ params = [
395
+ # name, value, lr
396
+ ("means", torch.nn.Parameter(points[masks]), cfg.lr_means * scene_scale),
397
+ ("scales", torch.nn.Parameter(scales[masks]), cfg.lr_scales),
398
+ ("quats", torch.nn.Parameter(quats[masks]), cfg.lr_quats),
399
+ ("opacities", torch.nn.Parameter(opacities[masks]), cfg.lr_opacities),
400
+ ]
401
+
402
+ params.append(("sh0", torch.nn.Parameter(harmonics[:, :1, :]), cfg.lr_sh))
403
+ params.append(("shN", torch.nn.Parameter(harmonics[:, 1:, :]), cfg.lr_sh/20))
404
+
405
+ splats = torch.nn.ParameterDict({n: v for n, v, _ in params}).to(device)
406
+ # Scale learning rate based on batch size, reference:
407
+ # https://www.cs.princeton.edu/~smalladi/blog/2024/01/22/SDEs-ScalingRules/
408
+ # Note that this would not make the training exactly equivalent, see
409
+ # https://arxiv.org/pdf/2402.18824v1
410
+ BS = batch_size * world_size
411
+ optimizer_class = None
412
+ if sparse_grad:
413
+ optimizer_class = torch.optim.SparseAdam
414
+ elif visible_adam:
415
+ optimizer_class = SelectiveAdam
416
+ else:
417
+ optimizer_class = torch.optim.Adam
418
+ optimizers = {
419
+ name: optimizer_class(
420
+ [{"params": splats[name], "lr": lr * math.sqrt(BS), "name": name}],
421
+ eps=1e-15 / math.sqrt(BS),
422
+ # TODO: check betas logic when BS is larger than 10 betas[0] will be zero.
423
+ betas=(1 - BS * (1 - 0.9), 1 - BS * (1 - 0.999)),
424
+ )
425
+ for name, _, lr in params
426
+ }
427
+ return splats, optimizers
428
+
429
+
430
+ class Runner:
431
+ """Engine for training and testing."""
432
+
433
+ def __init__(
434
+ self, local_rank: int, world_rank, world_size: int, cfg: Config
435
+ ) -> None:
436
+ set_random_seed(42 + local_rank)
437
+
438
+ self.cfg = cfg
439
+ self.world_rank = world_rank
440
+ self.local_rank = local_rank
441
+ self.world_size = world_size
442
+ self.device = f"cuda:{local_rank}"
443
+
444
+ # Where to dump results.
445
+ os.makedirs(cfg.result_dir, exist_ok=True)
446
+
447
+ # Setup output directories.
448
+ self.ckpt_dir = f"{cfg.result_dir}/ckpts"
449
+ os.makedirs(self.ckpt_dir, exist_ok=True)
450
+ self.stats_dir = f"{cfg.result_dir}/stats"
451
+ os.makedirs(self.stats_dir, exist_ok=True)
452
+ self.render_dir = f"{cfg.result_dir}/renders"
453
+ os.makedirs(self.render_dir, exist_ok=True)
454
+ self.ply_dir = f"{cfg.result_dir}/ply"
455
+ os.makedirs(self.ply_dir, exist_ok=True)
456
+
457
+ # Tensorboard
458
+ self.writer = SummaryWriter(log_dir=f"{cfg.result_dir}/tb")
459
+
460
+ # first get the initial 3DGS and camera poses
461
+ model = AnySplat.from_pretrained("lhjiang/anysplat")
462
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
463
+ model.to(device)
464
+ model.eval()
465
+ for param in model.parameters():
466
+ param.requires_grad = False
467
+
468
+ image_folder = cfg.data_dir
469
+ image_names = sorted([os.path.join(image_folder, f) for f in os.listdir(image_folder) if f.lower().endswith(('.png', '.jpg', '.jpeg'))])
470
+ images = [process_image(img_path) for img_path in image_names]
471
+ ctx_indices = [idx for idx, name in enumerate(image_names) if idx % cfg.test_every != 0]
472
+ tgt_indices = [idx for idx, name in enumerate(image_names) if idx % cfg.test_every == 0]
473
+
474
+ ctx_images = torch.stack([images[i] for i in ctx_indices], dim=0).unsqueeze(0).to(device)
475
+ tgt_images = torch.stack([images[i] for i in tgt_indices], dim=0).unsqueeze(0).to(device)
476
+ ctx_images = (ctx_images+1)*0.5
477
+ tgt_images = (tgt_images+1)*0.5
478
+ b, v, _, h, w = tgt_images.shape
479
+
480
+ # run inference
481
+ encoder_output = model.encoder(
482
+ ctx_images,
483
+ global_step=0,
484
+ visualization_dump={},
485
+ )
486
+ gaussians, pred_context_pose = encoder_output.gaussians, encoder_output.pred_context_pose
487
+
488
+ num_context_view = ctx_images.shape[1]
489
+ vggt_input_image = torch.cat((ctx_images, tgt_images), dim=1).to(torch.bfloat16)
490
+ with torch.no_grad(), torch.cuda.amp.autocast(enabled=False, dtype=torch.bfloat16):
491
+ aggregated_tokens_list, patch_start_idx = model.encoder.aggregator(vggt_input_image, intermediate_layer_idx=model.encoder.cfg.intermediate_layer_idx)
492
+ with torch.cuda.amp.autocast(enabled=False):
493
+ fp32_tokens = [token.float() for token in aggregated_tokens_list]
494
+ pred_all_pose_enc = model.encoder.camera_head(fp32_tokens)[-1]
495
+ pred_all_extrinsic, pred_all_intrinsic = pose_encoding_to_extri_intri(pred_all_pose_enc, vggt_input_image.shape[-2:])
496
+
497
+ extrinsic_padding = torch.tensor([0, 0, 0, 1], device=pred_all_extrinsic.device, dtype=pred_all_extrinsic.dtype).view(1, 1, 1, 4).repeat(b, vggt_input_image.shape[1], 1, 1)
498
+ pred_all_extrinsic = torch.cat([pred_all_extrinsic, extrinsic_padding], dim=2).inverse()
499
+
500
+ pred_all_intrinsic[:, :, 0] = pred_all_intrinsic[:, :, 0] / w
501
+ pred_all_intrinsic[:, :, 1] = pred_all_intrinsic[:, :, 1] / h
502
+ pred_all_context_extrinsic, pred_all_target_extrinsic = pred_all_extrinsic[:, :num_context_view], pred_all_extrinsic[:, num_context_view:]
503
+ pred_all_context_intrinsic, pred_all_target_intrinsic = pred_all_intrinsic[:, :num_context_view], pred_all_intrinsic[:, num_context_view:]
504
+
505
+ scale_factor = pred_context_pose['extrinsic'][:, :, :3, 3].mean() / pred_all_context_extrinsic[:, :, :3, 3].mean()
506
+ pred_all_target_extrinsic[..., :3, 3] = pred_all_target_extrinsic[..., :3, 3] * scale_factor
507
+ pred_all_context_extrinsic[..., :3, 3] = pred_all_context_extrinsic[..., :3, 3] * scale_factor
508
+ print("scale_factor:", scale_factor)
509
+
510
+ # Load data: Training data should contain initial points and colors.
511
+ # self.parser = Parser(
512
+ # data_dir=cfg.data_dir,
513
+ # factor=cfg.data_factor,
514
+ # normalize=cfg.normalize_world_space,
515
+ # test_every=cfg.test_every,
516
+ # )
517
+ self.trainset = Dataset(
518
+ split="train",
519
+ images=ctx_images[0].detach().cpu().numpy(),
520
+ camtoworlds=pred_all_context_extrinsic[0].detach().cpu().numpy(),
521
+ Ks=pred_all_context_intrinsic[0].detach().cpu().numpy(),
522
+ patch_size=cfg.patch_size,
523
+ load_depths=cfg.depth_loss,
524
+ )
525
+ self.valset = Dataset(
526
+ images=tgt_images[0].detach().cpu().numpy(),
527
+ camtoworlds=pred_all_target_extrinsic[0].detach().cpu().numpy(),
528
+ Ks=pred_all_target_intrinsic[0].detach().cpu().numpy(),
529
+ split="val"
530
+ )
531
+
532
+ # Model
533
+ feature_dim = 32 if cfg.app_opt else None
534
+ self.splats, self.optimizers = create_splats_with_optimizers(
535
+ gaussians=gaussians,
536
+ init_num_pts=cfg.init_num_pts,
537
+ init_extent=cfg.init_extent,
538
+ init_opacity=cfg.init_opa,
539
+ init_scale=cfg.init_scale,
540
+ sh_degree=cfg.sh_degree,
541
+ sparse_grad=cfg.sparse_grad,
542
+ visible_adam=cfg.visible_adam,
543
+ batch_size=cfg.batch_size,
544
+ feature_dim=feature_dim,
545
+ device=self.device,
546
+ world_rank=world_rank,
547
+ world_size=world_size,
548
+ cfg=cfg,
549
+ )
550
+ print("Model initialized. Number of GS:", len(self.splats["means"]))
551
+
552
+ # Densification Strategy
553
+ self.cfg.strategy.check_sanity(self.splats, self.optimizers)
554
+
555
+ if isinstance(self.cfg.strategy, DefaultStrategy):
556
+ self.strategy_state = self.cfg.strategy.initialize_state(
557
+ scene_scale=1.0
558
+ )
559
+ elif isinstance(self.cfg.strategy, MCMCStrategy):
560
+ self.strategy_state = self.cfg.strategy.initialize_state()
561
+ else:
562
+ assert_never(self.cfg.strategy)
563
+
564
+ # Compression Strategy
565
+ self.compression_method = None
566
+ if cfg.compression is not None:
567
+ if cfg.compression == "png":
568
+ self.compression_method = PngCompression()
569
+ else:
570
+ raise ValueError(f"Unknown compression strategy: {cfg.compression}")
571
+
572
+ self.pose_optimizers = []
573
+ if cfg.pose_opt:
574
+ self.pose_adjust = CameraOptModule(len(self.trainset)).to(self.device)
575
+ self.pose_adjust.zero_init()
576
+ self.pose_optimizers = [
577
+ torch.optim.Adam(
578
+ self.pose_adjust.parameters(),
579
+ lr=cfg.pose_opt_lr * math.sqrt(cfg.batch_size),
580
+ weight_decay=cfg.pose_opt_reg,
581
+ )
582
+ ]
583
+ if world_size > 1:
584
+ self.pose_adjust = DDP(self.pose_adjust)
585
+
586
+ if cfg.pose_noise > 0.0:
587
+ self.pose_perturb = CameraOptModule(len(self.trainset)).to(self.device)
588
+ self.pose_perturb.random_init(cfg.pose_noise)
589
+ if world_size > 1:
590
+ self.pose_perturb = DDP(self.pose_perturb)
591
+
592
+ self.app_optimizers = []
593
+ if cfg.app_opt:
594
+ assert feature_dim is not None
595
+ self.app_module = AppearanceOptModule(
596
+ len(self.trainset), feature_dim, cfg.app_embed_dim, cfg.sh_degree
597
+ ).to(self.device)
598
+ # initialize the last layer to be zero so that the initial output is zero.
599
+ torch.nn.init.zeros_(self.app_module.color_head[-1].weight)
600
+ torch.nn.init.zeros_(self.app_module.color_head[-1].bias)
601
+ self.app_optimizers = [
602
+ torch.optim.Adam(
603
+ self.app_module.embeds.parameters(),
604
+ lr=cfg.app_opt_lr * math.sqrt(cfg.batch_size) * 10.0,
605
+ weight_decay=cfg.app_opt_reg,
606
+ ),
607
+ torch.optim.Adam(
608
+ self.app_module.color_head.parameters(),
609
+ lr=cfg.app_opt_lr * math.sqrt(cfg.batch_size),
610
+ ),
611
+ ]
612
+ if world_size > 1:
613
+ self.app_module = DDP(self.app_module)
614
+
615
+ self.bil_grid_optimizers = []
616
+ if cfg.use_bilateral_grid:
617
+ self.bil_grids = BilateralGrid(
618
+ len(self.trainset),
619
+ grid_X=cfg.bilateral_grid_shape[0],
620
+ grid_Y=cfg.bilateral_grid_shape[1],
621
+ grid_W=cfg.bilateral_grid_shape[2],
622
+ ).to(self.device)
623
+ self.bil_grid_optimizers = [
624
+ torch.optim.Adam(
625
+ self.bil_grids.parameters(),
626
+ lr=2e-3 * math.sqrt(cfg.batch_size),
627
+ eps=1e-15,
628
+ ),
629
+ ]
630
+
631
+ # Losses & Metrics.
632
+ self.ssim = StructuralSimilarityIndexMeasure(data_range=1.0).to(self.device)
633
+ self.psnr = PeakSignalNoiseRatio(data_range=1.0).to(self.device)
634
+
635
+ if cfg.lpips_net == "alex":
636
+ self.lpips = LearnedPerceptualImagePatchSimilarity(
637
+ net_type="alex", normalize=True
638
+ ).to(self.device)
639
+ elif cfg.lpips_net == "vgg":
640
+ # The 3DGS official repo uses lpips vgg, which is equivalent with the following:
641
+ self.lpips = LearnedPerceptualImagePatchSimilarity(
642
+ net_type="vgg", normalize=False
643
+ ).to(self.device)
644
+ else:
645
+ raise ValueError(f"Unknown LPIPS network: {cfg.lpips_net}")
646
+
647
+ # Viewer
648
+ if not self.cfg.disable_viewer:
649
+ self.server = viser.ViserServer(port=cfg.port, verbose=False)
650
+ self.viewer = GsplatViewer(
651
+ server=self.server,
652
+ render_fn=self._viewer_render_fn,
653
+ output_dir=Path(cfg.result_dir),
654
+ mode="training",
655
+ )
656
+
657
+ def rasterize_splats(
658
+ self,
659
+ camtoworlds: Tensor,
660
+ Ks: Tensor,
661
+ width: int,
662
+ height: int,
663
+ masks: Optional[Tensor] = None,
664
+ rasterize_mode: Optional[Literal["classic", "antialiased"]] = None,
665
+ camera_model: Optional[Literal["pinhole", "ortho", "fisheye"]] = None,
666
+ **kwargs,
667
+ ) -> Tuple[Tensor, Tensor, Dict]:
668
+ means = self.splats["means"] # [N, 3]
669
+ # quats = F.normalize(self.splats["quats"], dim=-1) # [N, 4]
670
+ # rasterization does normalization internally
671
+ quats = self.splats["quats"] # [N, 4]
672
+ scales = torch.exp(self.splats["scales"]) # [N, 3]
673
+ opacities = torch.sigmoid(self.splats["opacities"]) # [N,]
674
+
675
+ image_ids = kwargs.pop("image_ids", None)
676
+ if self.cfg.app_opt:
677
+ colors = self.app_module(
678
+ features=self.splats["features"],
679
+ embed_ids=image_ids,
680
+ dirs=means[None, :, :] - camtoworlds[:, None, :3, 3],
681
+ sh_degree=kwargs.pop("sh_degree", self.cfg.sh_degree),
682
+ )
683
+ colors = colors + self.splats["colors"]
684
+ colors = torch.sigmoid(colors)
685
+ else:
686
+ colors = torch.cat([self.splats["sh0"], self.splats["shN"]], 1) # [N, K, 3]
687
+
688
+ if rasterize_mode is None:
689
+ rasterize_mode = "antialiased" if self.cfg.antialiased else "classic"
690
+ if camera_model is None:
691
+ camera_model = self.cfg.camera_model
692
+
693
+ # covariance = build_covariance(scales[None], quats[None]).squeeze(0)
694
+ render_colors, render_alphas, info = rasterization(
695
+ means=means,
696
+ quats=quats,
697
+ scales=scales,
698
+ opacities=opacities,
699
+ colors=colors,
700
+ # covars=covariance,
701
+ viewmats=torch.linalg.inv(camtoworlds), # [C, 4, 4]
702
+ Ks=Ks, # [C, 3, 3]
703
+ width=width,
704
+ height=height,
705
+ packed=self.cfg.packed,
706
+ absgrad=(
707
+ self.cfg.strategy.absgrad
708
+ if isinstance(self.cfg.strategy, DefaultStrategy)
709
+ else False
710
+ ),
711
+ sparse_grad=self.cfg.sparse_grad,
712
+ rasterize_mode=rasterize_mode,
713
+ distributed=self.world_size > 1,
714
+ camera_model=self.cfg.camera_model,
715
+ radius_clip=0.1,
716
+ backgrounds=torch.tensor([0.0, 0.0, 0.0]).cuda().unsqueeze(0).repeat(1, 1),
717
+ **kwargs,
718
+ )
719
+ if masks is not None:
720
+ render_colors[~masks] = 0
721
+ return render_colors, render_alphas, info
722
+
723
+ def train(self):
724
+ cfg = self.cfg
725
+ device = self.device
726
+ world_rank = self.world_rank
727
+ world_size = self.world_size
728
+
729
+ # Dump cfg.
730
+ if world_rank == 0:
731
+ with open(f"{cfg.result_dir}/cfg.yml", "w") as f:
732
+ yaml.dump(vars(cfg), f)
733
+
734
+ max_steps = cfg.max_steps
735
+ init_step = 0
736
+
737
+ schedulers = [
738
+ # means has a learning rate schedule, that end at 0.01 of the initial value
739
+ torch.optim.lr_scheduler.ExponentialLR(
740
+ self.optimizers["means"], gamma=0.01 ** (1.0 / max_steps)
741
+ ),
742
+ ]
743
+ if cfg.pose_opt:
744
+ # pose optimization has a learning rate schedule
745
+ schedulers.append(
746
+ torch.optim.lr_scheduler.ExponentialLR(
747
+ self.pose_optimizers[0], gamma=0.01 ** (1.0 / max_steps)
748
+ )
749
+ )
750
+ if cfg.use_bilateral_grid:
751
+ # bilateral grid has a learning rate schedule. Linear warmup for 1000 steps.
752
+ schedulers.append(
753
+ torch.optim.lr_scheduler.ChainedScheduler(
754
+ [
755
+ torch.optim.lr_scheduler.LinearLR(
756
+ self.bil_grid_optimizers[0],
757
+ start_factor=0.01,
758
+ total_iters=1000,
759
+ ),
760
+ torch.optim.lr_scheduler.ExponentialLR(
761
+ self.bil_grid_optimizers[0], gamma=0.01 ** (1.0 / max_steps)
762
+ ),
763
+ ]
764
+ )
765
+ )
766
+
767
+ trainloader = torch.utils.data.DataLoader(
768
+ self.trainset,
769
+ batch_size=cfg.batch_size,
770
+ shuffle=True,
771
+ num_workers=4,
772
+ persistent_workers=True,
773
+ pin_memory=True,
774
+ )
775
+ trainloader_iter = iter(trainloader)
776
+
777
+ # Training loop.
778
+ global_tic = time.time()
779
+ pbar = tqdm.tqdm(range(init_step, max_steps))
780
+ for step in pbar:
781
+ if not cfg.disable_viewer:
782
+ while self.viewer.state == "paused":
783
+ time.sleep(0.01)
784
+ self.viewer.lock.acquire()
785
+ tic = time.time()
786
+
787
+ try:
788
+ data = next(trainloader_iter)
789
+ except StopIteration:
790
+ trainloader_iter = iter(trainloader)
791
+ data = next(trainloader_iter)
792
+
793
+ camtoworlds = camtoworlds_gt = data["camtoworld"].to(device) # [1, 4, 4]
794
+ Ks = data["K"].to(device) # [1, 3, 3]
795
+ pixels = data["image"].to(device) / 255.0 # [1, H, W, 3]
796
+ num_train_rays_per_step = (
797
+ pixels.shape[0] * pixels.shape[1] * pixels.shape[2]
798
+ )
799
+ image_ids = data["image_id"].to(device)
800
+ masks = data["mask"].to(device) if "mask" in data else None # [1, H, W]
801
+ if cfg.depth_loss:
802
+ points = data["points"].to(device) # [1, M, 2]
803
+ depths_gt = data["depths"].to(device) # [1, M]
804
+
805
+ height, width = pixels.shape[1:3]
806
+
807
+ if cfg.pose_noise:
808
+ camtoworlds = self.pose_perturb(camtoworlds, image_ids)
809
+
810
+ if cfg.pose_opt:
811
+ camtoworlds = self.pose_adjust(camtoworlds, image_ids)
812
+
813
+ # sh schedule
814
+ # sh_degree_to_use = min(step // cfg.sh_degree_interval, cfg.sh_degree)
815
+ sh_degree_to_use = cfg.sh_degree
816
+
817
+ # forward
818
+ renders, alphas, info = self.rasterize_splats(
819
+ camtoworlds=camtoworlds,
820
+ Ks=Ks,
821
+ width=width,
822
+ height=height,
823
+ sh_degree=sh_degree_to_use,
824
+ near_plane=cfg.near_plane,
825
+ far_plane=cfg.far_plane,
826
+ image_ids=image_ids,
827
+ render_mode="RGB+ED" if cfg.depth_loss else "RGB",
828
+ masks=masks,
829
+ )
830
+ if renders.shape[-1] == 4:
831
+ colors, depths = renders[..., 0:3], renders[..., 3:4]
832
+ else:
833
+ colors, depths = renders, None
834
+
835
+ if cfg.use_bilateral_grid:
836
+ grid_y, grid_x = torch.meshgrid(
837
+ (torch.arange(height, device=self.device) + 0.5) / height,
838
+ (torch.arange(width, device=self.device) + 0.5) / width,
839
+ indexing="ij",
840
+ )
841
+ grid_xy = torch.stack([grid_x, grid_y], dim=-1).unsqueeze(0)
842
+ colors = slice(self.bil_grids, grid_xy, colors, image_ids)["rgb"]
843
+
844
+ if cfg.random_bkgd:
845
+ bkgd = torch.rand(1, 3, device=device)
846
+ colors = colors + bkgd * (1.0 - alphas)
847
+
848
+ self.cfg.strategy.step_pre_backward(
849
+ params=self.splats,
850
+ optimizers=self.optimizers,
851
+ state=self.strategy_state,
852
+ step=step,
853
+ info=info,
854
+ )
855
+
856
+ # loss
857
+ l1loss = F.l1_loss(colors, pixels)
858
+ ssimloss = 1.0 - fused_ssim(
859
+ colors.permute(0, 3, 1, 2), pixels.permute(0, 3, 1, 2), padding="valid"
860
+ )
861
+ loss = l1loss * (1.0 - cfg.ssim_lambda) + ssimloss * cfg.ssim_lambda
862
+ if cfg.depth_loss:
863
+ # query depths from depth map
864
+ points = torch.stack(
865
+ [
866
+ points[:, :, 0] / (width - 1) * 2 - 1,
867
+ points[:, :, 1] / (height - 1) * 2 - 1,
868
+ ],
869
+ dim=-1,
870
+ ) # normalize to [-1, 1]
871
+ grid = points.unsqueeze(2) # [1, M, 1, 2]
872
+ depths = F.grid_sample(
873
+ depths.permute(0, 3, 1, 2), grid, align_corners=True
874
+ ) # [1, 1, M, 1]
875
+ depths = depths.squeeze(3).squeeze(1) # [1, M]
876
+ # calculate loss in disparity space
877
+ disp = torch.where(depths > 0.0, 1.0 / depths, torch.zeros_like(depths))
878
+ disp_gt = 1.0 / depths_gt # [1, M]
879
+ depthloss = F.l1_loss(disp, disp_gt) * self.scene_scale
880
+ loss += depthloss * cfg.depth_lambda
881
+ if cfg.use_bilateral_grid:
882
+ tvloss = 10 * total_variation_loss(self.bil_grids.grids)
883
+ loss += tvloss
884
+
885
+ # regularizations
886
+ if cfg.opacity_reg > 0.0:
887
+ loss = (
888
+ loss
889
+ + cfg.opacity_reg
890
+ * torch.abs(torch.sigmoid(self.splats["opacities"])).mean()
891
+ )
892
+ if cfg.scale_reg > 0.0:
893
+ loss = (
894
+ loss
895
+ + cfg.scale_reg * torch.abs(torch.exp(self.splats["scales"])).mean()
896
+ )
897
+
898
+ loss.backward()
899
+
900
+ desc = f"loss={loss.item():.3f}| " f"sh degree={sh_degree_to_use}| "
901
+ if cfg.depth_loss:
902
+ desc += f"depth loss={depthloss.item():.6f}| "
903
+ if cfg.pose_opt and cfg.pose_noise:
904
+ # monitor the pose error if we inject noise
905
+ pose_err = F.l1_loss(camtoworlds_gt, camtoworlds)
906
+ desc += f"pose err={pose_err.item():.6f}| "
907
+ pbar.set_description(desc)
908
+
909
+ # write images (gt and render)
910
+ # if world_rank == 0 and step % 800 == 0:
911
+ # canvas = torch.cat([pixels, colors], dim=2).detach().cpu().numpy()
912
+ # canvas = canvas.reshape(-1, *canvas.shape[2:])
913
+ # imageio.imwrite(
914
+ # f"{self.render_dir}/train_rank{self.world_rank}.png",
915
+ # (canvas * 255).astype(np.uint8),
916
+ # )
917
+
918
+ if world_rank == 0 and cfg.tb_every > 0 and step % cfg.tb_every == 0:
919
+ mem = torch.cuda.max_memory_allocated() / 1024**3
920
+ self.writer.add_scalar("train/loss", loss.item(), step)
921
+ self.writer.add_scalar("train/l1loss", l1loss.item(), step)
922
+ self.writer.add_scalar("train/ssimloss", ssimloss.item(), step)
923
+ self.writer.add_scalar("train/num_GS", len(self.splats["means"]), step)
924
+ self.writer.add_scalar("train/mem", mem, step)
925
+ if cfg.depth_loss:
926
+ self.writer.add_scalar("train/depthloss", depthloss.item(), step)
927
+ if cfg.use_bilateral_grid:
928
+ self.writer.add_scalar("train/tvloss", tvloss.item(), step)
929
+ if cfg.tb_save_image:
930
+ canvas = torch.cat([pixels, colors], dim=2).detach().cpu().numpy()
931
+ canvas = canvas.reshape(-1, *canvas.shape[2:])
932
+ self.writer.add_image("train/render", canvas, step)
933
+ self.writer.flush()
934
+
935
+ # save checkpoint before updating the model
936
+ if step in [i - 1 for i in cfg.save_steps] or step == max_steps - 1:
937
+ mem = torch.cuda.max_memory_allocated() / 1024**3
938
+ stats = {
939
+ "mem": mem,
940
+ "ellipse_time": time.time() - global_tic,
941
+ "num_GS": len(self.splats["means"]),
942
+ }
943
+ print("Step: ", step, stats)
944
+ with open(
945
+ f"{self.stats_dir}/train_step{step:04d}_rank{self.world_rank}.json",
946
+ "w",
947
+ ) as f:
948
+ json.dump(stats, f)
949
+ data = {"step": step, "splats": self.splats.state_dict()}
950
+ if cfg.pose_opt:
951
+ if world_size > 1:
952
+ data["pose_adjust"] = self.pose_adjust.module.state_dict()
953
+ else:
954
+ data["pose_adjust"] = self.pose_adjust.state_dict()
955
+ if cfg.app_opt:
956
+ if world_size > 1:
957
+ data["app_module"] = self.app_module.module.state_dict()
958
+ else:
959
+ data["app_module"] = self.app_module.state_dict()
960
+ torch.save(
961
+ data, f"{self.ckpt_dir}/ckpt_{step}_rank{self.world_rank}.pt"
962
+ )
963
+ if (
964
+ step in [i - 1 for i in cfg.ply_steps] or step == max_steps - 1
965
+ ) and cfg.save_ply:
966
+
967
+ if self.cfg.app_opt:
968
+ # eval at origin to bake the appeareance into the colors
969
+ rgb = self.app_module(
970
+ features=self.splats["features"],
971
+ embed_ids=None,
972
+ dirs=torch.zeros_like(self.splats["means"][None, :, :]),
973
+ sh_degree=sh_degree_to_use,
974
+ )
975
+ rgb = rgb + self.splats["colors"]
976
+ rgb = torch.sigmoid(rgb).squeeze(0).unsqueeze(1)
977
+ sh0 = rgb_to_sh(rgb)
978
+ shN = torch.empty([sh0.shape[0], 0, 3], device=sh0.device)
979
+ else:
980
+ sh0 = self.splats["sh0"]
981
+ shN = self.splats["shN"]
982
+ # shN = torch.empty([sh0.shape[0], 0, 3], device=sh0.device)
983
+
984
+ means = self.splats["means"]
985
+ scales = self.splats["scales"]
986
+ quats = self.splats["quats"]
987
+ opacities = self.splats["opacities"]
988
+
989
+ # export_splats(
990
+ # means=means,
991
+ # scales=scales,
992
+ # quats=quats,
993
+ # opacities=opacities,
994
+ # sh0=sh0,
995
+ # shN=shN,
996
+ # format="ply",
997
+ # save_to=f"{self.ply_dir}/point_cloud_{step}.ply",
998
+ # )
999
+ export_ply(
1000
+ means=means,
1001
+ scales=scales,
1002
+ rotations=quats,
1003
+ harmonics=torch.cat([sh0, shN], dim=1).permute(0, 2, 1),
1004
+ opacities=opacities.sigmoid(),
1005
+ path=Path(f"{self.ply_dir}/point_cloud_{step}.ply"),
1006
+ )
1007
+
1008
+ # Turn Gradients into Sparse Tensor before running optimizer
1009
+ if cfg.sparse_grad:
1010
+ assert cfg.packed, "Sparse gradients only work with packed mode."
1011
+ gaussian_ids = info["gaussian_ids"]
1012
+ for k in self.splats.keys():
1013
+ grad = self.splats[k].grad
1014
+ if grad is None or grad.is_sparse:
1015
+ continue
1016
+ self.splats[k].grad = torch.sparse_coo_tensor(
1017
+ indices=gaussian_ids[None], # [1, nnz]
1018
+ values=grad[gaussian_ids], # [nnz, ...]
1019
+ size=self.splats[k].size(), # [N, ...]
1020
+ is_coalesced=len(Ks) == 1,
1021
+ )
1022
+
1023
+ if cfg.visible_adam:
1024
+ gaussian_cnt = self.splats.means.shape[0]
1025
+ if cfg.packed:
1026
+ visibility_mask = torch.zeros_like(
1027
+ self.splats["opacities"], dtype=bool
1028
+ )
1029
+ visibility_mask.scatter_(0, info["gaussian_ids"], 1)
1030
+ else:
1031
+ visibility_mask = (info["radii"] > 0).all(-1).any(0)
1032
+
1033
+ # optimize
1034
+ for optimizer in self.optimizers.values():
1035
+ if cfg.visible_adam:
1036
+ optimizer.step(visibility_mask)
1037
+ else:
1038
+ optimizer.step()
1039
+ optimizer.zero_grad(set_to_none=True)
1040
+ for optimizer in self.pose_optimizers:
1041
+ optimizer.step()
1042
+ optimizer.zero_grad(set_to_none=True)
1043
+ for optimizer in self.app_optimizers:
1044
+ optimizer.step()
1045
+ optimizer.zero_grad(set_to_none=True)
1046
+ for optimizer in self.bil_grid_optimizers:
1047
+ optimizer.step()
1048
+ optimizer.zero_grad(set_to_none=True)
1049
+ for scheduler in schedulers:
1050
+ scheduler.step()
1051
+
1052
+ # Run post-backward steps after backward and optimizer
1053
+ if isinstance(self.cfg.strategy, DefaultStrategy):
1054
+ self.cfg.strategy.step_post_backward(
1055
+ params=self.splats,
1056
+ optimizers=self.optimizers,
1057
+ state=self.strategy_state,
1058
+ step=step,
1059
+ info=info,
1060
+ packed=cfg.packed,
1061
+ )
1062
+ elif isinstance(self.cfg.strategy, MCMCStrategy):
1063
+ self.cfg.strategy.step_post_backward(
1064
+ params=self.splats,
1065
+ optimizers=self.optimizers,
1066
+ state=self.strategy_state,
1067
+ step=step,
1068
+ info=info,
1069
+ lr=schedulers[0].get_last_lr()[0],
1070
+ )
1071
+ else:
1072
+ assert_never(self.cfg.strategy)
1073
+
1074
+ # eval the full set
1075
+ if step in [i - 1 for i in cfg.eval_steps]:
1076
+ self.eval(step)
1077
+ # self.render_traj(step)
1078
+
1079
+ # run compression
1080
+ if cfg.compression is not None and step in [i - 1 for i in cfg.eval_steps]:
1081
+ self.run_compression(step=step)
1082
+
1083
+ if not cfg.disable_viewer:
1084
+ self.viewer.lock.release()
1085
+ num_train_steps_per_sec = 1.0 / (time.time() - tic)
1086
+ num_train_rays_per_sec = (
1087
+ num_train_rays_per_step * num_train_steps_per_sec
1088
+ )
1089
+ # Update the viewer state.
1090
+ self.viewer.render_tab_state.num_train_rays_per_sec = (
1091
+ num_train_rays_per_sec
1092
+ )
1093
+ # Update the scene.
1094
+ self.viewer.update(step, num_train_rays_per_step)
1095
+
1096
+ @torch.no_grad()
1097
+ def eval(self, step: int, stage: str = "val"):
1098
+ """Entry for evaluation."""
1099
+ print("Running evaluation...")
1100
+ cfg = self.cfg
1101
+ device = self.device
1102
+ world_rank = self.world_rank
1103
+ world_size = self.world_size
1104
+
1105
+ valloader = torch.utils.data.DataLoader(
1106
+ self.valset, batch_size=1, shuffle=False, num_workers=1
1107
+ )
1108
+ ellipse_time = 0
1109
+ metrics = defaultdict(list)
1110
+ for i, data in enumerate(valloader):
1111
+ camtoworlds = data["camtoworld"].to(device)
1112
+ Ks = data["K"].to(device)
1113
+ pixels = data["image"].to(device) / 255.0
1114
+ masks = data["mask"].to(device) if "mask" in data else None
1115
+ height, width = pixels.shape[1:3]
1116
+
1117
+ torch.cuda.synchronize()
1118
+ tic = time.time()
1119
+ render_colors, _, _ = self.rasterize_splats(
1120
+ camtoworlds=camtoworlds,
1121
+ Ks=Ks,
1122
+ width=width,
1123
+ height=height,
1124
+ sh_degree=cfg.sh_degree,
1125
+ near_plane=cfg.near_plane,
1126
+ far_plane=cfg.far_plane,
1127
+ # radius_clip=0.1,
1128
+ render_mode="RGB+ED",
1129
+ masks=masks,
1130
+ ) # [1, H, W, 3]
1131
+ torch.cuda.synchronize()
1132
+ ellipse_time += time.time() - tic
1133
+
1134
+ colors = render_colors[..., :3]
1135
+ depths = render_colors[..., 3]
1136
+
1137
+ colors = torch.clamp(colors, 0.0, 1.0)
1138
+ canvas_list = [pixels, colors]
1139
+
1140
+ if world_rank == 0:
1141
+ # write images
1142
+ canvas = torch.cat(canvas_list, dim=2).squeeze(0).cpu().numpy()
1143
+ canvas = (canvas * 255).astype(np.uint8)
1144
+ imageio.imwrite(
1145
+ f"{self.render_dir}/{stage}_step{step}_{i:04d}.png",
1146
+ canvas,
1147
+ )
1148
+ torchvision.utils.save_image(pixels.permute(0, 3, 1, 2), f"{self.render_dir}/gt_rgb_{stage}_step{step}_{i:04d}.png")
1149
+ torchvision.utils.save_image(colors.permute(0, 3, 1, 2), f"{self.render_dir}/render_rgb_{stage}_step{step}_{i:04d}.png")
1150
+ # save depth & normal map
1151
+
1152
+
1153
+ pixels_p = pixels.permute(0, 3, 1, 2) # [1, 3, H, W]
1154
+ colors_p = colors.permute(0, 3, 1, 2) # [1, 3, H, W]
1155
+
1156
+ metrics["psnr"].append(self.psnr(colors_p, pixels_p))
1157
+ metrics["ssim"].append(self.ssim(colors_p, pixels_p))
1158
+ metrics["lpips"].append(self.lpips(colors_p, pixels_p))
1159
+ if cfg.use_bilateral_grid:
1160
+ cc_colors = color_correct(colors, pixels)
1161
+ cc_colors_p = cc_colors.permute(0, 3, 1, 2) # [1, 3, H, W]
1162
+ metrics["cc_psnr"].append(self.psnr(cc_colors_p, pixels_p))
1163
+
1164
+ if world_rank == 0:
1165
+ ellipse_time /= len(valloader)
1166
+
1167
+ stats = {k: torch.stack(v).mean().item() for k, v in metrics.items()}
1168
+ stats.update(
1169
+ {
1170
+ "ellipse_time": ellipse_time,
1171
+ "num_GS": len(self.splats["means"]),
1172
+ }
1173
+ )
1174
+ print(
1175
+ f"PSNR: {stats['psnr']:.3f}, SSIM: {stats['ssim']:.4f}, LPIPS: {stats['lpips']:.3f} "
1176
+ f"Time: {stats['ellipse_time']:.3f}s/image "
1177
+ f"Number of GS: {stats['num_GS']}"
1178
+ )
1179
+ # save stats as json
1180
+ with open(f"{self.stats_dir}/{stage}_step{step:04d}.json", "w") as f:
1181
+ json.dump(stats, f)
1182
+ # save stats to tensorboard
1183
+ for k, v in stats.items():
1184
+ self.writer.add_scalar(f"{stage}/{k}", v, step)
1185
+ self.writer.flush()
1186
+
1187
+ @torch.no_grad()
1188
+ def render_traj(self, step: int):
1189
+ """Entry for trajectory rendering."""
1190
+ if self.cfg.disable_video:
1191
+ return
1192
+ print("Running trajectory rendering...")
1193
+ cfg = self.cfg
1194
+ device = self.device
1195
+
1196
+ camtoworlds_all = self.parser.camtoworlds[5:-5]
1197
+ if cfg.render_traj_path == "interp":
1198
+ camtoworlds_all = generate_interpolated_path(
1199
+ camtoworlds_all, 1
1200
+ ) # [N, 3, 4]
1201
+ elif cfg.render_traj_path == "ellipse":
1202
+ height = camtoworlds_all[:, 2, 3].mean()
1203
+ camtoworlds_all = generate_ellipse_path_z(
1204
+ camtoworlds_all, height=height
1205
+ ) # [N, 3, 4]
1206
+ elif cfg.render_traj_path == "spiral":
1207
+ camtoworlds_all = generate_spiral_path(
1208
+ camtoworlds_all,
1209
+ bounds=self.parser.bounds * self.scene_scale,
1210
+ spiral_scale_r=self.parser.extconf["spiral_radius_scale"],
1211
+ )
1212
+ else:
1213
+ raise ValueError(
1214
+ f"Render trajectory type not supported: {cfg.render_traj_path}"
1215
+ )
1216
+
1217
+ camtoworlds_all = np.concatenate(
1218
+ [
1219
+ camtoworlds_all,
1220
+ np.repeat(
1221
+ np.array([[[0.0, 0.0, 0.0, 1.0]]]), len(camtoworlds_all), axis=0
1222
+ ),
1223
+ ],
1224
+ axis=1,
1225
+ ) # [N, 4, 4]
1226
+
1227
+ camtoworlds_all = torch.from_numpy(camtoworlds_all).float().to(device)
1228
+ K = torch.from_numpy(list(self.parser.Ks_dict.values())[0]).float().to(device)
1229
+ width, height = list(self.parser.imsize_dict.values())[0]
1230
+
1231
+ # save to video
1232
+ video_dir = f"{cfg.result_dir}/videos"
1233
+ os.makedirs(video_dir, exist_ok=True)
1234
+ writer = imageio.get_writer(f"{video_dir}/traj_{step}.mp4", fps=30)
1235
+ for i in tqdm.trange(len(camtoworlds_all), desc="Rendering trajectory"):
1236
+ camtoworlds = camtoworlds_all[i : i + 1]
1237
+ Ks = K[None]
1238
+
1239
+ renders, _, _ = self.rasterize_splats(
1240
+ camtoworlds=camtoworlds,
1241
+ Ks=Ks,
1242
+ width=width,
1243
+ height=height,
1244
+ sh_degree=cfg.sh_degree,
1245
+ near_plane=cfg.near_plane,
1246
+ far_plane=cfg.far_plane,
1247
+ render_mode="RGB+ED",
1248
+ ) # [1, H, W, 4]
1249
+ colors = torch.clamp(renders[..., 0:3], 0.0, 1.0) # [1, H, W, 3]
1250
+ depths = renders[..., 3:4] # [1, H, W, 1]
1251
+ depths = (depths - depths.min()) / (depths.max() - depths.min())
1252
+ canvas_list = [colors, depths.repeat(1, 1, 1, 3)]
1253
+
1254
+ # write images
1255
+ canvas = torch.cat(canvas_list, dim=2).squeeze(0).cpu().numpy()
1256
+ canvas = (canvas * 255).astype(np.uint8)
1257
+ writer.append_data(canvas)
1258
+ writer.close()
1259
+ print(f"Video saved to {video_dir}/traj_{step}.mp4")
1260
+
1261
+ @torch.no_grad()
1262
+ def run_compression(self, step: int):
1263
+ """Entry for running compression."""
1264
+ print("Running compression...")
1265
+ world_rank = self.world_rank
1266
+
1267
+ compress_dir = f"{cfg.result_dir}/compression/rank{world_rank}"
1268
+ os.makedirs(compress_dir, exist_ok=True)
1269
+
1270
+ self.compression_method.compress(compress_dir, self.splats)
1271
+
1272
+ # evaluate compression
1273
+ splats_c = self.compression_method.decompress(compress_dir)
1274
+ for k in splats_c.keys():
1275
+ self.splats[k].data = splats_c[k].to(self.device)
1276
+ self.eval(step=step, stage="compress")
1277
+
1278
+ @torch.no_grad()
1279
+ def _viewer_render_fn(
1280
+ self, camera_state: CameraState, render_tab_state: RenderTabState
1281
+ ):
1282
+ assert isinstance(render_tab_state, GsplatRenderTabState)
1283
+ if render_tab_state.preview_render:
1284
+ width = render_tab_state.render_width
1285
+ height = render_tab_state.render_height
1286
+ else:
1287
+ width = render_tab_state.viewer_width
1288
+ height = render_tab_state.viewer_height
1289
+ c2w = camera_state.c2w
1290
+ K = camera_state.get_K((width, height))
1291
+ c2w = torch.from_numpy(c2w).float().to(self.device)
1292
+ K = torch.from_numpy(K).float().to(self.device)
1293
+
1294
+ RENDER_MODE_MAP = {
1295
+ "rgb": "RGB",
1296
+ "depth(accumulated)": "D",
1297
+ "depth(expected)": "ED",
1298
+ "alpha": "RGB",
1299
+ }
1300
+
1301
+ render_colors, render_alphas, info = self.rasterize_splats(
1302
+ camtoworlds=c2w[None],
1303
+ Ks=K[None],
1304
+ width=width,
1305
+ height=height,
1306
+ sh_degree=min(render_tab_state.max_sh_degree, self.cfg.sh_degree),
1307
+ near_plane=render_tab_state.near_plane,
1308
+ far_plane=render_tab_state.far_plane,
1309
+ radius_clip=render_tab_state.radius_clip,
1310
+ # radius_clip=0.1,
1311
+ eps2d=render_tab_state.eps2d,
1312
+ backgrounds=torch.tensor([render_tab_state.backgrounds], device=self.device)
1313
+ / 255.0,
1314
+ render_mode=RENDER_MODE_MAP[render_tab_state.render_mode],
1315
+ rasterize_mode=render_tab_state.rasterize_mode,
1316
+ camera_model=render_tab_state.camera_model,
1317
+ ) # [1, H, W, 3]
1318
+ render_tab_state.total_gs_count = len(self.splats["means"])
1319
+ render_tab_state.rendered_gs_count = (info["radii"] > 0).all(-1).sum().item()
1320
+
1321
+ if render_tab_state.render_mode == "rgb":
1322
+ # colors represented with sh are not guranteed to be in [0, 1]
1323
+ render_colors = render_colors[0, ..., 0:3].clamp(0, 1)
1324
+ renders = render_colors.cpu().numpy()
1325
+ elif render_tab_state.render_mode in ["depth(accumulated)", "depth(expected)"]:
1326
+ # normalize depth to [0, 1]
1327
+ depth = render_colors[0, ..., 0:1]
1328
+ if render_tab_state.normalize_nearfar:
1329
+ near_plane = render_tab_state.near_plane
1330
+ far_plane = render_tab_state.far_plane
1331
+ else:
1332
+ near_plane = depth.min()
1333
+ far_plane = depth.max()
1334
+ depth_norm = (depth - near_plane) / (far_plane - near_plane + 1e-10)
1335
+ depth_norm = torch.clip(depth_norm, 0, 1)
1336
+ if render_tab_state.inverse:
1337
+ depth_norm = 1 - depth_norm
1338
+ renders = (
1339
+ apply_float_colormap(depth_norm, render_tab_state.colormap)
1340
+ .cpu()
1341
+ .numpy()
1342
+ )
1343
+ elif render_tab_state.render_mode == "alpha":
1344
+ alpha = render_alphas[0, ..., 0:1]
1345
+ if render_tab_state.inverse:
1346
+ alpha = 1 - alpha
1347
+ renders = (
1348
+ apply_float_colormap(alpha, render_tab_state.colormap).cpu().numpy()
1349
+ )
1350
+ return renders
1351
+
1352
+
1353
+ def main(local_rank: int, world_rank, world_size: int, cfg: Config):
1354
+ if world_size > 1 and not cfg.disable_viewer:
1355
+ cfg.disable_viewer = True
1356
+ if world_rank == 0:
1357
+ print("Viewer is disabled in distributed training.")
1358
+
1359
+ runner = Runner(local_rank, world_rank, world_size, cfg)
1360
+
1361
+ if cfg.ckpt is not None:
1362
+ # run eval only
1363
+ ckpts = [
1364
+ torch.load(file, map_location=runner.device, weights_only=True)
1365
+ for file in cfg.ckpt
1366
+ ]
1367
+ for k in runner.splats.keys():
1368
+ runner.splats[k].data = torch.cat([ckpt["splats"][k] for ckpt in ckpts])
1369
+ step = ckpts[0]["step"]
1370
+ runner.eval(step=step)
1371
+ # runner.render_traj(step=step)
1372
+ if cfg.compression is not None:
1373
+ runner.run_compression(step=step)
1374
+ else:
1375
+ runner.train()
1376
+ runner.eval(step=runner.cfg.max_steps)
1377
+ # runner.render_traj(step=runner.cfg.max_steps)
1378
+ print("Training complete.")
1379
+ # runner.viewer.complete()
1380
+ # if not cfg.disable_viewer:
1381
+ # print("Viewer running... Ctrl+C to exit.")
1382
+ # time.sleep(1000000)
1383
+
1384
+
1385
+ if __name__ == "__main__":
1386
+ """
1387
+ Usage:
1388
+
1389
+ ```bash
1390
+ # Single GPU training
1391
+ CUDA_VISIBLE_DEVICES=9 python -m examples.simple_trainer default
1392
+
1393
+ # Distributed training on 4 GPUs: Effectively 4x batch size so run 4x less steps.
1394
+ CUDA_VISIBLE_DEVICES=0,1,2,3 python simple_trainer.py default --steps_scaler 0.25
1395
+
1396
+ """
1397
+
1398
+ # Config objects we can choose between.
1399
+ # Each is a tuple of (CLI description, config object).
1400
+ configs = {
1401
+ "default": (
1402
+ "Gaussian splatting training using densification heuristics from the original paper.",
1403
+ Config(
1404
+ strategy=DefaultStrategy(verbose=True),
1405
+ ),
1406
+ ),
1407
+ "mcmc": (
1408
+ "Gaussian splatting training using densification from the paper '3D Gaussian Splatting as Markov Chain Monte Carlo'.",
1409
+ Config(
1410
+ init_opa=0.5,
1411
+ init_scale=0.1,
1412
+ opacity_reg=0.01,
1413
+ scale_reg=0.01,
1414
+ strategy=MCMCStrategy(verbose=True),
1415
+ ),
1416
+ ),
1417
+ }
1418
+ cfg = tyro.extras.overridable_config_cli(configs)
1419
+ cfg.adjust_steps(cfg.steps_scaler)
1420
+
1421
+ # try import extra dependencies
1422
+ if cfg.compression == "png":
1423
+ try:
1424
+ import plas
1425
+ import torchpq
1426
+ except:
1427
+ raise ImportError(
1428
+ "To use PNG compression, you need to install "
1429
+ "torchpq (instruction at https://github.com/DeMoriarty/TorchPQ?tab=readme-ov-file#install) "
1430
+ "and plas (via 'pip install git+https://github.com/fraunhoferhhi/PLAS.git') "
1431
+ )
1432
+
1433
+ cli(main, cfg, verbose=True)
src/post_opt/simple_viewer.py ADDED
@@ -0,0 +1,275 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import math
3
+ import os
4
+ import time
5
+
6
+ import imageio
7
+ import numpy as np
8
+ import torch
9
+ import torch.nn.functional as F
10
+ import tqdm
11
+ import viser
12
+ from pathlib import Path
13
+ from gsplat._helper import load_test_data
14
+ from gsplat.distributed import cli
15
+ from gsplat.rendering import rasterization
16
+
17
+ from nerfview import CameraState, RenderTabState, apply_float_colormap
18
+ from examples.gsplat_viewer import GsplatViewer, GsplatRenderTabState
19
+
20
+
21
+ def main(local_rank: int, world_rank, world_size: int, args):
22
+ torch.manual_seed(42)
23
+ device = torch.device("cuda", local_rank)
24
+
25
+ if args.ckpt is None:
26
+ (
27
+ means,
28
+ quats,
29
+ scales,
30
+ opacities,
31
+ colors,
32
+ viewmats,
33
+ Ks,
34
+ width,
35
+ height,
36
+ ) = load_test_data(device=device, scene_grid=args.scene_grid)
37
+
38
+ assert world_size <= 2
39
+ means = means[world_rank::world_size].contiguous()
40
+ means.requires_grad = True
41
+ quats = quats[world_rank::world_size].contiguous()
42
+ quats.requires_grad = True
43
+ scales = scales[world_rank::world_size].contiguous()
44
+ scales.requires_grad = True
45
+ opacities = opacities[world_rank::world_size].contiguous()
46
+ opacities.requires_grad = True
47
+ colors = colors[world_rank::world_size].contiguous()
48
+ colors.requires_grad = True
49
+
50
+ viewmats = viewmats[world_rank::world_size][:1].contiguous()
51
+ Ks = Ks[world_rank::world_size][:1].contiguous()
52
+
53
+ sh_degree = None
54
+ C = len(viewmats)
55
+ N = len(means)
56
+ print("rank", world_rank, "Number of Gaussians:", N, "Number of Cameras:", C)
57
+
58
+ # batched render
59
+ for _ in tqdm.trange(1):
60
+ render_colors, render_alphas, meta = rasterization(
61
+ means, # [N, 3]
62
+ quats, # [N, 4]
63
+ scales, # [N, 3]
64
+ opacities, # [N]
65
+ colors, # [N, S, 3]
66
+ viewmats, # [C, 4, 4]
67
+ Ks, # [C, 3, 3]
68
+ width,
69
+ height,
70
+ render_mode="RGB+D",
71
+ packed=False,
72
+ distributed=world_size > 1,
73
+ )
74
+ C = render_colors.shape[0]
75
+ assert render_colors.shape == (C, height, width, 4)
76
+ assert render_alphas.shape == (C, height, width, 1)
77
+ render_colors.sum().backward()
78
+
79
+ render_rgbs = render_colors[..., 0:3]
80
+ render_depths = render_colors[..., 3:4]
81
+ render_depths = render_depths / render_depths.max()
82
+
83
+ # dump batch images
84
+ os.makedirs(args.output_dir, exist_ok=True)
85
+ canvas = (
86
+ torch.cat(
87
+ [
88
+ render_rgbs.reshape(C * height, width, 3),
89
+ render_depths.reshape(C * height, width, 1).expand(-1, -1, 3),
90
+ render_alphas.reshape(C * height, width, 1).expand(-1, -1, 3),
91
+ ],
92
+ dim=1,
93
+ )
94
+ .detach()
95
+ .cpu()
96
+ .numpy()
97
+ )
98
+ imageio.imsave(
99
+ f"{args.output_dir}/render_rank{world_rank}.png",
100
+ (canvas * 255).astype(np.uint8),
101
+ )
102
+ else:
103
+ means, quats, scales, opacities, sh0, shN = [], [], [], [], [], []
104
+ for ckpt_path in args.ckpt:
105
+ ckpt = torch.load(ckpt_path, map_location=device)["splats"]
106
+ means.append(ckpt["means"])
107
+ quats.append(F.normalize(ckpt["quats"], p=2, dim=-1))
108
+ scales.append(torch.exp(ckpt["scales"]))
109
+ opacities.append(torch.sigmoid(ckpt["opacities"]))
110
+ sh0.append(ckpt["sh0"])
111
+ shN.append(ckpt["shN"])
112
+ means = torch.cat(means, dim=0)
113
+ quats = torch.cat(quats, dim=0)
114
+ scales = torch.cat(scales, dim=0)
115
+ opacities = torch.cat(opacities, dim=0)
116
+ sh0 = torch.cat(sh0, dim=0)
117
+ shN = torch.cat(shN, dim=0)
118
+ colors = torch.cat([sh0, shN], dim=-2)
119
+ sh_degree = int(math.sqrt(colors.shape[-2]) - 1)
120
+
121
+ # # crop
122
+ # aabb = torch.tensor((-1.0, -1.0, -1.0, 1.0, 1.0, 0.7), device=device)
123
+ # edges = aabb[3:] - aabb[:3]
124
+ # sel = ((means >= aabb[:3]) & (means <= aabb[3:])).all(dim=-1)
125
+ # sel = torch.where(sel)[0]
126
+ # means, quats, scales, colors, opacities = (
127
+ # means[sel],
128
+ # quats[sel],
129
+ # scales[sel],
130
+ # colors[sel],
131
+ # opacities[sel],
132
+ # )
133
+
134
+ # # repeat the scene into a grid (to mimic a large-scale setting)
135
+ # repeats = args.scene_grid
136
+ # gridx, gridy = torch.meshgrid(
137
+ # [
138
+ # torch.arange(-(repeats // 2), repeats // 2 + 1, device=device),
139
+ # torch.arange(-(repeats // 2), repeats // 2 + 1, device=device),
140
+ # ],
141
+ # indexing="ij",
142
+ # )
143
+ # grid = torch.stack([gridx, gridy, torch.zeros_like(gridx)], dim=-1).reshape(
144
+ # -1, 3
145
+ # )
146
+ # means = means[None, :, :] + grid[:, None, :] * edges[None, None, :]
147
+ # means = means.reshape(-1, 3)
148
+ # quats = quats.repeat(repeats**2, 1)
149
+ # scales = scales.repeat(repeats**2, 1)
150
+ # colors = colors.repeat(repeats**2, 1, 1)
151
+ # opacities = opacities.repeat(repeats**2)
152
+ print("Number of Gaussians:", len(means))
153
+
154
+ # register and open viewer
155
+ @torch.no_grad()
156
+ def viewer_render_fn(camera_state: CameraState, render_tab_state: RenderTabState):
157
+ assert isinstance(render_tab_state, GsplatRenderTabState)
158
+ if render_tab_state.preview_render:
159
+ width = render_tab_state.render_width
160
+ height = render_tab_state.render_height
161
+ else:
162
+ width = render_tab_state.viewer_width
163
+ height = render_tab_state.viewer_height
164
+ c2w = camera_state.c2w
165
+ K = camera_state.get_K((width, height))
166
+ c2w = torch.from_numpy(c2w).float().to(device)
167
+ K = torch.from_numpy(K).float().to(device)
168
+ viewmat = c2w.inverse()
169
+
170
+ RENDER_MODE_MAP = {
171
+ "rgb": "RGB",
172
+ "depth(accumulated)": "D",
173
+ "depth(expected)": "ED",
174
+ "alpha": "RGB",
175
+ }
176
+
177
+ render_colors, render_alphas, info = rasterization(
178
+ means, # [N, 3]
179
+ quats, # [N, 4]
180
+ scales, # [N, 3]
181
+ opacities, # [N]
182
+ colors, # [N, S, 3]
183
+ viewmat[None], # [1, 4, 4]
184
+ K[None], # [1, 3, 3]
185
+ width,
186
+ height,
187
+ sh_degree=(
188
+ min(render_tab_state.max_sh_degree, sh_degree)
189
+ if sh_degree is not None
190
+ else None
191
+ ),
192
+ near_plane=render_tab_state.near_plane,
193
+ far_plane=render_tab_state.far_plane,
194
+ radius_clip=render_tab_state.radius_clip,
195
+ eps2d=render_tab_state.eps2d,
196
+ backgrounds=torch.tensor([render_tab_state.backgrounds], device=device)
197
+ / 255.0,
198
+ render_mode=RENDER_MODE_MAP[render_tab_state.render_mode],
199
+ rasterize_mode=render_tab_state.rasterize_mode,
200
+ camera_model=render_tab_state.camera_model,
201
+ )
202
+ render_tab_state.total_gs_count = len(means)
203
+ render_tab_state.rendered_gs_count = (info["radii"] > 0).all(-1).sum().item()
204
+
205
+ if render_tab_state.render_mode == "rgb":
206
+ # colors represented with sh are not guranteed to be in [0, 1]
207
+ render_colors = render_colors[0, ..., 0:3].clamp(0, 1)
208
+ renders = render_colors.cpu().numpy()
209
+ elif render_tab_state.render_mode in ["depth(accumulated)", "depth(expected)"]:
210
+ # normalize depth to [0, 1]
211
+ depth = render_colors[0, ..., 0:1]
212
+ if render_tab_state.normalize_nearfar:
213
+ near_plane = render_tab_state.near_plane
214
+ far_plane = render_tab_state.far_plane
215
+ else:
216
+ near_plane = depth.min()
217
+ far_plane = depth.max()
218
+ depth_norm = (depth - near_plane) / (far_plane - near_plane + 1e-10)
219
+ depth_norm = torch.clip(depth_norm, 0, 1)
220
+ if render_tab_state.inverse:
221
+ depth_norm = 1 - depth_norm
222
+ renders = (
223
+ apply_float_colormap(depth_norm, render_tab_state.colormap)
224
+ .cpu()
225
+ .numpy()
226
+ )
227
+ elif render_tab_state.render_mode == "alpha":
228
+ alpha = render_alphas[0, ..., 0:1]
229
+ if render_tab_state.inverse:
230
+ alpha = 1 - alpha
231
+ renders = (
232
+ apply_float_colormap(alpha, render_tab_state.colormap).cpu().numpy()
233
+ )
234
+ return renders
235
+
236
+ server = viser.ViserServer(port=args.port, verbose=False)
237
+ _ = GsplatViewer(
238
+ server=server,
239
+ render_fn=viewer_render_fn,
240
+ output_dir=Path(args.output_dir),
241
+ mode="rendering",
242
+ )
243
+ print("Viewer running... Ctrl+C to exit.")
244
+ time.sleep(100000)
245
+
246
+
247
+ if __name__ == "__main__":
248
+ """
249
+ # Use single GPU to view the scene
250
+ CUDA_VISIBLE_DEVICES=9 python -m simple_viewer \
251
+ --ckpt results/garden/ckpts/ckpt_6999_rank0.pt \
252
+ --output_dir results/garden/ \
253
+ --port 8082
254
+
255
+ CUDA_VISIBLE_DEVICES=9 python -m simple_viewer \
256
+ --output_dir results/garden/ \
257
+ --port 8082
258
+ """
259
+ parser = argparse.ArgumentParser()
260
+ parser.add_argument(
261
+ "--output_dir", type=str, default="results/", help="where to dump outputs"
262
+ )
263
+ parser.add_argument(
264
+ "--scene_grid", type=int, default=1, help="repeat the scene into a grid of NxN"
265
+ )
266
+ parser.add_argument(
267
+ "--ckpt", type=str, nargs="+", default=None, help="path to the .pt file"
268
+ )
269
+ parser.add_argument(
270
+ "--port", type=int, default=8080, help="port for the viewer server"
271
+ )
272
+ args = parser.parse_args()
273
+ assert args.scene_grid % 2 == 1, "scene_grid must be odd"
274
+
275
+ cli(main, args, verbose=True)
src/post_opt/utils.py ADDED
@@ -0,0 +1,224 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import random
2
+
3
+ import numpy as np
4
+ import torch
5
+ from sklearn.neighbors import NearestNeighbors
6
+ from torch import Tensor
7
+ import torch.nn.functional as F
8
+ import matplotlib.pyplot as plt
9
+ from matplotlib import colormaps
10
+
11
+
12
+ class CameraOptModule(torch.nn.Module):
13
+ """Camera pose optimization module."""
14
+
15
+ def __init__(self, n: int):
16
+ super().__init__()
17
+ # Delta positions (3D) + Delta rotations (6D)
18
+ self.embeds = torch.nn.Embedding(n, 9)
19
+ # Identity rotation in 6D representation
20
+ self.register_buffer("identity", torch.tensor([1.0, 0.0, 0.0, 0.0, 1.0, 0.0]))
21
+
22
+ def zero_init(self):
23
+ torch.nn.init.zeros_(self.embeds.weight)
24
+
25
+ def random_init(self, std: float):
26
+ torch.nn.init.normal_(self.embeds.weight, std=std)
27
+
28
+ def forward(self, camtoworlds: Tensor, embed_ids: Tensor) -> Tensor:
29
+ """Adjust camera pose based on deltas.
30
+
31
+ Args:
32
+ camtoworlds: (..., 4, 4)
33
+ embed_ids: (...,)
34
+
35
+ Returns:
36
+ updated camtoworlds: (..., 4, 4)
37
+ """
38
+ assert camtoworlds.shape[:-2] == embed_ids.shape
39
+ batch_shape = camtoworlds.shape[:-2]
40
+ pose_deltas = self.embeds(embed_ids) # (..., 9)
41
+ dx, drot = pose_deltas[..., :3], pose_deltas[..., 3:]
42
+ rot = rotation_6d_to_matrix(
43
+ drot + self.identity.expand(*batch_shape, -1)
44
+ ) # (..., 3, 3)
45
+ transform = torch.eye(4, device=pose_deltas.device).repeat((*batch_shape, 1, 1))
46
+ transform[..., :3, :3] = rot
47
+ transform[..., :3, 3] = dx
48
+ return torch.matmul(camtoworlds, transform)
49
+
50
+
51
+ class AppearanceOptModule(torch.nn.Module):
52
+ """Appearance optimization module."""
53
+
54
+ def __init__(
55
+ self,
56
+ n: int,
57
+ feature_dim: int,
58
+ embed_dim: int = 16,
59
+ sh_degree: int = 3,
60
+ mlp_width: int = 64,
61
+ mlp_depth: int = 2,
62
+ ):
63
+ super().__init__()
64
+ self.embed_dim = embed_dim
65
+ self.sh_degree = sh_degree
66
+ self.embeds = torch.nn.Embedding(n, embed_dim)
67
+ layers = []
68
+ layers.append(
69
+ torch.nn.Linear(embed_dim + feature_dim + (sh_degree + 1) ** 2, mlp_width)
70
+ )
71
+ layers.append(torch.nn.ReLU(inplace=True))
72
+ for _ in range(mlp_depth - 1):
73
+ layers.append(torch.nn.Linear(mlp_width, mlp_width))
74
+ layers.append(torch.nn.ReLU(inplace=True))
75
+ layers.append(torch.nn.Linear(mlp_width, 3))
76
+ self.color_head = torch.nn.Sequential(*layers)
77
+
78
+ def forward(
79
+ self, features: Tensor, embed_ids: Tensor, dirs: Tensor, sh_degree: int
80
+ ) -> Tensor:
81
+ """Adjust appearance based on embeddings.
82
+
83
+ Args:
84
+ features: (N, feature_dim)
85
+ embed_ids: (C,)
86
+ dirs: (C, N, 3)
87
+
88
+ Returns:
89
+ colors: (C, N, 3)
90
+ """
91
+ from gsplat.cuda._torch_impl import _eval_sh_bases_fast
92
+
93
+ C, N = dirs.shape[:2]
94
+ # Camera embeddings
95
+ if embed_ids is None:
96
+ embeds = torch.zeros(C, self.embed_dim, device=features.device)
97
+ else:
98
+ embeds = self.embeds(embed_ids) # [C, D2]
99
+ embeds = embeds[:, None, :].expand(-1, N, -1) # [C, N, D2]
100
+ # GS features
101
+ features = features[None, :, :].expand(C, -1, -1) # [C, N, D1]
102
+ # View directions
103
+ dirs = F.normalize(dirs, dim=-1) # [C, N, 3]
104
+ num_bases_to_use = (sh_degree + 1) ** 2
105
+ num_bases = (self.sh_degree + 1) ** 2
106
+ sh_bases = torch.zeros(C, N, num_bases, device=features.device) # [C, N, K]
107
+ sh_bases[:, :, :num_bases_to_use] = _eval_sh_bases_fast(num_bases_to_use, dirs)
108
+ # Get colors
109
+ if self.embed_dim > 0:
110
+ h = torch.cat([embeds, features, sh_bases], dim=-1) # [C, N, D1 + D2 + K]
111
+ else:
112
+ h = torch.cat([features, sh_bases], dim=-1)
113
+ colors = self.color_head(h)
114
+ return colors
115
+
116
+
117
+ def rotation_6d_to_matrix(d6: Tensor) -> Tensor:
118
+ """
119
+ Converts 6D rotation representation by Zhou et al. [1] to rotation matrix
120
+ using Gram--Schmidt orthogonalization per Section B of [1]. Adapted from pytorch3d.
121
+ Args:
122
+ d6: 6D rotation representation, of size (*, 6)
123
+
124
+ Returns:
125
+ batch of rotation matrices of size (*, 3, 3)
126
+
127
+ [1] Zhou, Y., Barnes, C., Lu, J., Yang, J., & Li, H.
128
+ On the Continuity of Rotation Representations in Neural Networks.
129
+ IEEE Conference on Computer Vision and Pattern Recognition, 2019.
130
+ Retrieved from http://arxiv.org/abs/1812.07035
131
+ """
132
+
133
+ a1, a2 = d6[..., :3], d6[..., 3:]
134
+ b1 = F.normalize(a1, dim=-1)
135
+ b2 = a2 - (b1 * a2).sum(-1, keepdim=True) * b1
136
+ b2 = F.normalize(b2, dim=-1)
137
+ b3 = torch.cross(b1, b2, dim=-1)
138
+ return torch.stack((b1, b2, b3), dim=-2)
139
+
140
+
141
+ def knn(x: Tensor, K: int = 4) -> Tensor:
142
+ x_np = x.cpu().numpy()
143
+ model = NearestNeighbors(n_neighbors=K, metric="euclidean").fit(x_np)
144
+ distances, _ = model.kneighbors(x_np)
145
+ return torch.from_numpy(distances).to(x)
146
+
147
+
148
+ def rgb_to_sh(rgb: Tensor) -> Tensor:
149
+ C0 = 0.28209479177387814
150
+ return (rgb - 0.5) / C0
151
+
152
+
153
+ def set_random_seed(seed: int):
154
+ random.seed(seed)
155
+ np.random.seed(seed)
156
+ torch.manual_seed(seed)
157
+
158
+
159
+ # ref: https://github.com/hbb1/2d-gaussian-splatting/blob/main/utils/general_utils.py#L163
160
+ def colormap(img, cmap="jet"):
161
+ W, H = img.shape[:2]
162
+ dpi = 300
163
+ fig, ax = plt.subplots(1, figsize=(H / dpi, W / dpi), dpi=dpi)
164
+ im = ax.imshow(img, cmap=cmap)
165
+ ax.set_axis_off()
166
+ fig.colorbar(im, ax=ax)
167
+ fig.tight_layout()
168
+ fig.canvas.draw()
169
+ data = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)
170
+ data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
171
+ img = torch.from_numpy(data).float().permute(2, 0, 1)
172
+ plt.close()
173
+ return img
174
+
175
+
176
+ def apply_float_colormap(img: torch.Tensor, colormap: str = "turbo") -> torch.Tensor:
177
+ """Convert single channel to a color img.
178
+
179
+ Args:
180
+ img (torch.Tensor): (..., 1) float32 single channel image.
181
+ colormap (str): Colormap for img.
182
+
183
+ Returns:
184
+ (..., 3) colored img with colors in [0, 1].
185
+ """
186
+ img = torch.nan_to_num(img, 0)
187
+ if colormap == "gray":
188
+ return img.repeat(1, 1, 3)
189
+ img_long = (img * 255).long()
190
+ img_long_min = torch.min(img_long)
191
+ img_long_max = torch.max(img_long)
192
+ assert img_long_min >= 0, f"the min value is {img_long_min}"
193
+ assert img_long_max <= 255, f"the max value is {img_long_max}"
194
+ return torch.tensor(
195
+ colormaps[colormap].colors, # type: ignore
196
+ device=img.device,
197
+ )[img_long[..., 0]]
198
+
199
+
200
+ def apply_depth_colormap(
201
+ depth: torch.Tensor,
202
+ acc: torch.Tensor = None,
203
+ near_plane: float = None,
204
+ far_plane: float = None,
205
+ ) -> torch.Tensor:
206
+ """Converts a depth image to color for easier analysis.
207
+
208
+ Args:
209
+ depth (torch.Tensor): (..., 1) float32 depth.
210
+ acc (torch.Tensor | None): (..., 1) optional accumulation mask.
211
+ near_plane: Closest depth to consider. If None, use min image value.
212
+ far_plane: Furthest depth to consider. If None, use max image value.
213
+
214
+ Returns:
215
+ (..., 3) colored depth image with colors in [0, 1].
216
+ """
217
+ near_plane = near_plane or float(torch.min(depth))
218
+ far_plane = far_plane or float(torch.max(depth))
219
+ depth = (depth - near_plane) / (far_plane - near_plane + 1e-10)
220
+ depth = torch.clip(depth, 0.0, 1.0)
221
+ img = apply_float_colormap(depth, colormap="turbo")
222
+ if acc is not None:
223
+ img = img * acc + (1.0 - acc)
224
+ return img
src/utils/__init.py ADDED
File without changes
src/utils/ba.py ADDED
@@ -0,0 +1,407 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pycolmap
2
+ import numpy as np
3
+
4
+ import torch
5
+ import torch.nn.functional as F
6
+ from src.model.encoder.vggt.utils.pose_enc import pose_encoding_to_extri_intri
7
+ from src.model.encoder.vggt.utils.geometry import closed_form_inverse_se3, unproject_depth_map_to_point_map
8
+ from lightglue import ALIKED, SuperPoint, SIFT
9
+ from src.utils.tensor_to_pycolmap import batch_matrix_to_pycolmap, pycolmap_to_batch_matrix
10
+
11
+
12
+ _RESNET_MEAN = [0.485, 0.456, 0.406]
13
+ _RESNET_STD = [0.229, 0.224, 0.225]
14
+
15
+
16
+ def generate_rank_by_dino(
17
+ images, query_frame_num, image_size=518, model_name="dinov2_vitb14_reg", device="cuda", spatial_similarity=True
18
+ ):
19
+ """
20
+ Generate a ranking of frames using DINO ViT features.
21
+
22
+ Args:
23
+ images: Tensor of shape (S, 3, H, W) with values in range [0, 1]
24
+ query_frame_num: Number of frames to select
25
+ image_size: Size to resize images to before processing
26
+ model_name: Name of the DINO model to use
27
+ device: Device to run the model on
28
+ spatial_similarity: Whether to use spatial token similarity or CLS token similarity
29
+
30
+ Returns:
31
+ List of frame indices ranked by their representativeness
32
+ """
33
+ dino_v2_model = torch.hub.load('facebookresearch/dinov2', model_name)
34
+ dino_v2_model.eval()
35
+ dino_v2_model = dino_v2_model.to(device)
36
+
37
+ resnet_mean = torch.tensor(_RESNET_MEAN, device=device).view(1, 3, 1, 1)
38
+ resnet_std = torch.tensor(_RESNET_STD, device=device).view(1, 3, 1, 1)
39
+ images_resnet_norm = (images - resnet_mean) / resnet_std
40
+
41
+ with torch.no_grad():
42
+ frame_feat = dino_v2_model(images_resnet_norm, is_training=True)
43
+
44
+ if spatial_similarity:
45
+ frame_feat = frame_feat["x_norm_patchtokens"]
46
+ frame_feat_norm = F.normalize(frame_feat, p=2, dim=1)
47
+
48
+ # Compute the similarity matrix
49
+ frame_feat_norm = frame_feat_norm.permute(1, 0, 2)
50
+ similarity_matrix = torch.bmm(
51
+ frame_feat_norm, frame_feat_norm.transpose(-1, -2)
52
+ )
53
+ similarity_matrix = similarity_matrix.mean(dim=0)
54
+ else:
55
+ frame_feat = frame_feat["x_norm_clstoken"]
56
+ frame_feat_norm = F.normalize(frame_feat, p=2, dim=1)
57
+ similarity_matrix = torch.mm(
58
+ frame_feat_norm, frame_feat_norm.transpose(-1, -2)
59
+ )
60
+
61
+ distance_matrix = 100 - similarity_matrix.clone()
62
+
63
+ # Ignore self-pairing
64
+ similarity_matrix.fill_diagonal_(-100)
65
+ similarity_sum = similarity_matrix.sum(dim=1)
66
+
67
+ # Find the most common frame
68
+ most_common_frame_index = torch.argmax(similarity_sum).item()
69
+
70
+ # Conduct FPS sampling starting from the most common frame
71
+ fps_idx = farthest_point_sampling(
72
+ distance_matrix, query_frame_num, most_common_frame_index
73
+ )
74
+
75
+ return fps_idx
76
+
77
+
78
+ def farthest_point_sampling(
79
+ distance_matrix, num_samples, most_common_frame_index=0
80
+ ):
81
+ """
82
+ Farthest point sampling algorithm to select diverse frames.
83
+
84
+ Args:
85
+ distance_matrix: Matrix of distances between frames
86
+ num_samples: Number of frames to select
87
+ most_common_frame_index: Index of the first frame to select
88
+
89
+ Returns:
90
+ List of selected frame indices
91
+ """
92
+ distance_matrix = distance_matrix.clamp(min=0)
93
+ N = distance_matrix.size(0)
94
+
95
+ # Initialize with the most common frame
96
+ selected_indices = [most_common_frame_index]
97
+ check_distances = distance_matrix[selected_indices]
98
+
99
+ while len(selected_indices) < num_samples:
100
+ # Find the farthest point from the current set of selected points
101
+ farthest_point = torch.argmax(check_distances)
102
+ selected_indices.append(farthest_point.item())
103
+
104
+ check_distances = distance_matrix[farthest_point]
105
+ # Mark already selected points to avoid selecting them again
106
+ check_distances[selected_indices] = 0
107
+
108
+ # Break if all points have been selected
109
+ if len(selected_indices) == N:
110
+ break
111
+
112
+ return selected_indices
113
+
114
+
115
+ def calculate_index_mappings(query_index, S, device=None):
116
+ """
117
+ Construct an order that switches [query_index] and [0]
118
+ so that the content of query_index would be placed at [0].
119
+
120
+ Args:
121
+ query_index: Index to swap with 0
122
+ S: Total number of elements
123
+ device: Device to place the tensor on
124
+
125
+ Returns:
126
+ Tensor of indices with the swapped order
127
+ """
128
+ new_order = torch.arange(S)
129
+ new_order[0] = query_index
130
+ new_order[query_index] = 0
131
+ if device is not None:
132
+ new_order = new_order.to(device)
133
+ return new_order
134
+
135
+
136
+ def switch_tensor_order(tensors, order, dim=1):
137
+ """
138
+ Reorder tensors along a specific dimension according to the given order.
139
+
140
+ Args:
141
+ tensors: List of tensors to reorder
142
+ order: Tensor of indices specifying the new order
143
+ dim: Dimension along which to reorder
144
+
145
+ Returns:
146
+ List of reordered tensors
147
+ """
148
+ return [
149
+ torch.index_select(tensor, dim, order) if tensor is not None else None
150
+ for tensor in tensors
151
+ ]
152
+
153
+
154
+ def predict_track(model, images, query_points, dtype=torch.bfloat16, use_tf32_for_track=True, iters=4):
155
+ """
156
+ Predict tracks for query points across frames.
157
+
158
+ Args:
159
+ model: VGGT model
160
+ images: Tensor of images of shape (S, 3, H, W)
161
+ query_points: Query points to track
162
+ dtype: Data type for computation
163
+ use_tf32_for_track: Whether to use TF32 precision for tracking
164
+ iters: Number of iterations for tracking
165
+
166
+ Returns:
167
+ Predicted tracks, visibility scores, and confidence scores
168
+ """
169
+ with torch.no_grad():
170
+ with torch.cuda.amp.autocast(dtype=dtype):
171
+ images = images[None] # add batch dimension
172
+ aggregated_tokens_list, ps_idx = model.aggregator(images)
173
+
174
+ if not use_tf32_for_track:
175
+ track_list, vis_score, conf_score = model.track_head(
176
+ aggregated_tokens_list, images, ps_idx, query_points=query_points, iters=iters
177
+ )
178
+
179
+ if use_tf32_for_track:
180
+ with torch.cuda.amp.autocast(enabled=False):
181
+ track_list, vis_score, conf_score = model.track_head(
182
+ aggregated_tokens_list, images, ps_idx, query_points=query_points, iters=iters
183
+ )
184
+
185
+ pred_track = track_list[-1]
186
+
187
+ return pred_track.squeeze(0), vis_score.squeeze(0), conf_score.squeeze(0)
188
+
189
+
190
+ def initialize_feature_extractors(max_query_num, det_thres, extractor_method="aliked", device="cuda"):
191
+ """
192
+ Initialize feature extractors that can be reused based on a method string.
193
+
194
+ Args:
195
+ max_query_num: Maximum number of keypoints to extract
196
+ det_thres: Detection threshold for keypoint extraction
197
+ extractor_method: String specifying which extractors to use (e.g., "aliked", "sp+sift", "aliked+sp+sift")
198
+ device: Device to run extraction on
199
+
200
+ Returns:
201
+ Dictionary of initialized extractors
202
+ """
203
+ extractors = {}
204
+ methods = extractor_method.lower().split('+')
205
+ active_extractors = len(methods)
206
+
207
+ for method in methods:
208
+ method = method.strip()
209
+ if method == "aliked":
210
+ aliked_max_points = max_query_num // active_extractors
211
+ aliked_extractor = ALIKED(max_num_keypoints=aliked_max_points, detection_threshold=det_thres)
212
+ extractors['aliked'] = aliked_extractor.to(device).eval()
213
+ elif method == "sp":
214
+ sp_max_points = max_query_num // active_extractors
215
+ sp_extractor = SuperPoint(max_num_keypoints=sp_max_points, detection_threshold=det_thres)
216
+ extractors['sp'] = sp_extractor.to(device).eval()
217
+ elif method == "sift":
218
+ sift_max_points = max_query_num // active_extractors
219
+ sift_extractor = SIFT(max_num_keypoints=sift_max_points)
220
+ extractors['sift'] = sift_extractor.to(device).eval()
221
+ else:
222
+ print(f"Warning: Unknown feature extractor '{method}', ignoring.")
223
+
224
+ if not extractors:
225
+ print(f"Warning: No valid extractors found in '{extractor_method}'. Using ALIKED by default.")
226
+ aliked_extractor = ALIKED(max_num_keypoints=max_query_num, detection_threshold=det_thres)
227
+ extractors['aliked'] = aliked_extractor.to(device).eval()
228
+
229
+ return extractors
230
+
231
+
232
+ def extract_keypoints(query_image, extractors):
233
+ """
234
+ Extract keypoints using pre-initialized feature extractors.
235
+
236
+ Args:
237
+ query_image: Input image tensor (3xHxW, range [0, 1])
238
+ extractors: Dictionary of initialized extractors
239
+
240
+ Returns:
241
+ Tensor of keypoint coordinates (1xNx2)
242
+ """
243
+ query_points_round = None
244
+
245
+ with torch.no_grad():
246
+ for extractor_name, extractor in extractors.items():
247
+ query_points_data = extractor.extract(query_image)
248
+ extractor_points = query_points_data["keypoints"].round()
249
+
250
+ if query_points_round is not None:
251
+ query_points_round = torch.cat([query_points_round, extractor_points], dim=1)
252
+ else:
253
+ query_points_round = extractor_points
254
+
255
+ return query_points_round
256
+
257
+
258
+ def run_vggt_with_ba(model, images, image_names=None, dtype=torch.bfloat16,
259
+ max_query_num=2048, det_thres=0.005, query_frame_num=3,
260
+ extractor_method="aliked+sp+sift",
261
+ max_reproj_error=4,
262
+ shared_camera=True,
263
+ ):
264
+ """
265
+ Run VGGT with bundle adjustment for pose estimation.
266
+
267
+ Args:
268
+ model: VGGT model
269
+ images: Tensor of images of shape (S, 3, H, W)
270
+ image_names: Optional list of image names
271
+ dtype: Data type for computation
272
+
273
+ Returns:
274
+ Predicted extrinsic camera parameters
275
+
276
+ TODO:
277
+ - [ ] Use VGGT's vit instead of dinov2 for rank generation
278
+ """
279
+ device = images.device
280
+ frame_num = images.shape[0]
281
+
282
+ # TODO: use vggt's vit instead of dinov2
283
+ # Select representative frames for feature extraction
284
+ query_frame_indexes = generate_rank_by_dino(
285
+ images, query_frame_num, image_size=518,
286
+ model_name="dinov2_vitb14_reg", device=device,
287
+ spatial_similarity=False
288
+ )
289
+
290
+ # Add the first image to the front if not already present
291
+ if 0 in query_frame_indexes:
292
+ query_frame_indexes.remove(0)
293
+ query_frame_indexes = [0, *query_frame_indexes]
294
+
295
+ # Get initial pose and depth predictions
296
+
297
+ with torch.no_grad(), torch.cuda.amp.autocast(enabled=False, dtype=torch.bfloat16):
298
+ aggregated_tokens_list, patch_start_idx = model.aggregator(images, intermediate_layer_idx=model.cfg.intermediate_layer_idx)
299
+ with torch.cuda.amp.autocast(enabled=False):
300
+ fp32_tokens = [token.float() for token in aggregated_tokens_list]
301
+ pred_all_pose_enc = model.camera_head(fp32_tokens)[-1]
302
+ pred_all_extrinsic, pred_all_intrinsic = pose_encoding_to_extri_intri(pred_all_pose_enc, images.shape[-2:])
303
+ pred_extrinsic = pred_all_extrinsic[0]
304
+ pred_intrinsic = pred_all_intrinsic[0]
305
+ depth_map, depth_conf = model.depth_head(aggregated_tokens_list, images=images, patch_start_idx=patch_start_idx)
306
+
307
+ world_points = unproject_depth_map_to_point_map(depth_map, pred_extrinsic, pred_intrinsic)
308
+ world_points = torch.from_numpy(world_points).to(device)
309
+ world_points_conf = depth_conf.to(device)
310
+
311
+ torch.cuda.empty_cache()
312
+
313
+ # Lists to store predictions
314
+ pred_tracks = []
315
+ pred_vis_scores = []
316
+ pred_conf_scores = []
317
+ pred_world_points = []
318
+ pred_world_points_conf = []
319
+
320
+ # Initialize feature extractors
321
+ extractors = initialize_feature_extractors(max_query_num, det_thres, extractor_method, device)
322
+
323
+ # Process each query frame
324
+ for query_index in query_frame_indexes:
325
+ query_image = images[query_index]
326
+ query_points_round = extract_keypoints(query_image, extractors)
327
+
328
+ # Reorder images to put query image first
329
+ reorder_index = calculate_index_mappings(query_index, frame_num, device=device)
330
+ reorder_images = switch_tensor_order([images], reorder_index, dim=0)[0]
331
+
332
+ # Track points across frames
333
+ reorder_tracks, reorder_vis_score, reorder_conf_score = predict_track(
334
+ model, reorder_images, query_points_round, dtype=dtype, use_tf32_for_track=True, iters=4
335
+ )
336
+
337
+ # Restore original order
338
+ pred_track, pred_vis, pred_score = switch_tensor_order(
339
+ [reorder_tracks, reorder_vis_score, reorder_conf_score], reorder_index, dim=0
340
+ )
341
+
342
+ pred_tracks.append(pred_track)
343
+ pred_vis_scores.append(pred_vis)
344
+ pred_conf_scores.append(pred_score)
345
+
346
+ # Get corresponding 3D points
347
+ query_points_round_long = query_points_round.squeeze(0).long()
348
+ query_world_points = world_points[query_index][
349
+ query_points_round_long[:, 1], query_points_round_long[:, 0]
350
+ ]
351
+ query_world_points_conf = world_points_conf[query_index][
352
+ query_points_round_long[:, 1], query_points_round_long[:, 0]
353
+ ]
354
+
355
+ pred_world_points.append(query_world_points)
356
+ pred_world_points_conf.append(query_world_points_conf)
357
+
358
+ # Concatenate prediction lists
359
+ pred_tracks = torch.cat(pred_tracks, dim=1)
360
+ pred_vis_scores = torch.cat(pred_vis_scores, dim=1)
361
+ pred_conf_scores = torch.cat(pred_conf_scores, dim=1)
362
+ pred_world_points = torch.cat(pred_world_points, dim=0)
363
+ pred_world_points_conf = torch.cat(pred_world_points_conf, dim=0)
364
+
365
+ # Filter points by confidence
366
+ filtered_flag = pred_world_points_conf > 1.5
367
+
368
+ if filtered_flag.sum() > 1024:
369
+ # well if the number of points is too small, we will not filter
370
+ pred_world_points = pred_world_points[filtered_flag]
371
+ pred_world_points_conf = pred_world_points_conf[filtered_flag]
372
+
373
+ pred_tracks = pred_tracks[:, filtered_flag]
374
+ pred_vis_scores = pred_vis_scores[:, filtered_flag]
375
+ pred_conf_scores = pred_conf_scores[:, filtered_flag]
376
+
377
+ torch.cuda.empty_cache()
378
+
379
+ # Bundle adjustment parameters
380
+ S, _, H, W = images.shape
381
+ image_size = torch.tensor([W, H], dtype=pred_tracks.dtype, device=device)
382
+
383
+ # Run bundle adjustment
384
+ reconstruction = batch_matrix_to_pycolmap(
385
+ pred_world_points,
386
+ pred_extrinsic,
387
+ pred_intrinsic,
388
+ pred_tracks,
389
+ image_size,
390
+ max_reproj_error=max_reproj_error,
391
+ shared_camera=shared_camera
392
+ )
393
+
394
+ ba_options = pycolmap.BundleAdjustmentOptions()
395
+ pycolmap.bundle_adjustment(reconstruction, ba_options)
396
+ _, updated_extrinsic, _, _ = pycolmap_to_batch_matrix(
397
+ reconstruction, device=device, camera_type="SIMPLE_PINHOLE"
398
+ )
399
+
400
+ return updated_extrinsic
401
+
402
+
403
+
404
+
405
+
406
+
407
+
src/utils/cropping.py ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (C) 2024-present Naver Corporation. All rights reserved.
2
+ # Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
3
+ #
4
+ # --------------------------------------------------------
5
+ # croppping utilities
6
+ # --------------------------------------------------------
7
+ import PIL.Image
8
+ import os
9
+ os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1"
10
+ import cv2 # noqa
11
+ import numpy as np # noqa
12
+ from src.utils.geometry import colmap_to_opencv_intrinsics, opencv_to_colmap_intrinsics # noqa
13
+ try:
14
+ lanczos = PIL.Image.Resampling.LANCZOS
15
+ bicubic = PIL.Image.Resampling.BICUBIC
16
+ except AttributeError:
17
+ lanczos = PIL.Image.LANCZOS
18
+ bicubic = PIL.Image.BICUBIC
19
+
20
+
21
+ class ImageList:
22
+ """ Convenience class to aply the same operation to a whole set of images.
23
+ """
24
+
25
+ def __init__(self, images):
26
+ if not isinstance(images, (tuple, list, set)):
27
+ images = [images]
28
+ self.images = []
29
+ for image in images:
30
+ if not isinstance(image, PIL.Image.Image):
31
+ image = PIL.Image.fromarray(image)
32
+ self.images.append(image)
33
+
34
+ def __len__(self):
35
+ return len(self.images)
36
+
37
+ def to_pil(self):
38
+ return tuple(self.images) if len(self.images) > 1 else self.images[0]
39
+
40
+ @property
41
+ def size(self):
42
+ sizes = [im.size for im in self.images]
43
+ assert all(sizes[0] == s for s in sizes)
44
+ return sizes[0]
45
+
46
+ def resize(self, *args, **kwargs):
47
+ return ImageList(self._dispatch('resize', *args, **kwargs))
48
+
49
+ def crop(self, *args, **kwargs):
50
+ return ImageList(self._dispatch('crop', *args, **kwargs))
51
+
52
+ def _dispatch(self, func, *args, **kwargs):
53
+ return [getattr(im, func)(*args, **kwargs) for im in self.images]
54
+
55
+
56
+ def rescale_image(image, camera_intrinsics, output_resolution, force=True):
57
+ """ Jointly rescale a (image, depthmap)
58
+ so that (out_width, out_height) >= output_res
59
+ """
60
+ image = ImageList(image)
61
+ input_resolution = np.array(image.size) # (W,H)
62
+ output_resolution = np.array(output_resolution)
63
+ # define output resolution
64
+ assert output_resolution.shape == (2,)
65
+ scale_final = max(output_resolution / image.size) + 1e-8
66
+ if scale_final >= 1 and not force: # image is already smaller than what is asked
67
+ return (image.to_pil(), camera_intrinsics)
68
+ output_resolution = np.floor(input_resolution * scale_final).astype(int)
69
+
70
+ # first rescale the image so that it contains the crop
71
+ image = image.resize(tuple(output_resolution), resample=lanczos if scale_final < 1 else bicubic)
72
+
73
+ # no offset here; simple rescaling
74
+ camera_intrinsics = camera_matrix_of_crop(
75
+ camera_intrinsics, input_resolution, output_resolution, scaling=scale_final)
76
+
77
+ return image.to_pil(), camera_intrinsics
78
+
79
+
80
+ def camera_matrix_of_crop(input_camera_matrix, input_resolution, output_resolution, scaling=1, offset_factor=0.5, offset=None):
81
+ # Margins to offset the origin
82
+ margins = np.asarray(input_resolution) * scaling - output_resolution
83
+ assert np.all(margins >= 0.0)
84
+ if offset is None:
85
+ offset = offset_factor * margins
86
+
87
+ # Generate new camera parameters
88
+ output_camera_matrix_colmap = opencv_to_colmap_intrinsics(input_camera_matrix)
89
+ output_camera_matrix_colmap[:2, :] *= scaling
90
+ output_camera_matrix_colmap[:2, 2] -= offset
91
+ output_camera_matrix = colmap_to_opencv_intrinsics(output_camera_matrix_colmap)
92
+
93
+ return output_camera_matrix
94
+
95
+
96
+ def crop_image(image, camera_intrinsics, crop_bbox):
97
+ """
98
+ Return a crop of the input view.
99
+ """
100
+ image = ImageList(image)
101
+ l, t, r, b = crop_bbox
102
+
103
+ image = image.crop((l, t, r, b))
104
+
105
+ camera_intrinsics = camera_intrinsics.copy()
106
+ camera_intrinsics[0, 2] -= l
107
+ camera_intrinsics[1, 2] -= t
108
+
109
+ return image.to_pil(), camera_intrinsics
110
+
111
+
112
+ def bbox_from_intrinsics_in_out(input_camera_matrix, output_camera_matrix, output_resolution):
113
+ out_width, out_height = output_resolution
114
+ l, t = np.int32(np.round(input_camera_matrix[:2, 2] - output_camera_matrix[:2, 2]))
115
+ crop_bbox = (l, t, l + out_width, t + out_height)
116
+ return crop_bbox
117
+
118
+ def rescale_image_depthmap(image, depthmap, camera_intrinsics, output_resolution, force=True):
119
+ """ Jointly rescale a (image, depthmap)
120
+ so that (out_width, out_height) >= output_res
121
+ """
122
+ image = ImageList(image)
123
+ input_resolution = np.array(image.size) # (W,H)
124
+ output_resolution = np.array(output_resolution)
125
+ if depthmap is not None:
126
+ # can also use this with masks instead of depthmaps
127
+ assert tuple(depthmap.shape[:2]) == image.size[::-1]
128
+
129
+ # define output resolution
130
+ assert output_resolution.shape == (2,)
131
+ scale_final = max(output_resolution / image.size) + 1e-8
132
+ if scale_final >= 1 and not force: # image is already smaller than what is asked
133
+ return (image.to_pil(), depthmap, camera_intrinsics)
134
+ output_resolution = np.floor(input_resolution * scale_final).astype(int)
135
+
136
+ # first rescale the image so that it contains the crop
137
+ # breakpoint()
138
+ image = image.resize(tuple(output_resolution), resample=lanczos if scale_final < 1 else bicubic)
139
+ if depthmap is not None:
140
+ depthmap = cv2.resize(depthmap, tuple(output_resolution), fx=scale_final,
141
+ fy=scale_final, interpolation=cv2.INTER_NEAREST)
142
+
143
+ # no offset here; simple rescaling
144
+ camera_intrinsics = camera_matrix_of_crop(
145
+ camera_intrinsics, input_resolution, tuple(output_resolution), scaling=scale_final)
146
+
147
+ return image.to_pil(), depthmap, camera_intrinsics
148
+
149
+ def crop_image_depthmap(image, depthmap, camera_intrinsics, crop_bbox):
150
+ """
151
+ Return a crop of the input view.
152
+ """
153
+ image = ImageList(image)
154
+ l, t, r, b = crop_bbox
155
+
156
+ image = image.crop((l, t, r, b))
157
+ depthmap = depthmap[t:b, l:r]
158
+
159
+ camera_intrinsics = camera_intrinsics.copy()
160
+ camera_intrinsics[0, 2] -= l
161
+ camera_intrinsics[1, 2] -= t
162
+
163
+ return image.to_pil(), depthmap, camera_intrinsics
src/utils/device.py ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+
4
+ # This source code is licensed under the license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ # Copyright (C) 2024-present Naver Corporation. All rights reserved.
8
+ # Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
9
+ #
10
+ # --------------------------------------------------------
11
+ # utilitary functions for DUSt3R
12
+ # --------------------------------------------------------
13
+ import numpy as np
14
+ import torch
15
+
16
+
17
+ def todevice(batch, device, callback=None, non_blocking=False):
18
+ """Transfer some variables to another device (i.e. GPU, CPU:torch, CPU:numpy).
19
+
20
+ batch: list, tuple, dict of tensors or other things
21
+ device: pytorch device or 'numpy'
22
+ callback: function that would be called on every sub-elements.
23
+ """
24
+ if callback:
25
+ batch = callback(batch)
26
+
27
+ if isinstance(batch, dict):
28
+ return {k: todevice(v, device) for k, v in batch.items()}
29
+
30
+ if isinstance(batch, (tuple, list)):
31
+ return type(batch)(todevice(x, device) for x in batch)
32
+
33
+ x = batch
34
+ if device == "numpy":
35
+ if isinstance(x, torch.Tensor):
36
+ x = x.detach().cpu().numpy()
37
+ elif x is not None:
38
+ if isinstance(x, np.ndarray):
39
+ x = torch.from_numpy(x)
40
+ if torch.is_tensor(x):
41
+ x = x.to(device, non_blocking=non_blocking)
42
+ return x
43
+
44
+
45
+ to_device = todevice # alias
46
+
47
+
48
+ def to_numpy(x):
49
+ return todevice(x, "numpy")
50
+
51
+
52
+ def to_cpu(x):
53
+ return todevice(x, "cpu")
54
+
55
+
56
+ def to_cuda(x):
57
+ return todevice(x, "cuda")
58
+
59
+
60
+ def collate_with_cat(whatever, lists=False):
61
+ if isinstance(whatever, dict):
62
+ return {k: collate_with_cat(vals, lists=lists) for k, vals in whatever.items()}
63
+
64
+ elif isinstance(whatever, (tuple, list)):
65
+ if len(whatever) == 0:
66
+ return whatever
67
+ elem = whatever[0]
68
+ T = type(whatever)
69
+
70
+ if elem is None:
71
+ return None
72
+ if isinstance(elem, (bool, float, int, str)):
73
+ return whatever
74
+ if isinstance(elem, tuple):
75
+ return T(collate_with_cat(x, lists=lists) for x in zip(*whatever))
76
+ if isinstance(elem, dict):
77
+ return {
78
+ k: collate_with_cat([e[k] for e in whatever], lists=lists) for k in elem
79
+ }
80
+
81
+ if isinstance(elem, torch.Tensor):
82
+ return listify(whatever) if lists else torch.cat(whatever)
83
+ if isinstance(elem, np.ndarray):
84
+ return (
85
+ listify(whatever)
86
+ if lists
87
+ else torch.cat([torch.from_numpy(x) for x in whatever])
88
+ )
89
+
90
+ # otherwise, we just chain lists
91
+ return sum(whatever, T())
92
+
93
+
94
+ def listify(elems):
95
+ return [x for e in elems for x in e]
src/utils/geometry.py ADDED
@@ -0,0 +1,428 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (C) 2024-present Naver Corporation. All rights reserved.
2
+ # Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
3
+ #
4
+ # --------------------------------------------------------
5
+ # geometry utilitary functions
6
+ # --------------------------------------------------------
7
+ import torch
8
+ import numpy as np
9
+ from scipy.spatial import cKDTree as KDTree
10
+
11
+ def invalid_to_nans(arr, valid_mask, ndim=999):
12
+ if valid_mask is not None:
13
+ arr = arr.clone()
14
+ arr[~valid_mask] = float('nan')
15
+ if arr.ndim > ndim:
16
+ arr = arr.flatten(-2 - (arr.ndim - ndim), -2)
17
+ return arr
18
+
19
+
20
+ def invalid_to_zeros(arr, valid_mask, ndim=999):
21
+ if valid_mask is not None:
22
+ arr = arr.clone()
23
+ arr[~valid_mask] = 0
24
+ nnz = valid_mask.view(len(valid_mask), -1).sum(1)
25
+ else:
26
+ nnz = arr.numel() // len(arr) if len(arr) else 0 # number of point per image
27
+ if arr.ndim > ndim:
28
+ arr = arr.flatten(-2 - (arr.ndim - ndim), -2)
29
+ return arr, nnz
30
+
31
+
32
+ def xy_grid(W, H, device=None, origin=(0, 0), unsqueeze=None, cat_dim=-1, homogeneous=False, **arange_kw):
33
+ """ Output a (H,W,2) array of int32
34
+ with output[j,i,0] = i + origin[0]
35
+ output[j,i,1] = j + origin[1]
36
+ """
37
+ if device is None:
38
+ # numpy
39
+ arange, meshgrid, stack, ones = np.arange, np.meshgrid, np.stack, np.ones
40
+ else:
41
+ # torch
42
+ arange = lambda *a, **kw: torch.arange(*a, device=device, **kw)
43
+ meshgrid, stack = torch.meshgrid, torch.stack
44
+ ones = lambda *a: torch.ones(*a, device=device)
45
+
46
+ tw, th = [arange(o, o + s, **arange_kw) for s, o in zip((W, H), origin)]
47
+ grid = meshgrid(tw, th, indexing='xy')
48
+ if homogeneous:
49
+ grid = grid + (ones((H, W)),)
50
+ if unsqueeze is not None:
51
+ grid = (grid[0].unsqueeze(unsqueeze), grid[1].unsqueeze(unsqueeze))
52
+ if cat_dim is not None:
53
+ grid = stack(grid, cat_dim)
54
+ return grid
55
+
56
+
57
+ def geotrf(Trf, pts, ncol=None, norm=False):
58
+ """ Apply a geometric transformation to a list of 3-D points.
59
+
60
+ H: 3x3 or 4x4 projection matrix (typically a Homography)
61
+ p: numpy/torch/tuple of coordinates. Shape must be (...,2) or (...,3)
62
+
63
+ ncol: int. number of columns of the result (2 or 3)
64
+ norm: float. if != 0, the resut is projected on the z=norm plane.
65
+
66
+ Returns an array of projected 2d points.
67
+ """
68
+ assert Trf.ndim >= 2
69
+ if isinstance(Trf, np.ndarray):
70
+ pts = np.asarray(pts)
71
+ elif isinstance(Trf, torch.Tensor):
72
+ pts = torch.as_tensor(pts, dtype=Trf.dtype)
73
+
74
+ # adapt shape if necessary
75
+ output_reshape = pts.shape[:-1]
76
+ ncol = ncol or pts.shape[-1]
77
+
78
+ # optimized code
79
+ if (isinstance(Trf, torch.Tensor) and isinstance(pts, torch.Tensor) and
80
+ Trf.ndim == 3 and pts.ndim == 4):
81
+ d = pts.shape[3]
82
+ if Trf.shape[-1] == d:
83
+ pts = torch.einsum("bij, bhwj -> bhwi", Trf, pts)
84
+ elif Trf.shape[-1] == d + 1:
85
+ pts = torch.einsum("bij, bhwj -> bhwi", Trf[:, :d, :d], pts) + Trf[:, None, None, :d, d]
86
+ else:
87
+ raise ValueError(f'bad shape, not ending with 3 or 4, for {pts.shape=}')
88
+ else:
89
+ if Trf.ndim >= 3:
90
+ n = Trf.ndim - 2
91
+ assert Trf.shape[:n] == pts.shape[:n], 'batch size does not match'
92
+ Trf = Trf.reshape(-1, Trf.shape[-2], Trf.shape[-1])
93
+
94
+ if pts.ndim > Trf.ndim:
95
+ # Trf == (B,d,d) & pts == (B,H,W,d) --> (B, H*W, d)
96
+ pts = pts.reshape(Trf.shape[0], -1, pts.shape[-1])
97
+ elif pts.ndim == 2:
98
+ # Trf == (B,d,d) & pts == (B,d) --> (B, 1, d)
99
+ pts = pts[:, None, :]
100
+
101
+ if pts.shape[-1] + 1 == Trf.shape[-1]:
102
+ Trf = Trf.swapaxes(-1, -2) # transpose Trf
103
+ pts = pts @ Trf[..., :-1, :] + Trf[..., -1:, :]
104
+ elif pts.shape[-1] == Trf.shape[-1]:
105
+ Trf = Trf.swapaxes(-1, -2) # transpose Trf
106
+ pts = pts @ Trf
107
+ else:
108
+ pts = Trf @ pts.T
109
+ if pts.ndim >= 2:
110
+ pts = pts.swapaxes(-1, -2)
111
+
112
+ if norm:
113
+ pts = pts / pts[..., -1:] # DONT DO /= BECAUSE OF WEIRD PYTORCH BUG
114
+ if norm != 1:
115
+ pts *= norm
116
+
117
+ res = pts[..., :ncol].reshape(*output_reshape, ncol)
118
+ return res
119
+
120
+
121
+ def inv(mat):
122
+ """ Invert a torch or numpy matrix
123
+ """
124
+ if isinstance(mat, torch.Tensor):
125
+ return torch.linalg.inv(mat)
126
+ if isinstance(mat, np.ndarray):
127
+ return np.linalg.inv(mat)
128
+ raise ValueError(f'bad matrix type = {type(mat)}')
129
+
130
+
131
+ def depthmap_to_pts3d(depth, pseudo_focal, pp=None, **_):
132
+ """
133
+ Args:
134
+ - depthmap (BxHxW array):
135
+ - pseudo_focal: [B,H,W] ; [B,2,H,W] or [B,1,H,W]
136
+ Returns:
137
+ pointmap of absolute coordinates (BxHxWx3 array)
138
+ """
139
+
140
+ if len(depth.shape) == 4:
141
+ B, H, W, n = depth.shape
142
+ else:
143
+ B, H, W = depth.shape
144
+ n = None
145
+
146
+ if len(pseudo_focal.shape) == 3: # [B,H,W]
147
+ pseudo_focalx = pseudo_focaly = pseudo_focal
148
+ elif len(pseudo_focal.shape) == 4: # [B,2,H,W] or [B,1,H,W]
149
+ pseudo_focalx = pseudo_focal[:, 0]
150
+ if pseudo_focal.shape[1] == 2:
151
+ pseudo_focaly = pseudo_focal[:, 1]
152
+ else:
153
+ pseudo_focaly = pseudo_focalx
154
+ else:
155
+ raise NotImplementedError("Error, unknown input focal shape format.")
156
+
157
+ assert pseudo_focalx.shape == depth.shape[:3]
158
+ assert pseudo_focaly.shape == depth.shape[:3]
159
+ grid_x, grid_y = xy_grid(W, H, cat_dim=0, device=depth.device)[:, None]
160
+
161
+ # set principal point
162
+ if pp is None:
163
+ grid_x = grid_x - (W - 1) / 2
164
+ grid_y = grid_y - (H - 1) / 2
165
+ else:
166
+ grid_x = grid_x.expand(B, -1, -1) - pp[:, 0, None, None]
167
+ grid_y = grid_y.expand(B, -1, -1) - pp[:, 1, None, None]
168
+
169
+ if n is None:
170
+ pts3d = torch.empty((B, H, W, 3), device=depth.device)
171
+ pts3d[..., 0] = depth * grid_x / pseudo_focalx
172
+ pts3d[..., 1] = depth * grid_y / pseudo_focaly
173
+ pts3d[..., 2] = depth
174
+ else:
175
+ pts3d = torch.empty((B, H, W, 3, n), device=depth.device)
176
+ pts3d[..., 0, :] = depth * (grid_x / pseudo_focalx)[..., None]
177
+ pts3d[..., 1, :] = depth * (grid_y / pseudo_focaly)[..., None]
178
+ pts3d[..., 2, :] = depth
179
+ return pts3d
180
+
181
+
182
+ def depthmap_to_camera_coordinates(depthmap, camera_intrinsics, pseudo_focal=None):
183
+ """
184
+ Args:
185
+ - depthmap (HxW array):
186
+ - camera_intrinsics: a 3x3 matrix
187
+ Returns:
188
+ pointmap of absolute coordinates (HxWx3 array), and a mask specifying valid pixels.
189
+ """
190
+ camera_intrinsics = np.float32(camera_intrinsics)
191
+ H, W = depthmap.shape
192
+
193
+ # Compute 3D ray associated with each pixel
194
+ # Strong assumption: there are no skew terms
195
+ assert camera_intrinsics[0, 1] == 0.0
196
+ assert camera_intrinsics[1, 0] == 0.0
197
+ if pseudo_focal is None:
198
+ fu = camera_intrinsics[0, 0]
199
+ fv = camera_intrinsics[1, 1]
200
+ else:
201
+ assert pseudo_focal.shape == (H, W)
202
+ fu = fv = pseudo_focal
203
+ cu = camera_intrinsics[0, 2]
204
+ cv = camera_intrinsics[1, 2]
205
+
206
+ u, v = np.meshgrid(np.arange(W), np.arange(H))
207
+ z_cam = depthmap
208
+ x_cam = (u - cu) * z_cam / fu
209
+ y_cam = (v - cv) * z_cam / fv
210
+ X_cam = np.stack((x_cam, y_cam, z_cam), axis=-1).astype(np.float32)
211
+
212
+ # Mask for valid coordinates
213
+ valid_mask = (depthmap > 0.0)
214
+ return X_cam, valid_mask
215
+
216
+
217
+ def depthmap_to_absolute_camera_coordinates(depthmap, camera_intrinsics, camera_pose=None, **kw):
218
+ """
219
+ Args:
220
+ - depthmap (HxW array):
221
+ - camera_intrinsics: a 3x3 matrix
222
+ - camera_pose: a 4x3 or 4x4 cam2world matrix
223
+ Returns:
224
+ pointmap of absolute coordinates (HxWx3 array), and a mask specifying valid pixels."""
225
+ X_cam, valid_mask = depthmap_to_camera_coordinates(depthmap, camera_intrinsics)
226
+
227
+ X_world = X_cam # default
228
+ if camera_pose is not None:
229
+ # R_cam2world = np.float32(camera_params["R_cam2world"])
230
+ # t_cam2world = np.float32(camera_params["t_cam2world"]).squeeze()
231
+ R_cam2world = camera_pose[:3, :3]
232
+ t_cam2world = camera_pose[:3, 3]
233
+
234
+ # Express in absolute coordinates (invalid depth values)
235
+ X_world = np.einsum("ik, vuk -> vui", R_cam2world, X_cam) + t_cam2world[None, None, :]
236
+
237
+ return X_world, valid_mask
238
+
239
+
240
+ def colmap_to_opencv_intrinsics(K):
241
+ """
242
+ Modify camera intrinsics to follow a different convention.
243
+ Coordinates of the center of the top-left pixels are by default:
244
+ - (0.5, 0.5) in Colmap
245
+ - (0,0) in OpenCV
246
+ """
247
+ K = K.copy()
248
+ K[0, 2] -= 0.5
249
+ K[1, 2] -= 0.5
250
+ return K
251
+
252
+
253
+ def opencv_to_colmap_intrinsics(K):
254
+ """
255
+ Modify camera intrinsics to follow a different convention.
256
+ Coordinates of the center of the top-left pixels are by default:
257
+ - (0.5, 0.5) in Colmap
258
+ - (0,0) in OpenCV
259
+ """
260
+ K = K.copy()
261
+ K[0, 2] += 0.5
262
+ K[1, 2] += 0.5
263
+ return K
264
+
265
+
266
+ def normalize_pointcloud(pts1, pts2, norm_mode='avg_dis', valid1=None, valid2=None, ret_factor=False):
267
+ """ renorm pointmaps pts1, pts2 with norm_mode
268
+ """
269
+ assert pts1.ndim >= 3 and pts1.shape[-1] == 3
270
+ assert pts2 is None or (pts2.ndim >= 3 and pts2.shape[-1] == 3)
271
+ norm_mode, dis_mode = norm_mode.split('_')
272
+
273
+ if norm_mode == 'avg':
274
+ # gather all points together (joint normalization)
275
+ nan_pts1, nnz1 = invalid_to_zeros(pts1, valid1, ndim=3)
276
+ nan_pts2, nnz2 = invalid_to_zeros(pts2, valid2, ndim=3) if pts2 is not None else (None, 0)
277
+ all_pts = torch.cat((nan_pts1, nan_pts2), dim=1) if pts2 is not None else nan_pts1
278
+
279
+ # compute distance to origin
280
+ all_dis = all_pts.norm(dim=-1)
281
+ if dis_mode == 'dis':
282
+ pass # do nothing
283
+ elif dis_mode == 'log1p':
284
+ all_dis = torch.log1p(all_dis)
285
+ elif dis_mode == 'warp-log1p':
286
+ # actually warp input points before normalizing them
287
+ log_dis = torch.log1p(all_dis)
288
+ warp_factor = log_dis / all_dis.clip(min=1e-8)
289
+ H1, W1 = pts1.shape[1:-1]
290
+ pts1 = pts1 * warp_factor[:, :W1 * H1].view(-1, H1, W1, 1)
291
+ if pts2 is not None:
292
+ H2, W2 = pts2.shape[1:-1]
293
+ pts2 = pts2 * warp_factor[:, W1 * H1:].view(-1, H2, W2, 1)
294
+ all_dis = log_dis # this is their true distance afterwards
295
+ else:
296
+ raise ValueError(f'bad {dis_mode=}')
297
+
298
+ norm_factor = all_dis.sum(dim=1) / (nnz1 + nnz2 + 1e-8)
299
+ else:
300
+ # gather all points together (joint normalization)
301
+ nan_pts1 = invalid_to_nans(pts1, valid1, ndim=3)
302
+ nan_pts2 = invalid_to_nans(pts2, valid2, ndim=3) if pts2 is not None else None
303
+ all_pts = torch.cat((nan_pts1, nan_pts2), dim=1) if pts2 is not None else nan_pts1
304
+
305
+ # compute distance to origin
306
+ all_dis = all_pts.norm(dim=-1)
307
+
308
+ if norm_mode == 'avg':
309
+ norm_factor = all_dis.nanmean(dim=1)
310
+ elif norm_mode == 'median':
311
+ norm_factor = all_dis.nanmedian(dim=1).values.detach()
312
+ elif norm_mode == 'sqrt':
313
+ norm_factor = all_dis.sqrt().nanmean(dim=1)**2
314
+ else:
315
+ raise ValueError(f'bad {norm_mode=}')
316
+
317
+ norm_factor = norm_factor.clip(min=1e-8)
318
+ while norm_factor.ndim < pts1.ndim:
319
+ norm_factor.unsqueeze_(-1)
320
+
321
+ res = pts1 / norm_factor
322
+ if pts2 is not None:
323
+ res = (res, pts2 / norm_factor)
324
+ if ret_factor:
325
+ res = res + (norm_factor,)
326
+ return res
327
+
328
+
329
+ @torch.no_grad()
330
+ def get_joint_pointcloud_depth(z1, z2, valid_mask1, valid_mask2=None, quantile=0.5):
331
+ # set invalid points to NaN
332
+ _z1 = invalid_to_nans(z1, valid_mask1).reshape(len(z1), -1)
333
+ _z2 = invalid_to_nans(z2, valid_mask2).reshape(len(z2), -1) if z2 is not None else None
334
+ _z = torch.cat((_z1, _z2), dim=-1) if z2 is not None else _z1
335
+
336
+ # compute median depth overall (ignoring nans)
337
+ if quantile == 0.5:
338
+ shift_z = torch.nanmedian(_z, dim=-1).values
339
+ else:
340
+ shift_z = torch.nanquantile(_z, quantile, dim=-1)
341
+ return shift_z # (B,)
342
+
343
+
344
+ @torch.no_grad()
345
+ def get_joint_pointcloud_center_scale(pts1, pts2, valid_mask1=None, valid_mask2=None, z_only=False, center=True):
346
+ # set invalid points to NaN
347
+ _pts1 = invalid_to_nans(pts1, valid_mask1).reshape(len(pts1), -1, 3)
348
+ _pts2 = invalid_to_nans(pts2, valid_mask2).reshape(len(pts2), -1, 3) if pts2 is not None else None
349
+ _pts = torch.cat((_pts1, _pts2), dim=1) if pts2 is not None else _pts1
350
+
351
+ # compute median center
352
+ _center = torch.nanmedian(_pts, dim=1, keepdim=True).values # (B,1,3)
353
+ if z_only:
354
+ _center[..., :2] = 0 # do not center X and Y
355
+
356
+ # compute median norm
357
+ _norm = ((_pts - _center) if center else _pts).norm(dim=-1)
358
+ scale = torch.nanmedian(_norm, dim=1).values
359
+ return _center[:, None, :, :], scale[:, None, None, None]
360
+
361
+
362
+ def find_reciprocal_matches(P1, P2):
363
+ """
364
+ returns 3 values:
365
+ 1 - reciprocal_in_P2: a boolean array of size P2.shape[0], a "True" value indicates a match
366
+ 2 - nn2_in_P1: a int array of size P2.shape[0], it contains the indexes of the closest points in P1
367
+ 3 - reciprocal_in_P2.sum(): the number of matches
368
+ """
369
+ tree1 = KDTree(P1)
370
+ tree2 = KDTree(P2)
371
+
372
+ _, nn1_in_P2 = tree2.query(P1, workers=8)
373
+ _, nn2_in_P1 = tree1.query(P2, workers=8)
374
+
375
+ reciprocal_in_P1 = (nn2_in_P1[nn1_in_P2] == np.arange(len(nn1_in_P2)))
376
+ reciprocal_in_P2 = (nn1_in_P2[nn2_in_P1] == np.arange(len(nn2_in_P1)))
377
+ assert reciprocal_in_P1.sum() == reciprocal_in_P2.sum()
378
+ return reciprocal_in_P2, nn2_in_P1, reciprocal_in_P2.sum()
379
+
380
+
381
+ def get_med_dist_between_poses(poses):
382
+ from scipy.spatial.distance import pdist
383
+ return np.median(pdist([p[:3, 3].detach().cpu().numpy() for p in poses]))
384
+
385
+ def get_rel_pos(_rel_xyz, free_space, voxel_size):
386
+ """
387
+ Args:
388
+ _rel_xyz: torch.tensor
389
+ shape [..., 3]
390
+ free_space: str
391
+ something like 'soft'
392
+ grid: GridBatch
393
+ fvdb grid
394
+
395
+ Returns:
396
+ rel_pos: torch.tensor
397
+ shape [..., 3], the position within a voxel (compared with the voxel center)
398
+ """
399
+ if free_space == "hard":
400
+ rel_pos = torch.sigmoid(_rel_xyz) * voxel_size
401
+ elif free_space == "soft":
402
+ # free space [-1, 2]
403
+ rel_pos = (torch.sigmoid(_rel_xyz) * 3 - 1) * voxel_size
404
+ elif free_space == "soft-2":
405
+ # free space [-2, 3]
406
+ rel_pos = (torch.sigmoid(_rel_xyz) * 5 - 2) * voxel_size
407
+ elif free_space == "soft-3":
408
+ # free space [-3, 4]
409
+ rel_pos = (torch.sigmoid(_rel_xyz) * 7 - 3) * voxel_size
410
+ elif free_space == "soft-4":
411
+ # free space [-4, 5]
412
+ rel_pos = (torch.sigmoid(_rel_xyz) * 9 - 4) * voxel_size
413
+ elif free_space == "soft-5":
414
+ # free space [-5, 6]
415
+ rel_pos = (torch.sigmoid(_rel_xyz) * 11 - 5) * voxel_size
416
+ elif free_space == "tanh-3":
417
+ # free space [-2.5, 3.5]
418
+ rel_pos = (torch.tanh(_rel_xyz) * 3 + 0.5) * voxel_size
419
+ elif free_space == "free-1":
420
+ rel_pos = _rel_xyz * voxel_size
421
+ elif free_space == "free-2":
422
+ rel_pos = _rel_xyz
423
+ elif free_space == "center":
424
+ rel_pos = (torch.zeros_like(_rel_xyz) + 0.5) * voxel_size
425
+ else:
426
+ raise NotImplementedError
427
+
428
+ return rel_pos
src/utils/image.py ADDED
@@ -0,0 +1,185 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+
4
+ # This source code is licensed under the license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ # Copyright (C) 2024-present Naver Corporation. All rights reserved.
8
+ # Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
9
+ #
10
+ # --------------------------------------------------------
11
+ # utilitary functions about images (loading/converting...)
12
+ # --------------------------------------------------------
13
+ import os
14
+
15
+ import numpy as np
16
+ import PIL.Image
17
+ import torch
18
+ import torchvision.transforms as tvf
19
+ from PIL.ImageOps import exif_transpose
20
+ from PIL import Image
21
+ import torchvision
22
+
23
+ os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1"
24
+ import cv2
25
+
26
+ try:
27
+ from pillow_heif import register_heif_opener
28
+
29
+ register_heif_opener()
30
+ heif_support_enabled = True
31
+ except ImportError:
32
+ heif_support_enabled = False
33
+
34
+ ImgNorm = tvf.Compose([tvf.ToTensor(), tvf.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
35
+
36
+
37
+ def imread_cv2(path, options=cv2.IMREAD_COLOR):
38
+ """Open an image or a depthmap with opencv-python."""
39
+ if path.endswith((".exr", "EXR")):
40
+ options = cv2.IMREAD_ANYDEPTH
41
+ img = cv2.imread(path, options)
42
+ if img is None:
43
+ raise IOError(f"Could not load image={path} with {options=}")
44
+ if img.ndim == 3:
45
+ img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
46
+ return img
47
+
48
+
49
+ def rgb(ftensor, true_shape=None):
50
+ if isinstance(ftensor, list):
51
+ return [rgb(x, true_shape=true_shape) for x in ftensor]
52
+ if isinstance(ftensor, torch.Tensor):
53
+ ftensor = ftensor.detach().cpu().numpy() # H,W,3
54
+ if ftensor.ndim == 3 and ftensor.shape[0] == 3:
55
+ ftensor = ftensor.transpose(1, 2, 0)
56
+ elif ftensor.ndim == 4 and ftensor.shape[1] == 3:
57
+ ftensor = ftensor.transpose(0, 2, 3, 1)
58
+ if true_shape is not None:
59
+ H, W = true_shape
60
+ ftensor = ftensor[:H, :W]
61
+ if ftensor.dtype == np.uint8:
62
+ img = np.float32(ftensor) / 255
63
+ else:
64
+ img = (ftensor * 0.5) + 0.5
65
+ return img.clip(min=0, max=1)
66
+
67
+
68
+ def _resize_pil_image(img, long_edge_size):
69
+ S = max(img.size)
70
+ if S > long_edge_size:
71
+ interp = PIL.Image.LANCZOS
72
+ elif S <= long_edge_size:
73
+ interp = PIL.Image.BICUBIC
74
+ new_size = tuple(int(round(x * long_edge_size / S)) for x in img.size)
75
+ return img.resize(new_size, interp)
76
+
77
+
78
+ def load_images(folder_or_list, size, square_ok=False, verbose=True, rotate_clockwise_90=False, crop_to_landscape=False):
79
+ """open and convert all images in a list or folder to proper input format for DUSt3R"""
80
+ if isinstance(folder_or_list, str):
81
+ if verbose:
82
+ print(f">> Loading images from {folder_or_list}")
83
+ root, folder_content = folder_or_list, sorted(os.listdir(folder_or_list))
84
+
85
+ elif isinstance(folder_or_list, list):
86
+ if verbose:
87
+ print(f">> Loading a list of {len(folder_or_list)} images")
88
+ root, folder_content = "", folder_or_list
89
+
90
+ else:
91
+ raise ValueError(f"bad {folder_or_list=} ({type(folder_or_list)})")
92
+
93
+ supported_images_extensions = [".jpg", ".jpeg", ".png"]
94
+ if heif_support_enabled:
95
+ supported_images_extensions += [".heic", ".heif"]
96
+ supported_images_extensions = tuple(supported_images_extensions)
97
+
98
+ imgs = []
99
+ for path in folder_content:
100
+ if not path.lower().endswith(supported_images_extensions):
101
+ continue
102
+ img = exif_transpose(PIL.Image.open(os.path.join(root, path))).convert("RGB")
103
+ if rotate_clockwise_90:
104
+ img = img.rotate(-90, expand=True)
105
+ if crop_to_landscape:
106
+ # Crop to a landscape aspect ratio (e.g., 16:9)
107
+ desired_aspect_ratio = 4 / 3
108
+ width, height = img.size
109
+ current_aspect_ratio = width / height
110
+
111
+ if current_aspect_ratio > desired_aspect_ratio:
112
+ # Wider than landscape: crop width
113
+ new_width = int(height * desired_aspect_ratio)
114
+ left = (width - new_width) // 2
115
+ right = left + new_width
116
+ top = 0
117
+ bottom = height
118
+ else:
119
+ # Taller than landscape: crop height
120
+ new_height = int(width / desired_aspect_ratio)
121
+ top = (height - new_height) // 2
122
+ bottom = top + new_height
123
+ left = 0
124
+ right = width
125
+
126
+ img = img.crop((left, top, right, bottom))
127
+
128
+ W1, H1 = img.size
129
+ if size == 224:
130
+ # resize short side to 224 (then crop)
131
+ img = _resize_pil_image(img, round(size * max(W1 / H1, H1 / W1)))
132
+ else:
133
+ # resize long side to 512
134
+ img = _resize_pil_image(img, size)
135
+ W, H = img.size
136
+ cx, cy = W // 2, H // 2
137
+ if size == 224:
138
+ half = min(cx, cy)
139
+ img = img.crop((cx - half, cy - half, cx + half, cy + half))
140
+ else:
141
+ halfw, halfh = ((2 * cx) // 16) * 8, ((2 * cy) // 16) * 8
142
+ if not (square_ok) and W == H:
143
+ halfh = 3 * halfw / 4
144
+ img = img.crop((cx - halfw, cy - halfh, cx + halfw, cy + halfh))
145
+
146
+ W2, H2 = img.size
147
+ if verbose:
148
+ print(f" - adding {path} with resolution {W1}x{H1} --> {W2}x{H2}")
149
+ imgs.append(
150
+ dict(
151
+ img=ImgNorm(img)[None],
152
+ true_shape=np.int32([img.size[::-1]]),
153
+ idx=len(imgs),
154
+ instance=str(len(imgs)),
155
+ )
156
+ )
157
+
158
+ assert imgs, "no images foud at " + root
159
+ if verbose:
160
+ print(f" (Found {len(imgs)} images)")
161
+ return imgs
162
+
163
+ def process_image(img_path):
164
+ img = Image.open(img_path)
165
+ if img.mode == 'RGBA':
166
+ # Convert RGBA to RGB by removing alpha channel
167
+ img = img.convert('RGB')
168
+ # Resize to maintain aspect ratio and then center crop to 448x448
169
+ width, height = img.size
170
+ if width > height:
171
+ new_height = 448
172
+ new_width = int(width * (new_height / height))
173
+ else:
174
+ new_width = 448
175
+ new_height = int(height * (new_width / width))
176
+ img = img.resize((new_width, new_height))
177
+
178
+ # Center crop
179
+ left = (new_width - 448) // 2
180
+ top = (new_height - 448) // 2
181
+ right = left + 448
182
+ bottom = top + 448
183
+ img = img.crop((left, top, right, bottom))
184
+ img_tensor = torchvision.transforms.ToTensor()(img) * 2.0 - 1.0 # [-1, 1]
185
+ return img_tensor
src/utils/misc.py ADDED
@@ -0,0 +1,531 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+
4
+ # This source code is licensed under the license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ # Copyright (C) 2022-present Naver Corporation. All rights reserved.
8
+ # Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
9
+ #
10
+ # --------------------------------------------------------
11
+ # utilitary functions for CroCo
12
+ # --------------------------------------------------------
13
+ # References:
14
+ # MAE: https://github.com/facebookresearch/mae
15
+ # DeiT: https://github.com/facebookresearch/deit
16
+ # BEiT: https://github.com/microsoft/unilm/tree/master/beit
17
+ # --------------------------------------------------------
18
+
19
+ import builtins
20
+ import datetime
21
+ import json
22
+ import math
23
+ import os
24
+ import time
25
+ from collections import defaultdict, deque
26
+ from pathlib import Path
27
+
28
+ import numpy as np
29
+ import torch
30
+ import torch.distributed as dist
31
+ from torch import inf
32
+
33
+
34
+ class SmoothedValue(object):
35
+ """Track a series of values and provide access to smoothed values over a
36
+ window or the global series average.
37
+ """
38
+
39
+ def __init__(self, window_size=20, fmt=None):
40
+ if fmt is None:
41
+ fmt = "{median:.4f} ({global_avg:.4f})"
42
+ self.deque = deque(maxlen=window_size)
43
+ self.total = 0.0
44
+ self.count = 0
45
+ self.fmt = fmt
46
+
47
+ def update(self, value, n=1):
48
+ self.deque.append(value)
49
+ self.count += n
50
+ self.total += value * n
51
+
52
+ def synchronize_between_processes(self):
53
+ """
54
+ Warning: does not synchronize the deque!
55
+ """
56
+ if not is_dist_avail_and_initialized():
57
+ return
58
+ t = torch.tensor([self.count, self.total], dtype=torch.float64, device="cuda")
59
+ dist.barrier()
60
+ dist.all_reduce(t)
61
+ t = t.tolist()
62
+ self.count = int(t[0])
63
+ self.total = t[1]
64
+
65
+ @property
66
+ def median(self):
67
+ d = torch.tensor(list(self.deque))
68
+ return d.median().item()
69
+
70
+ @property
71
+ def avg(self):
72
+ d = torch.tensor(list(self.deque), dtype=torch.float32)
73
+ return d.mean().item()
74
+
75
+ @property
76
+ def global_avg(self):
77
+ return self.total / self.count
78
+
79
+ @property
80
+ def max(self):
81
+ return max(self.deque)
82
+
83
+ @property
84
+ def value(self):
85
+ return self.deque[-1]
86
+
87
+ def __str__(self):
88
+ return self.fmt.format(
89
+ median=self.median,
90
+ avg=self.avg,
91
+ global_avg=self.global_avg,
92
+ max=self.max,
93
+ value=self.value,
94
+ )
95
+
96
+
97
+ class MetricLogger(object):
98
+ def __init__(self, delimiter="\t"):
99
+ self.meters = defaultdict(SmoothedValue)
100
+ self.delimiter = delimiter
101
+
102
+ def update(self, **kwargs):
103
+ for k, v in kwargs.items():
104
+ if v is None:
105
+ continue
106
+ if isinstance(v, torch.Tensor):
107
+ v = v.item()
108
+ assert isinstance(v, (float, int))
109
+ self.meters[k].update(v)
110
+
111
+ def __getattr__(self, attr):
112
+ if attr in self.meters:
113
+ return self.meters[attr]
114
+ if attr in self.__dict__:
115
+ return self.__dict__[attr]
116
+ raise AttributeError(
117
+ "'{}' object has no attribute '{}'".format(type(self).__name__, attr)
118
+ )
119
+
120
+ def __str__(self):
121
+ loss_str = []
122
+ for name, meter in self.meters.items():
123
+ loss_str.append("{}: {}".format(name, str(meter)))
124
+ return self.delimiter.join(loss_str)
125
+
126
+ def synchronize_between_processes(self):
127
+ for meter in self.meters.values():
128
+ meter.synchronize_between_processes()
129
+
130
+ def add_meter(self, name, meter):
131
+ self.meters[name] = meter
132
+
133
+ def log_every(self, iterable, print_freq, header=None, max_iter=None):
134
+ i = 0
135
+ if not header:
136
+ header = ""
137
+ start_time = time.time()
138
+ end = time.time()
139
+ iter_time = SmoothedValue(fmt="{avg:.4f}")
140
+ data_time = SmoothedValue(fmt="{avg:.4f}")
141
+ len_iterable = min(len(iterable), max_iter) if max_iter else len(iterable)
142
+ space_fmt = ":" + str(len(str(len_iterable))) + "d"
143
+ log_msg = [
144
+ header,
145
+ "[{0" + space_fmt + "}/{1}]",
146
+ "eta: {eta}",
147
+ "{meters}",
148
+ "time: {time}",
149
+ "data: {data}",
150
+ ]
151
+ if torch.cuda.is_available():
152
+ log_msg.append("max mem: {memory:.0f}")
153
+ log_msg = self.delimiter.join(log_msg)
154
+ MB = 1024.0 * 1024.0
155
+ for it, obj in enumerate(iterable):
156
+ data_time.update(time.time() - end)
157
+ yield obj
158
+ iter_time.update(time.time() - end)
159
+ if i % print_freq == 0 or i == len_iterable - 1:
160
+ eta_seconds = iter_time.global_avg * (len_iterable - i)
161
+ eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
162
+ if torch.cuda.is_available():
163
+ print(
164
+ log_msg.format(
165
+ i,
166
+ len_iterable,
167
+ eta=eta_string,
168
+ meters=str(self),
169
+ time=str(iter_time),
170
+ data=str(data_time),
171
+ memory=torch.cuda.max_memory_allocated() / MB,
172
+ )
173
+ )
174
+ else:
175
+ print(
176
+ log_msg.format(
177
+ i,
178
+ len_iterable,
179
+ eta=eta_string,
180
+ meters=str(self),
181
+ time=str(iter_time),
182
+ data=str(data_time),
183
+ )
184
+ )
185
+ i += 1
186
+ end = time.time()
187
+ if max_iter and it >= max_iter:
188
+ break
189
+ total_time = time.time() - start_time
190
+ total_time_str = str(datetime.timedelta(seconds=int(total_time)))
191
+ print(
192
+ "{} Total time: {} ({:.4f} s / it)".format(
193
+ header, total_time_str, total_time / len_iterable
194
+ )
195
+ )
196
+
197
+
198
+ def setup_for_distributed(is_master):
199
+ """
200
+ This function disables printing when not in master process
201
+ """
202
+ builtin_print = builtins.print
203
+
204
+ def print(*args, **kwargs):
205
+ force = kwargs.pop("force", False)
206
+ force = force or (get_world_size() > 8)
207
+ if is_master or force:
208
+ now = datetime.datetime.now().time()
209
+ builtin_print("[{}] ".format(now), end="") # print with time stamp
210
+ builtin_print(*args, **kwargs)
211
+
212
+ builtins.print = print
213
+
214
+
215
+ def is_dist_avail_and_initialized():
216
+ if not dist.is_available():
217
+ return False
218
+ if not dist.is_initialized():
219
+ return False
220
+ return True
221
+
222
+
223
+ def get_world_size():
224
+ if not is_dist_avail_and_initialized():
225
+ return 1
226
+ return dist.get_world_size()
227
+
228
+
229
+ def get_rank():
230
+ if not is_dist_avail_and_initialized():
231
+ return 0
232
+ return dist.get_rank()
233
+
234
+
235
+ def is_main_process():
236
+ return get_rank() == 0
237
+
238
+
239
+ def save_on_master(*args, **kwargs):
240
+ if is_main_process():
241
+ torch.save(*args, **kwargs)
242
+
243
+
244
+ def init_distributed_mode(args):
245
+ nodist = args.nodist if hasattr(args, "nodist") else False
246
+ if "RANK" in os.environ and "WORLD_SIZE" in os.environ and not nodist:
247
+ args.rank = int(os.environ["RANK"])
248
+ args.world_size = int(os.environ["WORLD_SIZE"])
249
+ args.gpu = int(os.environ["LOCAL_RANK"])
250
+ else:
251
+ print("Not using distributed mode")
252
+ setup_for_distributed(is_master=True) # hack
253
+ args.distributed = False
254
+ return
255
+
256
+ args.distributed = True
257
+
258
+ torch.cuda.set_device(args.gpu)
259
+ args.dist_backend = "nccl"
260
+ print(
261
+ "| distributed init (rank {}): {}, gpu {}".format(
262
+ args.rank, args.dist_url, args.gpu
263
+ ),
264
+ flush=True,
265
+ )
266
+ torch.distributed.init_process_group(
267
+ backend=args.dist_backend,
268
+ init_method=args.dist_url,
269
+ world_size=args.world_size,
270
+ rank=args.rank,
271
+ )
272
+ torch.distributed.barrier()
273
+ setup_for_distributed(args.rank == 0)
274
+
275
+
276
+ class NativeScalerWithGradNormCount:
277
+ state_dict_key = "amp_scaler"
278
+
279
+ def __init__(self, enabled=True):
280
+ self._scaler = torch.cuda.amp.GradScaler(enabled=enabled)
281
+
282
+ def __call__(
283
+ self,
284
+ loss,
285
+ optimizer,
286
+ clip_grad=None,
287
+ parameters=None,
288
+ create_graph=False,
289
+ update_grad=True,
290
+ ):
291
+ self._scaler.scale(loss).backward(create_graph=create_graph)
292
+ if update_grad:
293
+ if clip_grad is not None:
294
+ assert parameters is not None
295
+ self._scaler.unscale_(
296
+ optimizer
297
+ ) # unscale the gradients of optimizer's assigned params in-place
298
+ norm = torch.nn.utils.clip_grad_norm_(parameters, clip_grad)
299
+ else:
300
+ self._scaler.unscale_(optimizer)
301
+ # TODO: FIXME: get_grad_norm_ is a poorly-implemented func that is very slow, and its return not even used
302
+ # norm = get_grad_norm_(parameters)
303
+ norm = None
304
+ self._scaler.step(optimizer)
305
+ self._scaler.update()
306
+ else:
307
+ norm = None
308
+ return norm
309
+
310
+ def state_dict(self):
311
+ return self._scaler.state_dict()
312
+
313
+ def load_state_dict(self, state_dict):
314
+ self._scaler.load_state_dict(state_dict)
315
+
316
+
317
+ def get_grad_norm_(parameters, norm_type: float = 2.0) -> torch.Tensor:
318
+ if isinstance(parameters, torch.Tensor):
319
+ parameters = [parameters]
320
+ parameters = [p for p in parameters if p.grad is not None]
321
+ norm_type = float(norm_type)
322
+ if len(parameters) == 0:
323
+ return torch.tensor(0.0)
324
+ device = parameters[0].grad.device
325
+ if norm_type == inf:
326
+ total_norm = max(p.grad.detach().abs().max().to(device) for p in parameters)
327
+ else:
328
+ total_norm = torch.norm(
329
+ torch.stack(
330
+ [torch.norm(p.grad.detach(), norm_type).to(device) for p in parameters]
331
+ ),
332
+ norm_type,
333
+ )
334
+ return total_norm
335
+
336
+
337
+ def save_model(
338
+ args, epoch, model_without_ddp, optimizer, loss_scaler, fname=None, best_so_far=None
339
+ ):
340
+ output_dir = Path(args.output_dir)
341
+ if fname is None:
342
+ fname = str(epoch)
343
+ checkpoint_path = output_dir / ("checkpoint-%s.pth" % fname)
344
+ to_save = {
345
+ "model": model_without_ddp.state_dict(),
346
+ "optimizer": optimizer.state_dict(),
347
+ "scaler": loss_scaler.state_dict(),
348
+ "args": args,
349
+ "epoch": epoch,
350
+ }
351
+ if best_so_far is not None:
352
+ to_save["best_so_far"] = best_so_far
353
+ print(f">> Saving model to {checkpoint_path} ...")
354
+ save_on_master(to_save, checkpoint_path)
355
+
356
+
357
+ def load_model(args, model_without_ddp, optimizer, loss_scaler):
358
+ args.start_epoch = 0
359
+ best_so_far = None
360
+ if args.resume is not None:
361
+ if args.resume.startswith("https"):
362
+ checkpoint = torch.hub.load_state_dict_from_url(
363
+ args.resume, map_location="cpu", check_hash=True
364
+ )
365
+ else:
366
+ checkpoint = torch.load(args.resume, map_location="cpu")
367
+ print("Resume checkpoint %s" % args.resume)
368
+ model_without_ddp.load_state_dict(checkpoint["model"], strict=False)
369
+ args.start_epoch = checkpoint["epoch"] + 1
370
+ optimizer.load_state_dict(checkpoint["optimizer"])
371
+ if "scaler" in checkpoint:
372
+ loss_scaler.load_state_dict(checkpoint["scaler"])
373
+ if "best_so_far" in checkpoint:
374
+ best_so_far = checkpoint["best_so_far"]
375
+ print(" & best_so_far={:g}".format(best_so_far))
376
+ else:
377
+ print("")
378
+ print("With optim & sched! start_epoch={:d}".format(args.start_epoch), end="")
379
+ return best_so_far
380
+
381
+
382
+ def all_reduce_mean(x):
383
+ world_size = get_world_size()
384
+ if world_size > 1:
385
+ x_reduce = torch.tensor(x).cuda()
386
+ dist.all_reduce(x_reduce)
387
+ x_reduce /= world_size
388
+ return x_reduce.item()
389
+ else:
390
+ return x
391
+
392
+
393
+ def _replace(text, src, tgt, rm=""):
394
+ """Advanced string replacement.
395
+ Given a text:
396
+ - replace all elements in src by the corresponding element in tgt
397
+ - remove all elements in rm
398
+ """
399
+ if len(tgt) == 1:
400
+ tgt = tgt * len(src)
401
+ assert len(src) == len(tgt), f"'{src}' and '{tgt}' should have the same len"
402
+ for s, t in zip(src, tgt):
403
+ text = text.replace(s, t)
404
+ for c in rm:
405
+ text = text.replace(c, "")
406
+ return text
407
+
408
+
409
+ def filename(obj):
410
+ """transform a python obj or cmd into a proper filename.
411
+ - \1 gets replaced by slash '/'
412
+ - \2 gets replaced by comma ','
413
+ """
414
+ if not isinstance(obj, str):
415
+ obj = repr(obj)
416
+ obj = str(obj).replace("()", "")
417
+ obj = _replace(obj, "_,(*/\1\2", "-__x%/,", rm=" )'\"")
418
+ assert all(len(s) < 256 for s in obj.split(os.sep)), (
419
+ "filename too long (>256 characters):\n" + obj
420
+ )
421
+ return obj
422
+
423
+
424
+ def _get_num_layer_for_vit(var_name, enc_depth, dec_depth):
425
+ if var_name in ("cls_token", "mask_token", "pos_embed", "global_tokens"):
426
+ return 0
427
+ elif var_name.startswith("patch_embed"):
428
+ return 0
429
+ elif var_name.startswith("enc_blocks"):
430
+ layer_id = int(var_name.split(".")[1])
431
+ return layer_id + 1
432
+ elif var_name.startswith("decoder_embed") or var_name.startswith(
433
+ "enc_norm"
434
+ ): # part of the last black
435
+ return enc_depth
436
+ elif var_name.startswith("dec_blocks"):
437
+ layer_id = int(var_name.split(".")[1])
438
+ return enc_depth + layer_id + 1
439
+ elif var_name.startswith("dec_norm"): # part of the last block
440
+ return enc_depth + dec_depth
441
+ elif any(var_name.startswith(k) for k in ["head", "prediction_head"]):
442
+ return enc_depth + dec_depth + 1
443
+ else:
444
+ raise NotImplementedError(var_name)
445
+
446
+
447
+ def get_parameter_groups(
448
+ model, weight_decay, layer_decay=1.0, skip_list=(), no_lr_scale_list=[]
449
+ ):
450
+ parameter_group_names = {}
451
+ parameter_group_vars = {}
452
+ enc_depth, dec_depth = None, None
453
+ # prepare layer decay values
454
+ assert layer_decay == 1.0 or 0.0 < layer_decay < 1.0
455
+ if layer_decay < 1.0:
456
+ enc_depth = model.enc_depth
457
+ dec_depth = model.dec_depth if hasattr(model, "dec_blocks") else 0
458
+ num_layers = enc_depth + dec_depth
459
+ layer_decay_values = list(
460
+ layer_decay ** (num_layers + 1 - i) for i in range(num_layers + 2)
461
+ )
462
+
463
+ for name, param in model.named_parameters():
464
+ if not param.requires_grad:
465
+ continue # frozen weights
466
+
467
+ # Assign weight decay values
468
+ if len(param.shape) == 1 or name.endswith(".bias") or name in skip_list:
469
+ group_name = "no_decay"
470
+ this_weight_decay = 0.0
471
+ else:
472
+ group_name = "decay"
473
+ this_weight_decay = weight_decay
474
+
475
+ # Assign layer ID for LR scaling
476
+ if layer_decay < 1.0:
477
+ skip_scale = False
478
+ layer_id = _get_num_layer_for_vit(name, enc_depth, dec_depth)
479
+ group_name = "layer_%d_%s" % (layer_id, group_name)
480
+ if name in no_lr_scale_list:
481
+ skip_scale = True
482
+ group_name = f"{group_name}_no_lr_scale"
483
+ else:
484
+ layer_id = 0
485
+ skip_scale = True
486
+
487
+ if group_name not in parameter_group_names:
488
+ if not skip_scale:
489
+ scale = layer_decay_values[layer_id]
490
+ else:
491
+ scale = 1.0
492
+
493
+ parameter_group_names[group_name] = {
494
+ "weight_decay": this_weight_decay,
495
+ "params": [],
496
+ "lr_scale": scale,
497
+ }
498
+ parameter_group_vars[group_name] = {
499
+ "weight_decay": this_weight_decay,
500
+ "params": [],
501
+ "lr_scale": scale,
502
+ }
503
+
504
+ parameter_group_vars[group_name]["params"].append(param)
505
+ parameter_group_names[group_name]["params"].append(name)
506
+ print("Param groups = %s" % json.dumps(parameter_group_names, indent=2))
507
+ return list(parameter_group_vars.values())
508
+
509
+
510
+ def adjust_learning_rate(optimizer, epoch, args):
511
+ """Decay the learning rate with half-cycle cosine after warmup"""
512
+
513
+ if epoch < args.warmup_epochs:
514
+ lr = args.lr * epoch / args.warmup_epochs
515
+ else:
516
+ lr = args.min_lr + (args.lr - args.min_lr) * 0.5 * (
517
+ 1.0
518
+ + math.cos(
519
+ math.pi
520
+ * (epoch - args.warmup_epochs)
521
+ / (args.epochs - args.warmup_epochs)
522
+ )
523
+ )
524
+
525
+ for param_group in optimizer.param_groups:
526
+ if "lr_scale" in param_group:
527
+ param_group["lr"] = lr * param_group["lr_scale"]
528
+ else:
529
+ param_group["lr"] = lr
530
+
531
+ return lr
src/utils/point.py ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from torch import Tensor
3
+
4
+ def get_normal_map(depth_map: torch.Tensor, intrinsic: torch.Tensor) -> torch.Tensor:
5
+ """
6
+ Convert a depth map to camera coordinates.
7
+
8
+ Args:
9
+ depth_map (torch.Tensor): Depth map of shape (H, W).
10
+ intrinsic (torch.Tensor): Camera intrinsic matrix of shape (3, 3).
11
+
12
+ Returns:
13
+ tuple[torch.Tensor, torch.Tensor]: Camera coordinates (H, W, 3)
14
+ """
15
+ B, H, W = depth_map.shape
16
+ assert intrinsic.shape == (B, 3, 3), "Intrinsic matrix must be Bx3x3"
17
+ assert (intrinsic[:, 0, 1] == 0).all() and (intrinsic[:, 1, 0] == 0).all(), "Intrinsic matrix must have zero skew"
18
+
19
+ # Intrinsic parameters
20
+ fu = intrinsic[:, 0, 0] * W # (B,)
21
+ fv = intrinsic[:, 1, 1] * H # (B,)
22
+ cu = intrinsic[:, 0, 2] * W # (B,)
23
+ cv = intrinsic[:, 1, 2] * H # (B,)
24
+
25
+ # Generate grid of pixel coordinates
26
+ u = torch.arange(W, device=depth_map.device)[None, None, :].expand(B, H, W)
27
+ v = torch.arange(H, device=depth_map.device)[None, :, None].expand(B, H, W)
28
+
29
+ # Unproject to camera coordinates (B, H, W)
30
+ x_cam = (u - cu[:, None, None]) * depth_map / fu[:, None, None]
31
+ y_cam = (v - cv[:, None, None]) * depth_map / fv[:, None, None]
32
+ z_cam = depth_map
33
+
34
+ # Stack to form camera coordinates (B, H, W, 3)
35
+ cam_coords = torch.stack((x_cam, y_cam, z_cam), dim=-1).to(dtype=torch.float32)
36
+
37
+ output = torch.zeros_like(cam_coords)
38
+ # Calculate dx using batch dimension (B, H-2, W-2, 3)
39
+ dx = cam_coords[:, 2:, 1:-1] - cam_coords[:, :-2, 1:-1]
40
+ # Calculate dy using batch dimension (B, H-2, W-2, 3)
41
+ dy = cam_coords[:, 1:-1, 2:] - cam_coords[:, 1:-1, :-2]
42
+ # Cross product and normalization (B, H-2, W-2, 3)
43
+ normal_map = torch.nn.functional.normalize(torch.cross(dx, dy, dim=-1), dim=-1)
44
+ # Assign the computed normal map to the output tensor
45
+ output[:, 1:-1, 1:-1, :] = normal_map
46
+
47
+ return output
src/utils/pose.py ADDED
@@ -0,0 +1,218 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import numpy as np
3
+ from src.model.encoder.vggt.utils.rotation import mat_to_quat
4
+ from src.model.encoder.vggt.utils.geometry import closed_form_inverse_se3, unproject_depth_map_to_point_map
5
+
6
+
7
+ def convert_pt3d_RT_to_opencv(Rot, Trans):
8
+ """
9
+ Convert Point3D extrinsic matrices to OpenCV convention.
10
+
11
+ Args:
12
+ Rot: 3D rotation matrix in Point3D format
13
+ Trans: 3D translation vector in Point3D format
14
+
15
+ Returns:
16
+ extri_opencv: 3x4 extrinsic matrix in OpenCV format
17
+ """
18
+ rot_pt3d = np.array(Rot)
19
+ trans_pt3d = np.array(Trans)
20
+
21
+ trans_pt3d[:2] *= -1
22
+ rot_pt3d[:, :2] *= -1
23
+ rot_pt3d = rot_pt3d.transpose(1, 0)
24
+ extri_opencv = np.hstack((rot_pt3d, trans_pt3d[:, None]))
25
+ return extri_opencv
26
+
27
+
28
+ def build_pair_index(N, B=1):
29
+ """
30
+ Build indices for all possible pairs of frames.
31
+
32
+ Args:
33
+ N: Number of frames
34
+ B: Batch size
35
+
36
+ Returns:
37
+ i1, i2: Indices for all possible pairs
38
+ """
39
+ i1_, i2_ = torch.combinations(torch.arange(N), 2, with_replacement=False).unbind(-1)
40
+ i1, i2 = [(i[None] + torch.arange(B)[:, None] * N).reshape(-1) for i in [i1_, i2_]]
41
+ return i1, i2
42
+
43
+
44
+ def rotation_angle(rot_gt, rot_pred, batch_size=None, eps=1e-15):
45
+ """
46
+ Calculate rotation angle error between ground truth and predicted rotations.
47
+
48
+ Args:
49
+ rot_gt: Ground truth rotation matrices
50
+ rot_pred: Predicted rotation matrices
51
+ batch_size: Batch size for reshaping the result
52
+ eps: Small value to avoid numerical issues
53
+
54
+ Returns:
55
+ Rotation angle error in degrees
56
+ """
57
+ q_pred = mat_to_quat(rot_pred)
58
+ q_gt = mat_to_quat(rot_gt)
59
+
60
+ loss_q = (1 - (q_pred * q_gt).sum(dim=1) ** 2).clamp(min=eps)
61
+ err_q = torch.arccos(1 - 2 * loss_q)
62
+
63
+ rel_rangle_deg = err_q * 180 / np.pi
64
+
65
+ if batch_size is not None:
66
+ rel_rangle_deg = rel_rangle_deg.reshape(batch_size, -1)
67
+
68
+ return rel_rangle_deg
69
+
70
+
71
+ def translation_angle(tvec_gt, tvec_pred, batch_size=None, ambiguity=True):
72
+ """
73
+ Calculate translation angle error between ground truth and predicted translations.
74
+
75
+ Args:
76
+ tvec_gt: Ground truth translation vectors
77
+ tvec_pred: Predicted translation vectors
78
+ batch_size: Batch size for reshaping the result
79
+ ambiguity: Whether to handle direction ambiguity
80
+
81
+ Returns:
82
+ Translation angle error in degrees
83
+ """
84
+ rel_tangle_deg = compare_translation_by_angle(tvec_gt, tvec_pred)
85
+ rel_tangle_deg = rel_tangle_deg * 180.0 / np.pi
86
+
87
+ if ambiguity:
88
+ rel_tangle_deg = torch.min(rel_tangle_deg, (180 - rel_tangle_deg).abs())
89
+
90
+ if batch_size is not None:
91
+ rel_tangle_deg = rel_tangle_deg.reshape(batch_size, -1)
92
+
93
+ return rel_tangle_deg
94
+
95
+
96
+ def compare_translation_by_angle(t_gt, t, eps=1e-15, default_err=1e6):
97
+ """
98
+ Normalize the translation vectors and compute the angle between them.
99
+
100
+ Args:
101
+ t_gt: Ground truth translation vectors
102
+ t: Predicted translation vectors
103
+ eps: Small value to avoid division by zero
104
+ default_err: Default error value for invalid cases
105
+
106
+ Returns:
107
+ Angular error between translation vectors in radians
108
+ """
109
+ t_norm = torch.norm(t, dim=1, keepdim=True)
110
+ t = t / (t_norm + eps)
111
+
112
+ t_gt_norm = torch.norm(t_gt, dim=1, keepdim=True)
113
+ t_gt = t_gt / (t_gt_norm + eps)
114
+
115
+ loss_t = torch.clamp_min(1.0 - torch.sum(t * t_gt, dim=1) ** 2, eps)
116
+ err_t = torch.acos(torch.sqrt(1 - loss_t))
117
+
118
+ err_t[torch.isnan(err_t) | torch.isinf(err_t)] = default_err
119
+ return err_t
120
+
121
+
122
+ def calculate_auc(r_error, t_error, max_threshold=30, return_list=False):
123
+ """
124
+ Calculate the Area Under the Curve (AUC) for the given error arrays using PyTorch.
125
+
126
+ Args:
127
+ r_error: torch.Tensor representing R error values (Degree)
128
+ t_error: torch.Tensor representing T error values (Degree)
129
+ max_threshold: Maximum threshold value for binning the histogram
130
+ return_list: Whether to return the normalized histogram as well
131
+
132
+ Returns:
133
+ AUC value, and optionally the normalized histogram
134
+ """
135
+ error_matrix = torch.stack((r_error, t_error), dim=1)
136
+ max_errors, _ = torch.max(error_matrix, dim=1)
137
+ histogram = torch.histc(
138
+ max_errors, bins=max_threshold + 1, min=0, max=max_threshold
139
+ )
140
+ num_pairs = float(max_errors.size(0))
141
+ normalized_histogram = histogram / num_pairs
142
+
143
+ if return_list:
144
+ return (
145
+ torch.cumsum(normalized_histogram, dim=0).mean(),
146
+ normalized_histogram,
147
+ )
148
+ return torch.cumsum(normalized_histogram, dim=0).mean()
149
+
150
+
151
+ def calculate_auc_np(r_error, t_error, max_threshold=30):
152
+ """
153
+ Calculate the Area Under the Curve (AUC) for the given error arrays using NumPy.
154
+
155
+ Args:
156
+ r_error: numpy array representing R error values (Degree)
157
+ t_error: numpy array representing T error values (Degree)
158
+ max_threshold: Maximum threshold value for binning the histogram
159
+
160
+ Returns:
161
+ AUC value and the normalized histogram
162
+ """
163
+ error_matrix = np.concatenate((r_error[:, None], t_error[:, None]), axis=1)
164
+ max_errors = np.max(error_matrix, axis=1)
165
+ bins = np.arange(max_threshold + 1)
166
+ histogram, _ = np.histogram(max_errors, bins=bins)
167
+ num_pairs = float(len(max_errors))
168
+ normalized_histogram = histogram.astype(float) / num_pairs
169
+ return np.mean(np.cumsum(normalized_histogram)), normalized_histogram
170
+
171
+
172
+ def se3_to_relative_pose_error(pred_se3, gt_se3, num_frames):
173
+ """
174
+ Compute rotation and translation errors between predicted and ground truth poses.
175
+
176
+ Args:
177
+ pred_se3: Predicted SE(3) transformations
178
+ gt_se3: Ground truth SE(3) transformations
179
+ num_frames: Number of frames
180
+
181
+ Returns:
182
+ Rotation and translation angle errors in degrees
183
+ """
184
+ pair_idx_i1, pair_idx_i2 = build_pair_index(num_frames)
185
+
186
+ # Compute relative camera poses between pairs
187
+ # We use closed_form_inverse to avoid potential numerical loss by torch.inverse()
188
+ relative_pose_gt = closed_form_inverse_se3(gt_se3[pair_idx_i1]).bmm(
189
+ gt_se3[pair_idx_i2]
190
+ )
191
+ relative_pose_pred = closed_form_inverse_se3(pred_se3[pair_idx_i1]).bmm(
192
+ pred_se3[pair_idx_i2]
193
+ )
194
+
195
+ # Compute the difference in rotation and translation
196
+ rel_rangle_deg = rotation_angle(
197
+ relative_pose_gt[:, :3, :3], relative_pose_pred[:, :3, :3]
198
+ )
199
+ rel_tangle_deg = translation_angle(
200
+ relative_pose_gt[:, :3, 3], relative_pose_pred[:, :3, 3]
201
+ )
202
+
203
+ return rel_rangle_deg, rel_tangle_deg
204
+
205
+
206
+ def align_to_first_camera(camera_poses):
207
+ """
208
+ Align all camera poses to the first camera's coordinate frame.
209
+
210
+ Args:
211
+ camera_poses: Tensor of shape (N, 4, 4) containing camera poses as SE3 transformations
212
+
213
+ Returns:
214
+ Tensor of shape (N, 4, 4) containing aligned camera poses
215
+ """
216
+ first_cam_extrinsic_inv = closed_form_inverse_se3(camera_poses[0][None])
217
+ aligned_poses = torch.matmul(camera_poses, first_cam_extrinsic_inv)
218
+ return aligned_poses
src/utils/render.py ADDED
@@ -0,0 +1,284 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2022 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import numpy as np
16
+ import os
17
+ import enum
18
+ import types
19
+ from typing import List, Mapping, Optional, Text, Tuple, Union
20
+ import copy
21
+ from PIL import Image
22
+ # import mediapy as media
23
+ from matplotlib import cm
24
+ from tqdm import tqdm
25
+
26
+ import torch
27
+
28
+ def normalize(x: np.ndarray) -> np.ndarray:
29
+ """Normalization helper function."""
30
+ return x / np.linalg.norm(x)
31
+
32
+ def pad_poses(p: np.ndarray) -> np.ndarray:
33
+ """Pad [..., 3, 4] pose matrices with a homogeneous bottom row [0,0,0,1]."""
34
+ bottom = np.broadcast_to([0, 0, 0, 1.], p[..., :1, :4].shape)
35
+ return np.concatenate([p[..., :3, :4], bottom], axis=-2)
36
+
37
+
38
+ def unpad_poses(p: np.ndarray) -> np.ndarray:
39
+ """Remove the homogeneous bottom row from [..., 4, 4] pose matrices."""
40
+ return p[..., :3, :4]
41
+
42
+
43
+ def recenter_poses(poses: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
44
+ """Recenter poses around the origin."""
45
+ cam2world = average_pose(poses)
46
+ transform = np.linalg.inv(pad_poses(cam2world))
47
+ poses = transform @ pad_poses(poses)
48
+ return unpad_poses(poses), transform
49
+
50
+
51
+ def average_pose(poses: np.ndarray) -> np.ndarray:
52
+ """New pose using average position, z-axis, and up vector of input poses."""
53
+ position = poses[:, :3, 3].mean(0)
54
+ z_axis = poses[:, :3, 2].mean(0)
55
+ up = poses[:, :3, 1].mean(0)
56
+ cam2world = viewmatrix(z_axis, up, position)
57
+ return cam2world
58
+
59
+ def viewmatrix(lookdir: np.ndarray, up: np.ndarray,
60
+ position: np.ndarray) -> np.ndarray:
61
+ """Construct lookat view matrix."""
62
+ vec2 = normalize(lookdir)
63
+ vec0 = normalize(np.cross(up, vec2))
64
+ vec1 = normalize(np.cross(vec2, vec0))
65
+ m = np.stack([vec0, vec1, vec2, position], axis=1)
66
+ return m
67
+
68
+ def focus_point_fn(poses: np.ndarray) -> np.ndarray:
69
+ """Calculate nearest point to all focal axes in poses."""
70
+ directions, origins = poses[:, :3, 2:3], poses[:, :3, 3:4]
71
+ m = np.eye(3) - directions * np.transpose(directions, [0, 2, 1])
72
+ mt_m = np.transpose(m, [0, 2, 1]) @ m
73
+ focus_pt = np.linalg.inv(mt_m.mean(0)) @ (mt_m @ origins).mean(0)[:, 0]
74
+ return focus_pt
75
+
76
+ def transform_poses_pca(poses: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
77
+ """Transforms poses so principal components lie on XYZ axes.
78
+
79
+ Args:
80
+ poses: a (N, 3, 4) array containing the cameras' camera to world transforms.
81
+
82
+ Returns:
83
+ A tuple (poses, transform), with the transformed poses and the applied
84
+ camera_to_world transforms.
85
+ """
86
+ t = poses[:, :3, 3]
87
+ t_mean = t.mean(axis=0)
88
+ t = t - t_mean
89
+
90
+ eigval, eigvec = np.linalg.eig(t.T @ t)
91
+ # Sort eigenvectors in order of largest to smallest eigenvalue.
92
+ inds = np.argsort(eigval)[::-1]
93
+ eigvec = eigvec[:, inds]
94
+ rot = eigvec.T
95
+ if np.linalg.det(rot) < 0:
96
+ rot = np.diag(np.array([1, 1, -1])) @ rot
97
+
98
+ transform = np.concatenate([rot, rot @ -t_mean[:, None]], -1)
99
+ poses_recentered = unpad_poses(transform @ pad_poses(poses))
100
+ transform = np.concatenate([transform, np.eye(4)[3:]], axis=0)
101
+
102
+ # Flip coordinate system if z component of y-axis is negative
103
+ if poses_recentered.mean(axis=0)[2, 1] < 0:
104
+ poses_recentered = np.diag(np.array([1, -1, -1])) @ poses_recentered
105
+ transform = np.diag(np.array([1, -1, -1, 1])) @ transform
106
+
107
+ return poses_recentered, transform
108
+ # points = np.random.rand(3,100)
109
+ # points_h = np.concatenate((points,np.ones_like(points[:1])), axis=0)
110
+ # (poses_recentered @ points_h)[0]
111
+ # (transform @ pad_poses(poses) @ points_h)[0,:3]
112
+ # import pdb; pdb.set_trace()
113
+
114
+ # # Just make sure it's it in the [-1, 1]^3 cube
115
+ # scale_factor = 1. / np.max(np.abs(poses_recentered[:, :3, 3]))
116
+ # poses_recentered[:, :3, 3] *= scale_factor
117
+ # transform = np.diag(np.array([scale_factor] * 3 + [1])) @ transform
118
+
119
+ # return poses_recentered, transform
120
+
121
+ def generate_ellipse_path(poses: np.ndarray,
122
+ n_frames: int = 120,
123
+ const_speed: bool = True,
124
+ z_variation: float = 0.,
125
+ z_phase: float = 0.) -> np.ndarray:
126
+ """Generate an elliptical render path based on the given poses."""
127
+ # Calculate the focal point for the path (cameras point toward this).
128
+ center = focus_point_fn(poses)
129
+ # Path height sits at z=0 (in middle of zero-mean capture pattern).
130
+ offset = np.array([center[0], center[1], 0])
131
+
132
+ # Calculate scaling for ellipse axes based on input camera positions.
133
+ sc = np.percentile(np.abs(poses[:, :3, 3] - offset), 90, axis=0)
134
+ # Use ellipse that is symmetric about the focal point in xy.
135
+ low = -sc + offset
136
+ high = sc + offset
137
+ # Optional height variation need not be symmetric
138
+ z_low = np.percentile((poses[:, :3, 3]), 10, axis=0)
139
+ z_high = np.percentile((poses[:, :3, 3]), 90, axis=0)
140
+
141
+ def get_positions(theta):
142
+ # Interpolate between bounds with trig functions to get ellipse in x-y.
143
+ # Optionally also interpolate in z to change camera height along path.
144
+ return np.stack([
145
+ low[0] + (high - low)[0] * (np.cos(theta) * .5 + .5),
146
+ low[1] + (high - low)[1] * (np.sin(theta) * .5 + .5),
147
+ z_variation * (z_low[2] + (z_high - z_low)[2] *
148
+ (np.cos(theta + 2 * np.pi * z_phase) * .5 + .5)),
149
+ ], -1)
150
+
151
+ theta = np.linspace(0, 2. * np.pi, n_frames + 1, endpoint=True)
152
+ positions = get_positions(theta)
153
+
154
+ #if const_speed:
155
+
156
+ # # Resample theta angles so that the velocity is closer to constant.
157
+ # lengths = np.linalg.norm(positions[1:] - positions[:-1], axis=-1)
158
+ # theta = stepfun.sample(None, theta, np.log(lengths), n_frames + 1)
159
+ # positions = get_positions(theta)
160
+
161
+ # Throw away duplicated last position.
162
+ positions = positions[:-1]
163
+
164
+ # Set path's up vector to axis closest to average of input pose up vectors.
165
+ avg_up = poses[:, :3, 1].mean(0)
166
+ avg_up = avg_up / np.linalg.norm(avg_up)
167
+ ind_up = np.argmax(np.abs(avg_up))
168
+ up = np.eye(3)[ind_up] * np.sign(avg_up[ind_up])
169
+
170
+ return np.stack([viewmatrix(p - center, up, p) for p in positions])
171
+
172
+
173
+ def generate_path(viewpoint_cameras, n_frames=480):
174
+ # c2ws = np.array([np.linalg.inv(np.asarray((cam.world_view_transform.T).cpu().numpy())) for cam in viewpoint_cameras])
175
+ c2ws = viewpoint_cameras.cpu().numpy()
176
+ pose = c2ws[:,:3,:] @ np.diag([1, -1, -1, 1])
177
+ pose_recenter, colmap_to_world_transform = transform_poses_pca(pose)
178
+
179
+ # generate new poses
180
+ new_poses = generate_ellipse_path(poses=pose_recenter, n_frames=n_frames)
181
+ # warp back to orignal scale
182
+ new_poses = np.linalg.inv(colmap_to_world_transform) @ pad_poses(new_poses)
183
+
184
+ return new_poses
185
+
186
+ # traj = []
187
+ # for c2w in new_poses:
188
+ # c2w = c2w @ np.diag([1, -1, -1, 1])
189
+ # cam = copy.deepcopy(viewpoint_cameras[0])
190
+ # cam.image_height = int(cam.image_height / 2) * 2
191
+ # cam.image_width = int(cam.image_width / 2) * 2
192
+ # cam.world_view_transform = torch.from_numpy(np.linalg.inv(c2w).T).float().cuda()
193
+ # cam.full_proj_transform = (cam.world_view_transform.unsqueeze(0).bmm(cam.projection_matrix.unsqueeze(0))).squeeze(0)
194
+ # cam.camera_center = cam.world_view_transform.inverse()[3, :3]
195
+ # traj.append(cam)
196
+
197
+ # return traj
198
+
199
+ def load_img(pth: str) -> np.ndarray:
200
+ """Load an image and cast to float32."""
201
+ with open(pth, 'rb') as f:
202
+ image = np.array(Image.open(f), dtype=np.float32)
203
+ return image
204
+
205
+
206
+ def create_videos(base_dir, input_dir, out_name, num_frames=480):
207
+ """Creates videos out of the images saved to disk."""
208
+ # Last two parts of checkpoint path are experiment name and scene name.
209
+ video_prefix = f'{out_name}'
210
+
211
+ zpad = max(5, len(str(num_frames - 1)))
212
+ idx_to_str = lambda idx: str(idx).zfill(zpad)
213
+
214
+ os.makedirs(base_dir, exist_ok=True)
215
+ render_dist_curve_fn = np.log
216
+
217
+ # Load one example frame to get image shape and depth range.
218
+ depth_file = os.path.join(input_dir, 'vis', f'depth_{idx_to_str(0)}.tiff')
219
+ depth_frame = load_img(depth_file)
220
+ shape = depth_frame.shape
221
+ p = 3
222
+ distance_limits = np.percentile(depth_frame.flatten(), [p, 100 - p])
223
+ lo, hi = [render_dist_curve_fn(x) for x in distance_limits]
224
+ print(f'Video shape is {shape[:2]}')
225
+
226
+ video_kwargs = {
227
+ 'shape': shape[:2],
228
+ 'codec': 'h264',
229
+ 'fps': 60,
230
+ 'crf': 18,
231
+ }
232
+
233
+ for k in ['depth', 'normal', 'color']:
234
+ video_file = os.path.join(base_dir, f'{video_prefix}_{k}.mp4')
235
+ input_format = 'gray' if k == 'alpha' else 'rgb'
236
+
237
+
238
+ file_ext = 'png' if k in ['color', 'normal'] else 'tiff'
239
+ idx = 0
240
+
241
+ if k == 'color':
242
+ file0 = os.path.join(input_dir, 'renders', f'{idx_to_str(0)}.{file_ext}')
243
+ else:
244
+ file0 = os.path.join(input_dir, 'vis', f'{k}_{idx_to_str(0)}.{file_ext}')
245
+
246
+ if not os.path.exists(file0):
247
+ print(f'Images missing for tag {k}')
248
+ continue
249
+ print(f'Making video {video_file}...')
250
+ with media.VideoWriter(
251
+ video_file, **video_kwargs, input_format=input_format) as writer:
252
+ for idx in tqdm(range(num_frames)):
253
+ # img_file = os.path.join(input_dir, f'{k}_{idx_to_str(idx)}.{file_ext}')
254
+ if k == 'color':
255
+ img_file = os.path.join(input_dir, 'renders', f'{idx_to_str(idx)}.{file_ext}')
256
+ else:
257
+ img_file = os.path.join(input_dir, 'vis', f'{k}_{idx_to_str(idx)}.{file_ext}')
258
+
259
+ if not os.path.exists(img_file):
260
+ ValueError(f'Image file {img_file} does not exist.')
261
+ img = load_img(img_file)
262
+ if k in ['color', 'normal']:
263
+ img = img / 255.
264
+ elif k.startswith('depth'):
265
+ img = render_dist_curve_fn(img)
266
+ img = np.clip((img - np.minimum(lo, hi)) / np.abs(hi - lo), 0, 1)
267
+ img = cm.get_cmap('turbo')(img)[..., :3]
268
+
269
+ frame = (np.clip(np.nan_to_num(img), 0., 1.) * 255.).astype(np.uint8)
270
+ writer.add_image(frame)
271
+ idx += 1
272
+
273
+ def save_img_u8(img, pth):
274
+ """Save an image (probably RGB) in [0, 1] to disk as a uint8 PNG."""
275
+ with open(pth, 'wb') as f:
276
+ Image.fromarray(
277
+ (np.clip(np.nan_to_num(img), 0., 1.) * 255.).astype(np.uint8)).save(
278
+ f, 'PNG')
279
+
280
+
281
+ def save_img_f32(depthmap, pth):
282
+ """Save an image (probably a depthmap) to disk as a float32 TIFF."""
283
+ with open(pth, 'wb') as f:
284
+ Image.fromarray(np.nan_to_num(depthmap).astype(np.float32)).save(f, 'TIFF')
src/utils/tensor_to_pycolmap.py ADDED
@@ -0,0 +1,320 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+ #
4
+ # This source code is licensed under the license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+
8
+ import torch
9
+ import torch.nn as nn
10
+ import torch.nn.functional as F
11
+
12
+ import numpy as np
13
+ import pycolmap
14
+
15
+ # TODO: frame_idx should start from 1 instead of 0 in colmap
16
+ def batch_matrix_to_pycolmap(
17
+ points3d,
18
+ extrinsics,
19
+ intrinsics,
20
+ tracks,
21
+ image_size,
22
+ masks=None,
23
+ max_reproj_error=None,
24
+ max_points3D_val=3000,
25
+ shared_camera=False,
26
+ camera_type="SIMPLE_PINHOLE",
27
+ extra_params=None,
28
+ ):
29
+ """
30
+ Convert Batched Pytorch Tensors to PyCOLMAP
31
+
32
+ Check https://github.com/colmap/pycolmap for more details about its format
33
+ """
34
+
35
+ # points3d: Px3
36
+ # extrinsics: Nx3x4
37
+ # intrinsics: Nx3x3
38
+ # tracks: NxPx2
39
+ # masks: NxP
40
+ # image_size: 2, assume all the frames have been padded to the same size
41
+ # where N is the number of frames and P is the number of tracks
42
+
43
+ N, P, _ = tracks.shape
44
+ assert len(extrinsics) == N
45
+ assert len(intrinsics) == N
46
+ assert len(points3d) == P
47
+ assert image_size.shape[0] == 2
48
+
49
+ projected_points_2d, projected_points_cam = project_3D_points(points3d, extrinsics, intrinsics, return_points_cam=True)
50
+ projected_diff = (projected_points_2d - tracks).norm(dim=-1)
51
+ projected_points_2d[projected_points_cam[:, -1] <= 0] = 1e6
52
+ reproj_mask = projected_diff < max_reproj_error
53
+
54
+ if masks is not None:
55
+ masks = torch.logical_and(masks, reproj_mask)
56
+ else:
57
+ masks = reproj_mask
58
+
59
+ extrinsics = extrinsics.cpu().numpy()
60
+ intrinsics = intrinsics.cpu().numpy()
61
+
62
+ if extra_params is not None:
63
+ extra_params = extra_params.cpu().numpy()
64
+
65
+
66
+ tracks = tracks.cpu().numpy()
67
+ points3d = points3d.cpu().numpy()
68
+ image_size = image_size.cpu().numpy()
69
+
70
+ # Reconstruction object, following the format of PyCOLMAP/COLMAP
71
+ reconstruction = pycolmap.Reconstruction()
72
+
73
+ masks = masks.cpu().numpy()
74
+
75
+ inlier_num = masks.sum(0)
76
+ valid_mask = inlier_num >= 2 # a track is invalid if without two inliers
77
+ valid_idx = np.nonzero(valid_mask)[0]
78
+
79
+ # Only add 3D points that have sufficient 2D points
80
+ for vidx in valid_idx:
81
+ reconstruction.add_point3D(
82
+ points3d[vidx], pycolmap.Track(), np.zeros(3)
83
+ )
84
+
85
+ num_points3D = len(valid_idx)
86
+
87
+ camera = None
88
+ # frame idx
89
+ for fidx in range(N):
90
+ # set camera
91
+ if camera is None or (not shared_camera):
92
+ if camera_type == "SIMPLE_RADIAL":
93
+ focal = (intrinsics[fidx][0, 0] + intrinsics[fidx][1, 1]) / 2
94
+ pycolmap_intri = np.array(
95
+ [
96
+ focal,
97
+ intrinsics[fidx][0, 2],
98
+ intrinsics[fidx][1, 2],
99
+ extra_params[fidx][0],
100
+ ]
101
+ )
102
+ elif camera_type == "SIMPLE_PINHOLE":
103
+ focal = (intrinsics[fidx][0, 0] + intrinsics[fidx][1, 1]) / 2
104
+ pycolmap_intri = np.array(
105
+ [
106
+ focal,
107
+ intrinsics[fidx][0, 2],
108
+ intrinsics[fidx][1, 2],
109
+ ]
110
+ )
111
+ else:
112
+ raise ValueError(
113
+ f"Camera type {camera_type} is not supported yet"
114
+ )
115
+
116
+ camera = pycolmap.Camera(
117
+ model=camera_type,
118
+ width=image_size[0],
119
+ height=image_size[1],
120
+ params=pycolmap_intri,
121
+ camera_id=fidx,
122
+ )
123
+
124
+ # add camera
125
+ reconstruction.add_camera(camera)
126
+
127
+ # set image
128
+ cam_from_world = pycolmap.Rigid3d(
129
+ pycolmap.Rotation3d(extrinsics[fidx][:3, :3]),
130
+ extrinsics[fidx][:3, 3],
131
+ ) # Rot and Trans
132
+ image = pycolmap.Image(
133
+ id=fidx,
134
+ name=f"image_{fidx}",
135
+ camera_id=camera.camera_id,
136
+ cam_from_world=cam_from_world,
137
+ )
138
+
139
+ points2D_list = []
140
+
141
+ point2D_idx = 0
142
+ # NOTE point3D_id start by 1
143
+ for point3D_id in range(1, num_points3D + 1):
144
+ original_track_idx = valid_idx[point3D_id - 1]
145
+
146
+ if (
147
+ reconstruction.points3D[point3D_id].xyz < max_points3D_val
148
+ ).all():
149
+ if masks[fidx][original_track_idx]:
150
+ # It seems we don't need +0.5 for BA
151
+ point2D_xy = tracks[fidx][original_track_idx]
152
+ # Please note when adding the Point2D object
153
+ # It not only requires the 2D xy location, but also the id to 3D point
154
+ points2D_list.append(
155
+ pycolmap.Point2D(point2D_xy, point3D_id)
156
+ )
157
+
158
+ # add element
159
+ track = reconstruction.points3D[point3D_id].track
160
+ track.add_element(fidx, point2D_idx)
161
+ point2D_idx += 1
162
+
163
+ assert point2D_idx == len(points2D_list)
164
+
165
+ try:
166
+ image.points2D = pycolmap.ListPoint2D(points2D_list)
167
+ image.registered = True
168
+ except:
169
+ print(f"frame {fidx} is out of BA")
170
+ image.registered = False
171
+
172
+ # add image
173
+ reconstruction.add_image(image)
174
+
175
+ return reconstruction
176
+
177
+
178
+ def pycolmap_to_batch_matrix(
179
+ reconstruction, device="cuda", camera_type="SIMPLE_PINHOLE"
180
+ ):
181
+ """
182
+ Convert a PyCOLMAP Reconstruction Object to batched PyTorch tensors.
183
+
184
+ Args:
185
+ reconstruction (pycolmap.Reconstruction): The reconstruction object from PyCOLMAP.
186
+ device (str): The device to place the tensors on (default: "cuda").
187
+ camera_type (str): The type of camera model used (default: "SIMPLE_PINHOLE").
188
+
189
+ Returns:
190
+ tuple: A tuple containing points3D, extrinsics, intrinsics, and optionally extra_params.
191
+ """
192
+
193
+ num_images = len(reconstruction.images)
194
+ max_points3D_id = max(reconstruction.point3D_ids())
195
+ points3D = np.zeros((max_points3D_id, 3))
196
+
197
+ for point3D_id in reconstruction.points3D:
198
+ points3D[point3D_id - 1] = reconstruction.points3D[point3D_id].xyz
199
+ points3D = torch.from_numpy(points3D).to(device)
200
+
201
+ extrinsics = []
202
+ intrinsics = []
203
+
204
+ extra_params = [] if camera_type == "SIMPLE_RADIAL" else None
205
+
206
+ for i in range(num_images):
207
+ # Extract and append extrinsics
208
+ pyimg = reconstruction.images[i]
209
+ pycam = reconstruction.cameras[pyimg.camera_id]
210
+ matrix = pyimg.cam_from_world.matrix()
211
+ extrinsics.append(matrix)
212
+
213
+ # Extract and append intrinsics
214
+ calibration_matrix = pycam.calibration_matrix()
215
+ intrinsics.append(calibration_matrix)
216
+
217
+ if camera_type == "SIMPLE_RADIAL":
218
+ extra_params.append(pycam.params[-1])
219
+
220
+ # Convert lists to torch tensors
221
+ extrinsics = torch.from_numpy(np.stack(extrinsics)).to(device)
222
+
223
+ intrinsics = torch.from_numpy(np.stack(intrinsics)).to(device)
224
+
225
+ if camera_type == "SIMPLE_RADIAL":
226
+ extra_params = torch.from_numpy(np.stack(extra_params)).to(device)
227
+ extra_params = extra_params[:, None]
228
+
229
+ return points3D, extrinsics, intrinsics, extra_params
230
+
231
+
232
+
233
+
234
+
235
+ def project_3D_points(
236
+ points3D,
237
+ extrinsics,
238
+ intrinsics=None,
239
+ extra_params=None,
240
+ return_points_cam=False,
241
+ default=0,
242
+ only_points_cam=False,
243
+ ):
244
+ """
245
+ Transforms 3D points to 2D using extrinsic and intrinsic parameters.
246
+ Args:
247
+ points3D (torch.Tensor): 3D points of shape Px3.
248
+ extrinsics (torch.Tensor): Extrinsic parameters of shape Bx3x4.
249
+ intrinsics (torch.Tensor): Intrinsic parameters of shape Bx3x3.
250
+ extra_params (torch.Tensor): Extra parameters of shape BxN, which is used for radial distortion.
251
+ Returns:
252
+ torch.Tensor: Transformed 2D points of shape BxNx2.
253
+ """
254
+ with torch.cuda.amp.autocast(dtype=torch.double):
255
+ N = points3D.shape[0] # Number of points
256
+ B = extrinsics.shape[0] # Batch size, i.e., number of cameras
257
+ points3D_homogeneous = torch.cat(
258
+ [points3D, torch.ones_like(points3D[..., 0:1])], dim=1
259
+ ) # Nx4
260
+ # Reshape for batch processing
261
+ points3D_homogeneous = points3D_homogeneous.unsqueeze(0).expand(
262
+ B, -1, -1
263
+ ) # BxNx4
264
+
265
+ # Step 1: Apply extrinsic parameters
266
+ # Transform 3D points to camera coordinate system for all cameras
267
+ points_cam = torch.bmm(
268
+ extrinsics, points3D_homogeneous.transpose(-1, -2)
269
+ )
270
+
271
+ if only_points_cam:
272
+ return points_cam
273
+
274
+ # Step 2: Apply intrinsic parameters and (optional) distortion
275
+ points2D = img_from_cam(intrinsics, points_cam, extra_params)
276
+
277
+ if return_points_cam:
278
+ return points2D, points_cam
279
+ return points2D
280
+
281
+
282
+ def img_from_cam(intrinsics, points_cam, extra_params=None, default=0.0):
283
+ """
284
+ Applies intrinsic parameters and optional distortion to the given 3D points.
285
+
286
+ Args:
287
+ intrinsics (torch.Tensor): Intrinsic camera parameters of shape Bx3x3.
288
+ points_cam (torch.Tensor): 3D points in camera coordinates of shape Bx3xN.
289
+ extra_params (torch.Tensor, optional): Distortion parameters of shape BxN, where N can be 1, 2, or 4.
290
+ default (float, optional): Default value to replace NaNs in the output.
291
+
292
+ Returns:
293
+ points2D (torch.Tensor): 2D points in pixel coordinates of shape BxNx2.
294
+ """
295
+
296
+ # Normalize by the third coordinate (homogeneous division)
297
+ points_cam = points_cam / points_cam[:, 2:3, :]
298
+ # Extract uv
299
+ uv = points_cam[:, :2, :]
300
+
301
+ # Apply distortion if extra_params are provided
302
+ if extra_params is not None:
303
+ uu, vv = apply_distortion(extra_params, uv[:, 0], uv[:, 1])
304
+ uv = torch.stack([uu, vv], dim=1)
305
+
306
+ # Prepare points_cam for batch matrix multiplication
307
+ points_cam_homo = torch.cat(
308
+ (uv, torch.ones_like(uv[:, :1, :])), dim=1
309
+ ) # Bx3xN
310
+ # Apply intrinsic parameters using batch matrix multiplication
311
+ points2D_homo = torch.bmm(intrinsics, points_cam_homo) # Bx3xN
312
+
313
+ # Extract x and y coordinates
314
+ points2D = points2D_homo[:, :2, :] # Bx2xN
315
+
316
+ # Replace NaNs with default value
317
+ points2D = torch.nan_to_num(points2D, nan=default)
318
+
319
+ return points2D.transpose(1, 2) # BxNx2
320
+
src/utils/transforms.py ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+
4
+ # This source code is licensed under the license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ # Copyright (C) 2024-present Naver Corporation. All rights reserved.
8
+ # Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
9
+ #
10
+ # --------------------------------------------------------
11
+ # DUST3R default transforms
12
+ # --------------------------------------------------------
13
+ import torchvision.transforms as tvf
14
+
15
+ from model.dataset.utils.image import ImgNorm
16
+
17
+ # define the standard image transforms
18
+ ColorJitter = tvf.Compose([tvf.ColorJitter(0.5, 0.5, 0.5, 0.1), ImgNorm])
src/utils/viz.py ADDED
@@ -0,0 +1,377 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Meta Platforms, Inc. and affiliates.
2
+ # All rights reserved.
3
+
4
+ # This source code is licensed under the license found in the
5
+ # LICENSE file in the root directory of this source tree.
6
+
7
+ # Copyright (C) 2024-present Naver Corporation. All rights reserved.
8
+ # Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
9
+ #
10
+ # --------------------------------------------------------
11
+ # Visualization utilities using trimesh
12
+ # --------------------------------------------------------
13
+ import numpy as np
14
+ import PIL.Image
15
+ import torch
16
+ from scipy.spatial.transform import Rotation
17
+
18
+ from model.dataset.utils.device import to_numpy
19
+ from model.dataset.utils.geometry import geotrf, get_med_dist_between_poses
20
+ from model.dataset.utils.image import rgb
21
+
22
+ try:
23
+ import trimesh
24
+ except ImportError:
25
+ print("/!\\ module trimesh is not installed, cannot visualize results /!\\")
26
+
27
+
28
+ def cat_3d(vecs):
29
+ if isinstance(vecs, (np.ndarray, torch.Tensor)):
30
+ vecs = [vecs]
31
+ return np.concatenate([p.reshape(-1, 3) for p in to_numpy(vecs)])
32
+
33
+
34
+ def show_raw_pointcloud(pts3d, colors, point_size=2):
35
+ scene = trimesh.Scene()
36
+
37
+ pct = trimesh.PointCloud(cat_3d(pts3d), colors=cat_3d(colors))
38
+ scene.add_geometry(pct)
39
+
40
+ scene.show(line_settings={"point_size": point_size})
41
+
42
+
43
+ def pts3d_to_trimesh(img, pts3d, valid=None):
44
+ H, W, THREE = img.shape
45
+ assert THREE == 3
46
+ assert img.shape == pts3d.shape
47
+
48
+ vertices = pts3d.reshape(-1, 3)
49
+
50
+ # make squares: each pixel == 2 triangles
51
+ idx = np.arange(len(vertices)).reshape(H, W)
52
+ idx1 = idx[:-1, :-1].ravel() # top-left corner
53
+ idx2 = idx[:-1, +1:].ravel() # right-left corner
54
+ idx3 = idx[+1:, :-1].ravel() # bottom-left corner
55
+ idx4 = idx[+1:, +1:].ravel() # bottom-right corner
56
+ faces = np.concatenate(
57
+ (
58
+ np.c_[idx1, idx2, idx3],
59
+ np.c_[
60
+ idx3, idx2, idx1
61
+ ], # same triangle, but backward (cheap solution to cancel face culling)
62
+ np.c_[idx2, idx3, idx4],
63
+ np.c_[
64
+ idx4, idx3, idx2
65
+ ], # same triangle, but backward (cheap solution to cancel face culling)
66
+ ),
67
+ axis=0,
68
+ )
69
+
70
+ # prepare triangle colors
71
+ face_colors = np.concatenate(
72
+ (
73
+ img[:-1, :-1].reshape(-1, 3),
74
+ img[:-1, :-1].reshape(-1, 3),
75
+ img[+1:, +1:].reshape(-1, 3),
76
+ img[+1:, +1:].reshape(-1, 3),
77
+ ),
78
+ axis=0,
79
+ )
80
+
81
+ # remove invalid faces
82
+ if valid is not None:
83
+ assert valid.shape == (H, W)
84
+ valid_idxs = valid.ravel()
85
+ valid_faces = valid_idxs[faces].all(axis=-1)
86
+ faces = faces[valid_faces]
87
+ face_colors = face_colors[valid_faces]
88
+
89
+ assert len(faces) == len(face_colors)
90
+ return dict(vertices=vertices, face_colors=face_colors, faces=faces)
91
+
92
+
93
+ def cat_meshes(meshes):
94
+ vertices, faces, colors = zip(
95
+ *[(m["vertices"], m["faces"], m["face_colors"]) for m in meshes]
96
+ )
97
+ n_vertices = np.cumsum([0] + [len(v) for v in vertices])
98
+ for i in range(len(faces)):
99
+ faces[i][:] += n_vertices[i]
100
+
101
+ vertices = np.concatenate(vertices)
102
+ colors = np.concatenate(colors)
103
+ faces = np.concatenate(faces)
104
+ return dict(vertices=vertices, face_colors=colors, faces=faces)
105
+
106
+
107
+ def show_duster_pairs(view1, view2, pred1, pred2):
108
+ import matplotlib.pyplot as pl
109
+
110
+ pl.ion()
111
+
112
+ for e in range(len(view1["instance"])):
113
+ i = view1["idx"][e]
114
+ j = view2["idx"][e]
115
+ img1 = rgb(view1["img"][e])
116
+ img2 = rgb(view2["img"][e])
117
+ conf1 = pred1["conf"][e].squeeze()
118
+ conf2 = pred2["conf"][e].squeeze()
119
+ score = conf1.mean() * conf2.mean()
120
+ print(f">> Showing pair #{e} {i}-{j} {score=:g}")
121
+ pl.clf()
122
+ pl.subplot(221).imshow(img1)
123
+ pl.subplot(223).imshow(img2)
124
+ pl.subplot(222).imshow(conf1, vmin=1, vmax=30)
125
+ pl.subplot(224).imshow(conf2, vmin=1, vmax=30)
126
+ pts1 = pred1["pts3d"][e]
127
+ pts2 = pred2["pts3d_in_other_view"][e]
128
+ pl.subplots_adjust(0, 0, 1, 1, 0, 0)
129
+ if input("show pointcloud? (y/n) ") == "y":
130
+ show_raw_pointcloud(cat(pts1, pts2), cat(img1, img2), point_size=5)
131
+
132
+
133
+ def auto_cam_size(im_poses):
134
+ return 0.1 * get_med_dist_between_poses(im_poses)
135
+
136
+
137
+ class SceneViz:
138
+ def __init__(self):
139
+ self.scene = trimesh.Scene()
140
+
141
+ def add_pointcloud(self, pts3d, color, mask=None):
142
+ pts3d = to_numpy(pts3d)
143
+ mask = to_numpy(mask)
144
+ if mask is None:
145
+ mask = [slice(None)] * len(pts3d)
146
+ pts = np.concatenate([p[m] for p, m in zip(pts3d, mask)])
147
+ pct = trimesh.PointCloud(pts.reshape(-1, 3))
148
+
149
+ if isinstance(color, (list, np.ndarray, torch.Tensor)):
150
+ color = to_numpy(color)
151
+ col = np.concatenate([p[m] for p, m in zip(color, mask)])
152
+ assert col.shape == pts.shape
153
+ pct.visual.vertex_colors = uint8(col.reshape(-1, 3))
154
+ else:
155
+ assert len(color) == 3
156
+ pct.visual.vertex_colors = np.broadcast_to(uint8(color), pts.shape)
157
+
158
+ self.scene.add_geometry(pct)
159
+ return self
160
+
161
+ def add_camera(
162
+ self,
163
+ pose_c2w,
164
+ focal=None,
165
+ color=(0, 0, 0),
166
+ image=None,
167
+ imsize=None,
168
+ cam_size=0.03,
169
+ ):
170
+ pose_c2w, focal, color, image = to_numpy((pose_c2w, focal, color, image))
171
+ add_scene_cam(self.scene, pose_c2w, color, image, focal, screen_width=cam_size)
172
+ return self
173
+
174
+ def add_cameras(
175
+ self, poses, focals=None, images=None, imsizes=None, colors=None, **kw
176
+ ):
177
+ def get(arr, idx):
178
+ return None if arr is None else arr[idx]
179
+
180
+ for i, pose_c2w in enumerate(poses):
181
+ self.add_camera(
182
+ pose_c2w,
183
+ get(focals, i),
184
+ image=get(images, i),
185
+ color=get(colors, i),
186
+ imsize=get(imsizes, i),
187
+ **kw,
188
+ )
189
+ return self
190
+
191
+ def show(self, point_size=2, viewer=None):
192
+ return self.scene.show(viewer=viewer, line_settings={"point_size": point_size})
193
+
194
+
195
+ def show_raw_pointcloud_with_cams(
196
+ imgs, pts3d, mask, focals, cams2world, point_size=2, cam_size=0.05, cam_color=None
197
+ ):
198
+ """Visualization of a pointcloud with cameras
199
+ imgs = (N, H, W, 3) or N-size list of [(H,W,3), ...]
200
+ pts3d = (N, H, W, 3) or N-size list of [(H,W,3), ...]
201
+ focals = (N,) or N-size list of [focal, ...]
202
+ cams2world = (N,4,4) or N-size list of [(4,4), ...]
203
+ """
204
+ assert len(pts3d) == len(mask) <= len(imgs) <= len(cams2world) == len(focals)
205
+ pts3d = to_numpy(pts3d)
206
+ imgs = to_numpy(imgs)
207
+ focals = to_numpy(focals)
208
+ cams2world = to_numpy(cams2world)
209
+
210
+ scene = trimesh.Scene()
211
+
212
+ # full pointcloud
213
+ pts = np.concatenate([p[m] for p, m in zip(pts3d, mask)])
214
+ col = np.concatenate([p[m] for p, m in zip(imgs, mask)])
215
+ pct = trimesh.PointCloud(pts.reshape(-1, 3), colors=col.reshape(-1, 3))
216
+ scene.add_geometry(pct)
217
+
218
+ # add each camera
219
+ for i, pose_c2w in enumerate(cams2world):
220
+ if isinstance(cam_color, list):
221
+ camera_edge_color = cam_color[i]
222
+ else:
223
+ camera_edge_color = cam_color or CAM_COLORS[i % len(CAM_COLORS)]
224
+ add_scene_cam(
225
+ scene,
226
+ pose_c2w,
227
+ camera_edge_color,
228
+ imgs[i] if i < len(imgs) else None,
229
+ focals[i],
230
+ screen_width=cam_size,
231
+ )
232
+
233
+ scene.show(line_settings={"point_size": point_size})
234
+
235
+
236
+ def add_scene_cam(
237
+ scene, pose_c2w, edge_color, image=None, focal=None, imsize=None, screen_width=0.03
238
+ ):
239
+ if image is not None:
240
+ H, W, THREE = image.shape
241
+ assert THREE == 3
242
+ if image.dtype != np.uint8:
243
+ image = np.uint8(255 * image)
244
+ elif imsize is not None:
245
+ W, H = imsize
246
+ elif focal is not None:
247
+ H = W = focal / 1.1
248
+ else:
249
+ H = W = 1
250
+
251
+ if focal is None:
252
+ focal = min(H, W) * 1.1 # default value
253
+ elif isinstance(focal, np.ndarray):
254
+ focal = focal[0]
255
+
256
+ # create fake camera
257
+ height = focal * screen_width / H
258
+ width = screen_width * 0.5**0.5
259
+ rot45 = np.eye(4)
260
+ rot45[:3, :3] = Rotation.from_euler("z", np.deg2rad(45)).as_matrix()
261
+ rot45[2, 3] = -height # set the tip of the cone = optical center
262
+ aspect_ratio = np.eye(4)
263
+ aspect_ratio[0, 0] = W / H
264
+ transform = pose_c2w @ OPENGL @ aspect_ratio @ rot45
265
+ cam = trimesh.creation.cone(width, height, sections=4) # , transform=transform)
266
+
267
+ # this is the image
268
+ if image is not None:
269
+ vertices = geotrf(transform, cam.vertices[[4, 5, 1, 3]])
270
+ faces = np.array([[0, 1, 2], [0, 2, 3], [2, 1, 0], [3, 2, 0]])
271
+ img = trimesh.Trimesh(vertices=vertices, faces=faces)
272
+ uv_coords = np.float32([[0, 0], [1, 0], [1, 1], [0, 1]])
273
+ img.visual = trimesh.visual.TextureVisuals(
274
+ uv_coords, image=PIL.Image.fromarray(image)
275
+ )
276
+ scene.add_geometry(img)
277
+
278
+ # this is the camera mesh
279
+ rot2 = np.eye(4)
280
+ rot2[:3, :3] = Rotation.from_euler("z", np.deg2rad(2)).as_matrix()
281
+ vertices = np.r_[cam.vertices, 0.95 * cam.vertices, geotrf(rot2, cam.vertices)]
282
+ vertices = geotrf(transform, vertices)
283
+ faces = []
284
+ for face in cam.faces:
285
+ if 0 in face:
286
+ continue
287
+ a, b, c = face
288
+ a2, b2, c2 = face + len(cam.vertices)
289
+ a3, b3, c3 = face + 2 * len(cam.vertices)
290
+
291
+ # add 3 pseudo-edges
292
+ faces.append((a, b, b2))
293
+ faces.append((a, a2, c))
294
+ faces.append((c2, b, c))
295
+
296
+ faces.append((a, b, b3))
297
+ faces.append((a, a3, c))
298
+ faces.append((c3, b, c))
299
+
300
+ # no culling
301
+ faces += [(c, b, a) for a, b, c in faces]
302
+
303
+ cam = trimesh.Trimesh(vertices=vertices, faces=faces)
304
+ cam.visual.face_colors[:, :3] = edge_color
305
+ scene.add_geometry(cam)
306
+
307
+
308
+ def cat(a, b):
309
+ return np.concatenate((a.reshape(-1, 3), b.reshape(-1, 3)))
310
+
311
+
312
+ OPENGL = np.array([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]])
313
+
314
+
315
+ CAM_COLORS = [
316
+ (255, 0, 0),
317
+ (0, 0, 255),
318
+ (0, 255, 0),
319
+ (255, 0, 255),
320
+ (255, 204, 0),
321
+ (0, 204, 204),
322
+ (128, 255, 255),
323
+ (255, 128, 255),
324
+ (255, 255, 128),
325
+ (0, 0, 0),
326
+ (128, 128, 128),
327
+ ]
328
+
329
+
330
+ def uint8(colors):
331
+ if not isinstance(colors, np.ndarray):
332
+ colors = np.array(colors)
333
+ if np.issubdtype(colors.dtype, np.floating):
334
+ colors *= 255
335
+ assert 0 <= colors.min() and colors.max() < 256
336
+ return np.uint8(colors)
337
+
338
+
339
+ def segment_sky(image):
340
+ import cv2
341
+ from scipy import ndimage
342
+
343
+ # Convert to HSV
344
+ image = to_numpy(image)
345
+ if np.issubdtype(image.dtype, np.floating):
346
+ image = np.uint8(255 * image.clip(min=0, max=1))
347
+ hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
348
+
349
+ # Define range for blue color and create mask
350
+ lower_blue = np.array([0, 0, 100])
351
+ upper_blue = np.array([30, 255, 255])
352
+ mask = cv2.inRange(hsv, lower_blue, upper_blue).view(bool)
353
+
354
+ # add luminous gray
355
+ mask |= (hsv[:, :, 1] < 10) & (hsv[:, :, 2] > 150)
356
+ mask |= (hsv[:, :, 1] < 30) & (hsv[:, :, 2] > 180)
357
+ mask |= (hsv[:, :, 1] < 50) & (hsv[:, :, 2] > 220)
358
+
359
+ # Morphological operations
360
+ kernel = np.ones((5, 5), np.uint8)
361
+ mask2 = ndimage.binary_opening(mask, structure=kernel)
362
+
363
+ # keep only largest CC
364
+ _, labels, stats, _ = cv2.connectedComponentsWithStats(
365
+ mask2.view(np.uint8), connectivity=8
366
+ )
367
+ cc_sizes = stats[1:, cv2.CC_STAT_AREA]
368
+ order = cc_sizes.argsort()[::-1] # bigger first
369
+ i = 0
370
+ selection = []
371
+ while i < len(order) and cc_sizes[order[i]] > cc_sizes[order[0]] / 2:
372
+ selection.append(1 + order[i])
373
+ i += 1
374
+ mask3 = np.in1d(labels, selection).reshape(labels.shape)
375
+
376
+ # Apply mask
377
+ return torch.from_numpy(mask3)
src/visualization/annotation.py ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pathlib import Path
2
+ from string import ascii_letters, digits, punctuation
3
+
4
+ import numpy as np
5
+ import torch
6
+ from einops import rearrange
7
+ from jaxtyping import Float
8
+ from PIL import Image, ImageDraw, ImageFont
9
+ from torch import Tensor
10
+
11
+ from .layout import vcat
12
+
13
+ EXPECTED_CHARACTERS = digits + punctuation + ascii_letters
14
+
15
+
16
+ def draw_label(
17
+ text: str,
18
+ font: Path,
19
+ font_size: int,
20
+ device: torch.device = torch.device("cpu"),
21
+ ) -> Float[Tensor, "3 height width"]:
22
+ """Draw a black label on a white background with no border."""
23
+ try:
24
+ font = ImageFont.truetype(str(font), font_size)
25
+ except OSError:
26
+ font = ImageFont.load_default()
27
+ left, _, right, _ = font.getbbox(text)
28
+ width = right - left
29
+ _, top, _, bottom = font.getbbox(EXPECTED_CHARACTERS)
30
+ height = bottom - top
31
+ image = Image.new("RGB", (width, height), color="white")
32
+ draw = ImageDraw.Draw(image)
33
+ draw.text((0, 0), text, font=font, fill="black")
34
+ image = torch.tensor(np.array(image) / 255, dtype=torch.float32, device=device)
35
+ return rearrange(image, "h w c -> c h w")
36
+
37
+
38
+ def add_label(
39
+ image: Float[Tensor, "3 width height"],
40
+ label: str,
41
+ font: Path = Path("assets/Inter-Regular.otf"),
42
+ font_size: int = 24,
43
+ ) -> Float[Tensor, "3 width_with_label height_with_label"]:
44
+ return vcat(
45
+ draw_label(label, font, font_size, image.device),
46
+ image,
47
+ align="left",
48
+ gap=4,
49
+ )
src/visualization/camera_trajectory/interpolation.py ADDED
@@ -0,0 +1,255 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from einops import einsum, rearrange, reduce
3
+ from jaxtyping import Float
4
+ from scipy.spatial.transform import Rotation as R
5
+ from torch import Tensor
6
+
7
+
8
+ def interpolate_intrinsics(
9
+ initial: Float[Tensor, "*#batch 3 3"],
10
+ final: Float[Tensor, "*#batch 3 3"],
11
+ t: Float[Tensor, " time_step"],
12
+ ) -> Float[Tensor, "*batch time_step 3 3"]:
13
+ initial = rearrange(initial, "... i j -> ... () i j")
14
+ final = rearrange(final, "... i j -> ... () i j")
15
+ t = rearrange(t, "t -> t () ()")
16
+ return initial + (final - initial) * t
17
+
18
+
19
+ def intersect_rays(
20
+ a_origins: Float[Tensor, "*#batch dim"],
21
+ a_directions: Float[Tensor, "*#batch dim"],
22
+ b_origins: Float[Tensor, "*#batch dim"],
23
+ b_directions: Float[Tensor, "*#batch dim"],
24
+ ) -> Float[Tensor, "*batch dim"]:
25
+ """Compute the least-squares intersection of rays. Uses the math from here:
26
+ https://math.stackexchange.com/a/1762491/286022
27
+ """
28
+
29
+ # Broadcast and stack the tensors.
30
+ a_origins, a_directions, b_origins, b_directions = torch.broadcast_tensors(
31
+ a_origins, a_directions, b_origins, b_directions
32
+ )
33
+ origins = torch.stack((a_origins, b_origins), dim=-2)
34
+ directions = torch.stack((a_directions, b_directions), dim=-2)
35
+
36
+ # Compute n_i * n_i^T - eye(3) from the equation.
37
+ n = einsum(directions, directions, "... n i, ... n j -> ... n i j")
38
+ n = n - torch.eye(3, dtype=origins.dtype, device=origins.device)
39
+
40
+ # Compute the left-hand side of the equation.
41
+ lhs = reduce(n, "... n i j -> ... i j", "sum")
42
+
43
+ # Compute the right-hand side of the equation.
44
+ rhs = einsum(n, origins, "... n i j, ... n j -> ... n i")
45
+ rhs = reduce(rhs, "... n i -> ... i", "sum")
46
+
47
+ # Left-matrix-multiply both sides by the inverse of lhs to find p.
48
+ return torch.linalg.lstsq(lhs, rhs).solution
49
+
50
+
51
+ def normalize(a: Float[Tensor, "*#batch dim"]) -> Float[Tensor, "*#batch dim"]:
52
+ return a / a.norm(dim=-1, keepdim=True)
53
+
54
+
55
+ def generate_coordinate_frame(
56
+ y: Float[Tensor, "*#batch 3"],
57
+ z: Float[Tensor, "*#batch 3"],
58
+ ) -> Float[Tensor, "*batch 3 3"]:
59
+ """Generate a coordinate frame given perpendicular, unit-length Y and Z vectors."""
60
+ y, z = torch.broadcast_tensors(y, z)
61
+ return torch.stack([y.cross(z), y, z], dim=-1)
62
+
63
+
64
+ def generate_rotation_coordinate_frame(
65
+ a: Float[Tensor, "*#batch 3"],
66
+ b: Float[Tensor, "*#batch 3"],
67
+ eps: float = 1e-4,
68
+ ) -> Float[Tensor, "*batch 3 3"]:
69
+ """Generate a coordinate frame where the Y direction is normal to the plane defined
70
+ by unit vectors a and b. The other axes are arbitrary."""
71
+ device = a.device
72
+
73
+ # Replace every entry in b that's parallel to the corresponding entry in a with an
74
+ # arbitrary vector.
75
+ b = b.detach().clone()
76
+ parallel = (einsum(a, b, "... i, ... i -> ...").abs() - 1).abs() < eps
77
+ b[parallel] = torch.tensor([0, 0, 1], dtype=b.dtype, device=device)
78
+ parallel = (einsum(a, b, "... i, ... i -> ...").abs() - 1).abs() < eps
79
+ b[parallel] = torch.tensor([0, 1, 0], dtype=b.dtype, device=device)
80
+
81
+ # Generate the coordinate frame. The initial cross product defines the plane.
82
+ return generate_coordinate_frame(normalize(a.cross(b)), a)
83
+
84
+
85
+ def matrix_to_euler(
86
+ rotations: Float[Tensor, "*batch 3 3"],
87
+ pattern: str,
88
+ ) -> Float[Tensor, "*batch 3"]:
89
+ *batch, _, _ = rotations.shape
90
+ rotations = rotations.reshape(-1, 3, 3)
91
+ angles_np = R.from_matrix(rotations.detach().cpu().numpy()).as_euler(pattern)
92
+ rotations = torch.tensor(angles_np, dtype=rotations.dtype, device=rotations.device)
93
+ return rotations.reshape(*batch, 3)
94
+
95
+
96
+ def euler_to_matrix(
97
+ rotations: Float[Tensor, "*batch 3"],
98
+ pattern: str,
99
+ ) -> Float[Tensor, "*batch 3 3"]:
100
+ *batch, _ = rotations.shape
101
+ rotations = rotations.reshape(-1, 3)
102
+ matrix_np = R.from_euler(pattern, rotations.detach().cpu().numpy()).as_matrix()
103
+ rotations = torch.tensor(matrix_np, dtype=rotations.dtype, device=rotations.device)
104
+ return rotations.reshape(*batch, 3, 3)
105
+
106
+
107
+ def extrinsics_to_pivot_parameters(
108
+ extrinsics: Float[Tensor, "*#batch 4 4"],
109
+ pivot_coordinate_frame: Float[Tensor, "*#batch 3 3"],
110
+ pivot_point: Float[Tensor, "*#batch 3"],
111
+ ) -> Float[Tensor, "*batch 5"]:
112
+ """Convert the extrinsics to a representation with 5 degrees of freedom:
113
+ 1. Distance from pivot point in the "X" (look cross pivot axis) direction.
114
+ 2. Distance from pivot point in the "Y" (pivot axis) direction.
115
+ 3. Distance from pivot point in the Z (look) direction
116
+ 4. Angle in plane
117
+ 5. Twist (rotation not in plane)
118
+ """
119
+
120
+ # The pivot coordinate frame's Z axis is normal to the plane.
121
+ pivot_axis = pivot_coordinate_frame[..., :, 1]
122
+
123
+ # Compute the translation elements of the pivot parametrization.
124
+ translation_frame = generate_coordinate_frame(pivot_axis, extrinsics[..., :3, 2])
125
+ origin = extrinsics[..., :3, 3]
126
+ delta = pivot_point - origin
127
+ translation = einsum(translation_frame, delta, "... i j, ... i -> ... j")
128
+
129
+ # Add the rotation elements of the pivot parametrization.
130
+ inverted = pivot_coordinate_frame.inverse() @ extrinsics[..., :3, :3]
131
+ y, _, z = matrix_to_euler(inverted, "YXZ").unbind(dim=-1)
132
+
133
+ return torch.cat([translation, y[..., None], z[..., None]], dim=-1)
134
+
135
+
136
+ def pivot_parameters_to_extrinsics(
137
+ parameters: Float[Tensor, "*#batch 5"],
138
+ pivot_coordinate_frame: Float[Tensor, "*#batch 3 3"],
139
+ pivot_point: Float[Tensor, "*#batch 3"],
140
+ ) -> Float[Tensor, "*batch 4 4"]:
141
+ translation, y, z = parameters.split((3, 1, 1), dim=-1)
142
+
143
+ euler = torch.cat((y, torch.zeros_like(y), z), dim=-1)
144
+ rotation = pivot_coordinate_frame @ euler_to_matrix(euler, "YXZ")
145
+
146
+ # The pivot coordinate frame's Z axis is normal to the plane.
147
+ pivot_axis = pivot_coordinate_frame[..., :, 1]
148
+
149
+ translation_frame = generate_coordinate_frame(pivot_axis, rotation[..., :3, 2])
150
+ delta = einsum(translation_frame, translation, "... i j, ... j -> ... i")
151
+ origin = pivot_point - delta
152
+
153
+ *batch, _ = origin.shape
154
+ extrinsics = torch.eye(4, dtype=parameters.dtype, device=parameters.device)
155
+ extrinsics = extrinsics.broadcast_to((*batch, 4, 4)).clone()
156
+ extrinsics[..., 3, 3] = 1
157
+ extrinsics[..., :3, :3] = rotation
158
+ extrinsics[..., :3, 3] = origin
159
+ return extrinsics
160
+
161
+
162
+ def interpolate_circular(
163
+ a: Float[Tensor, "*#batch"],
164
+ b: Float[Tensor, "*#batch"],
165
+ t: Float[Tensor, "*#batch"],
166
+ ) -> Float[Tensor, " *batch"]:
167
+ a, b, t = torch.broadcast_tensors(a, b, t)
168
+
169
+ tau = 2 * torch.pi
170
+ a = a % tau
171
+ b = b % tau
172
+
173
+ # Consider piecewise edge cases.
174
+ d = (b - a).abs()
175
+ a_left = a - tau
176
+ d_left = (b - a_left).abs()
177
+ a_right = a + tau
178
+ d_right = (b - a_right).abs()
179
+ use_d = (d < d_left) & (d < d_right)
180
+ use_d_left = (d_left < d_right) & (~use_d)
181
+ use_d_right = (~use_d) & (~use_d_left)
182
+
183
+ result = a + (b - a) * t
184
+ result[use_d_left] = (a_left + (b - a_left) * t)[use_d_left]
185
+ result[use_d_right] = (a_right + (b - a_right) * t)[use_d_right]
186
+
187
+ return result
188
+
189
+
190
+ def interpolate_pivot_parameters(
191
+ initial: Float[Tensor, "*#batch 5"],
192
+ final: Float[Tensor, "*#batch 5"],
193
+ t: Float[Tensor, " time_step"],
194
+ ) -> Float[Tensor, "*batch time_step 5"]:
195
+ initial = rearrange(initial, "... d -> ... () d")
196
+ final = rearrange(final, "... d -> ... () d")
197
+ t = rearrange(t, "t -> t ()")
198
+ ti, ri = initial.split((3, 2), dim=-1)
199
+ tf, rf = final.split((3, 2), dim=-1)
200
+
201
+ t_lerp = ti + (tf - ti) * t
202
+ r_lerp = interpolate_circular(ri, rf, t)
203
+
204
+ return torch.cat((t_lerp, r_lerp), dim=-1)
205
+
206
+
207
+ @torch.no_grad()
208
+ def interpolate_extrinsics(
209
+ initial: Float[Tensor, "*#batch 4 4"],
210
+ final: Float[Tensor, "*#batch 4 4"],
211
+ t: Float[Tensor, " time_step"],
212
+ eps: float = 1e-4,
213
+ ) -> Float[Tensor, "*batch time_step 4 4"]:
214
+ """Interpolate extrinsics by rotating around their "focus point," which is the
215
+ least-squares intersection between the look vectors of the initial and final
216
+ extrinsics.
217
+ """
218
+
219
+ initial = initial.type(torch.float64)
220
+ final = final.type(torch.float64)
221
+ t = t.type(torch.float64)
222
+
223
+ # Based on the dot product between the look vectors, pick from one of two cases:
224
+ # 1. Look vectors are parallel: interpolate about their origins' midpoint.
225
+ # 3. Look vectors aren't parallel: interpolate about their focus point.
226
+ initial_look = initial[..., :3, 2]
227
+ final_look = final[..., :3, 2]
228
+ dot_products = einsum(initial_look, final_look, "... i, ... i -> ...")
229
+ parallel_mask = (dot_products.abs() - 1).abs() < eps
230
+
231
+ # Pick focus points.
232
+ initial_origin = initial[..., :3, 3]
233
+ final_origin = final[..., :3, 3]
234
+ pivot_point = 0.5 * (initial_origin + final_origin)
235
+ pivot_point[~parallel_mask] = intersect_rays(
236
+ initial_origin[~parallel_mask],
237
+ initial_look[~parallel_mask],
238
+ final_origin[~parallel_mask],
239
+ final_look[~parallel_mask],
240
+ )
241
+
242
+ # Convert to pivot parameters.
243
+ pivot_frame = generate_rotation_coordinate_frame(initial_look, final_look, eps=eps)
244
+ initial_params = extrinsics_to_pivot_parameters(initial, pivot_frame, pivot_point)
245
+ final_params = extrinsics_to_pivot_parameters(final, pivot_frame, pivot_point)
246
+
247
+ # Interpolate the pivot parameters.
248
+ interpolated_params = interpolate_pivot_parameters(initial_params, final_params, t)
249
+
250
+ # Convert back.
251
+ return pivot_parameters_to_extrinsics(
252
+ interpolated_params.type(torch.float32),
253
+ rearrange(pivot_frame, "... i j -> ... () i j").type(torch.float32),
254
+ rearrange(pivot_point, "... xyz -> ... () xyz").type(torch.float32),
255
+ )
src/visualization/camera_trajectory/spin.py ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+ import torch
3
+ from einops import repeat
4
+ from jaxtyping import Float
5
+ from scipy.spatial.transform import Rotation as R
6
+ from torch import Tensor
7
+
8
+
9
+ def generate_spin(
10
+ num_frames: int,
11
+ device: torch.device,
12
+ elevation: float,
13
+ radius: float,
14
+ ) -> Float[Tensor, "frame 4 4"]:
15
+ # Translate back along the camera's look vector.
16
+ tf_translation = torch.eye(4, dtype=torch.float32, device=device)
17
+ tf_translation[:2] *= -1
18
+ tf_translation[2, 3] = -radius
19
+
20
+ # Generate the transformation for the azimuth.
21
+ phi = 2 * np.pi * (np.arange(num_frames) / num_frames)
22
+ rotation_vectors = np.stack([np.zeros_like(phi), phi, np.zeros_like(phi)], axis=-1)
23
+
24
+ azimuth = R.from_rotvec(rotation_vectors).as_matrix()
25
+ azimuth = torch.tensor(azimuth, dtype=torch.float32, device=device)
26
+ tf_azimuth = torch.eye(4, dtype=torch.float32, device=device)
27
+ tf_azimuth = repeat(tf_azimuth, "i j -> b i j", b=num_frames).clone()
28
+ tf_azimuth[:, :3, :3] = azimuth
29
+
30
+ # Generate the transformation for the elevation.
31
+ deg_elevation = np.deg2rad(elevation)
32
+ elevation = R.from_rotvec(np.array([deg_elevation, 0, 0], dtype=np.float32))
33
+ elevation = torch.tensor(elevation.as_matrix())
34
+ tf_elevation = torch.eye(4, dtype=torch.float32, device=device)
35
+ tf_elevation[:3, :3] = elevation
36
+
37
+ return tf_azimuth @ tf_elevation @ tf_translation
src/visualization/camera_trajectory/wobble.py ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from einops import rearrange
3
+ from jaxtyping import Float
4
+ from torch import Tensor
5
+
6
+
7
+ @torch.no_grad()
8
+ def generate_wobble_transformation(
9
+ radius: Float[Tensor, "*#batch"],
10
+ t: Float[Tensor, " time_step"],
11
+ num_rotations: int = 1,
12
+ scale_radius_with_t: bool = True,
13
+ ) -> Float[Tensor, "*batch time_step 4 4"]:
14
+ # Generate a translation in the image plane.
15
+ tf = torch.eye(4, dtype=torch.float32, device=t.device)
16
+ tf = tf.broadcast_to((*radius.shape, t.shape[0], 4, 4)).clone()
17
+ radius = radius[..., None]
18
+ if scale_radius_with_t:
19
+ radius = radius * t
20
+ tf[..., 0, 3] = torch.sin(2 * torch.pi * num_rotations * t) * radius
21
+ tf[..., 1, 3] = -torch.cos(2 * torch.pi * num_rotations * t) * radius
22
+ return tf
23
+
24
+
25
+ @torch.no_grad()
26
+ def generate_wobble(
27
+ extrinsics: Float[Tensor, "*#batch 4 4"],
28
+ radius: Float[Tensor, "*#batch"],
29
+ t: Float[Tensor, " time_step"],
30
+ ) -> Float[Tensor, "*batch time_step 4 4"]:
31
+ tf = generate_wobble_transformation(radius, t)
32
+ return rearrange(extrinsics, "... i j -> ... () i j") @ tf
src/visualization/color_map.py ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from colorspacious import cspace_convert
3
+ from einops import rearrange
4
+ from jaxtyping import Float
5
+ from matplotlib import cm
6
+ from torch import Tensor
7
+
8
+
9
+ def apply_color_map(
10
+ x: Float[Tensor, " *batch"],
11
+ color_map: str = "inferno",
12
+ ) -> Float[Tensor, "*batch 3"]:
13
+ cmap = cm.get_cmap(color_map)
14
+
15
+ # Convert to NumPy so that Matplotlib color maps can be used.
16
+ mapped = cmap(x.detach().clip(min=0, max=1).cpu().numpy())[..., :3]
17
+
18
+ # Convert back to the original format.
19
+ return torch.tensor(mapped, device=x.device, dtype=torch.float32)
20
+
21
+
22
+ def apply_color_map_to_image(
23
+ image: Float[Tensor, "*batch height width"],
24
+ color_map: str = "inferno",
25
+ ) -> Float[Tensor, "*batch 3 height with"]:
26
+ image = apply_color_map(image, color_map)
27
+ return rearrange(image, "... h w c -> ... c h w")
28
+
29
+
30
+ def apply_color_map_2d(
31
+ x: Float[Tensor, "*#batch"],
32
+ y: Float[Tensor, "*#batch"],
33
+ ) -> Float[Tensor, "*batch 3"]:
34
+ red = cspace_convert((189, 0, 0), "sRGB255", "CIELab")
35
+ blue = cspace_convert((0, 45, 255), "sRGB255", "CIELab")
36
+ white = cspace_convert((255, 255, 255), "sRGB255", "CIELab")
37
+ x_np = x.detach().clip(min=0, max=1).cpu().numpy()[..., None]
38
+ y_np = y.detach().clip(min=0, max=1).cpu().numpy()[..., None]
39
+
40
+ # Interpolate between red and blue on the x axis.
41
+ interpolated = x_np * red + (1 - x_np) * blue
42
+
43
+ # Interpolate between color and white on the y axis.
44
+ interpolated = y_np * interpolated + (1 - y_np) * white
45
+
46
+ # Convert to RGB.
47
+ rgb = cspace_convert(interpolated, "CIELab", "sRGB1")
48
+ return torch.tensor(rgb, device=x.device, dtype=torch.float32).clip(min=0, max=1)
src/visualization/colors.py ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from PIL import ImageColor
2
+
3
+ # https://sashamaps.net/docs/resources/20-colors/
4
+ DISTINCT_COLORS = [
5
+ "#e6194b",
6
+ "#3cb44b",
7
+ "#ffe119",
8
+ "#4363d8",
9
+ "#f58231",
10
+ "#911eb4",
11
+ "#46f0f0",
12
+ "#f032e6",
13
+ "#bcf60c",
14
+ "#fabebe",
15
+ "#008080",
16
+ "#e6beff",
17
+ "#9a6324",
18
+ "#fffac8",
19
+ "#800000",
20
+ "#aaffc3",
21
+ "#808000",
22
+ "#ffd8b1",
23
+ "#000075",
24
+ "#808080",
25
+ "#ffffff",
26
+ "#000000",
27
+ ]
28
+
29
+
30
+ def get_distinct_color(index: int) -> tuple[float, float, float]:
31
+ hex = DISTINCT_COLORS[index % len(DISTINCT_COLORS)]
32
+ return tuple(x / 255 for x in ImageColor.getcolor(hex, "RGB"))
src/visualization/drawing/cameras.py ADDED
@@ -0,0 +1,195 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional
2
+
3
+ import torch
4
+ from einops import einsum, rearrange, repeat
5
+ from jaxtyping import Float
6
+ from torch import Tensor
7
+
8
+ from ...geometry.projection import unproject
9
+ from ..annotation import add_label
10
+ from .lines import draw_lines
11
+ from .types import Scalar, sanitize_scalar
12
+
13
+
14
+ def draw_cameras(
15
+ resolution: int,
16
+ extrinsics: Float[Tensor, "batch 4 4"],
17
+ intrinsics: Float[Tensor, "batch 3 3"],
18
+ color: Float[Tensor, "batch 3"],
19
+ near: Optional[Scalar] = None,
20
+ far: Optional[Scalar] = None,
21
+ margin: float = 0.1, # relative to AABB
22
+ frustum_scale: float = 0.05, # relative to image resolution
23
+ ) -> Float[Tensor, "3 3 height width"]:
24
+ device = extrinsics.device
25
+
26
+ # Compute scene bounds.
27
+ minima, maxima = compute_aabb(extrinsics, intrinsics, near, far)
28
+ scene_minima, scene_maxima = compute_equal_aabb_with_margin(
29
+ minima, maxima, margin=margin
30
+ )
31
+ span = (scene_maxima - scene_minima).max()
32
+
33
+ # Compute frustum locations.
34
+ corner_depth = (span * frustum_scale)[None]
35
+ frustum_corners = unproject_frustum_corners(extrinsics, intrinsics, corner_depth)
36
+ if near is not None:
37
+ near_corners = unproject_frustum_corners(extrinsics, intrinsics, near)
38
+ if far is not None:
39
+ far_corners = unproject_frustum_corners(extrinsics, intrinsics, far)
40
+
41
+ # Project the cameras onto each axis-aligned plane.
42
+ projections = []
43
+ for projected_axis in range(3):
44
+ image = torch.zeros(
45
+ (3, resolution, resolution),
46
+ dtype=torch.float32,
47
+ device=device,
48
+ )
49
+ image_x_axis = (projected_axis + 1) % 3
50
+ image_y_axis = (projected_axis + 2) % 3
51
+
52
+ def project(points: Float[Tensor, "*batch 3"]) -> Float[Tensor, "*batch 2"]:
53
+ x = points[..., image_x_axis]
54
+ y = points[..., image_y_axis]
55
+ return torch.stack([x, y], dim=-1)
56
+
57
+ x_range, y_range = torch.stack(
58
+ (project(scene_minima), project(scene_maxima)), dim=-1
59
+ )
60
+
61
+ # Draw near and far planes.
62
+ if near is not None:
63
+ projected_near_corners = project(near_corners)
64
+ image = draw_lines(
65
+ image,
66
+ rearrange(projected_near_corners, "b p xy -> (b p) xy"),
67
+ rearrange(projected_near_corners.roll(1, 1), "b p xy -> (b p) xy"),
68
+ color=0.25,
69
+ width=2,
70
+ x_range=x_range,
71
+ y_range=y_range,
72
+ )
73
+ if far is not None:
74
+ projected_far_corners = project(far_corners)
75
+ image = draw_lines(
76
+ image,
77
+ rearrange(projected_far_corners, "b p xy -> (b p) xy"),
78
+ rearrange(projected_far_corners.roll(1, 1), "b p xy -> (b p) xy"),
79
+ color=0.25,
80
+ width=2,
81
+ x_range=x_range,
82
+ y_range=y_range,
83
+ )
84
+ if near is not None and far is not None:
85
+ image = draw_lines(
86
+ image,
87
+ rearrange(projected_near_corners, "b p xy -> (b p) xy"),
88
+ rearrange(projected_far_corners, "b p xy -> (b p) xy"),
89
+ color=0.25,
90
+ width=2,
91
+ x_range=x_range,
92
+ y_range=y_range,
93
+ )
94
+
95
+ # Draw the camera frustums themselves.
96
+ projected_origins = project(extrinsics[:, :3, 3])
97
+ projected_frustum_corners = project(frustum_corners)
98
+ start = [
99
+ repeat(projected_origins, "b xy -> (b p) xy", p=4),
100
+ rearrange(projected_frustum_corners.roll(1, 1), "b p xy -> (b p) xy"),
101
+ ]
102
+ start = rearrange(torch.cat(start, dim=0), "(r b p) xy -> (b r p) xy", r=2, p=4)
103
+ image = draw_lines(
104
+ image,
105
+ start,
106
+ repeat(projected_frustum_corners, "b p xy -> (b r p) xy", r=2),
107
+ color=repeat(color, "b c -> (b r p) c", r=2, p=4),
108
+ width=2,
109
+ x_range=x_range,
110
+ y_range=y_range,
111
+ )
112
+
113
+ x_name = "XYZ"[image_x_axis]
114
+ y_name = "XYZ"[image_y_axis]
115
+ image = add_label(image, f"{x_name}{y_name} Projection")
116
+
117
+ # TODO: Draw axis indicators.
118
+ projections.append(image)
119
+
120
+ return torch.stack(projections)
121
+
122
+
123
+ def compute_aabb(
124
+ extrinsics: Float[Tensor, "batch 4 4"],
125
+ intrinsics: Float[Tensor, "batch 3 3"],
126
+ near: Optional[Scalar] = None,
127
+ far: Optional[Scalar] = None,
128
+ ) -> tuple[
129
+ Float[Tensor, "3"], # minima of the scene
130
+ Float[Tensor, "3"], # maxima of the scene
131
+ ]:
132
+ """Compute an axis-aligned bounding box for the camera frustums."""
133
+
134
+ device = extrinsics.device
135
+
136
+ # These points are included in the AABB.
137
+ points = [extrinsics[:, :3, 3]]
138
+
139
+ if near is not None:
140
+ near = sanitize_scalar(near, device)
141
+ corners = unproject_frustum_corners(extrinsics, intrinsics, near)
142
+ points.append(rearrange(corners, "b p xyz -> (b p) xyz"))
143
+
144
+ if far is not None:
145
+ far = sanitize_scalar(far, device)
146
+ corners = unproject_frustum_corners(extrinsics, intrinsics, far)
147
+ points.append(rearrange(corners, "b p xyz -> (b p) xyz"))
148
+
149
+ points = torch.cat(points, dim=0)
150
+ return points.min(dim=0).values, points.max(dim=0).values
151
+
152
+
153
+ def compute_equal_aabb_with_margin(
154
+ minima: Float[Tensor, "*#batch 3"],
155
+ maxima: Float[Tensor, "*#batch 3"],
156
+ margin: float = 0.1,
157
+ ) -> tuple[
158
+ Float[Tensor, "*batch 3"], # minima of the scene
159
+ Float[Tensor, "*batch 3"], # maxima of the scene
160
+ ]:
161
+ midpoint = (maxima + minima) * 0.5
162
+ span = (maxima - minima).max() * (1 + margin)
163
+ scene_minima = midpoint - 0.5 * span
164
+ scene_maxima = midpoint + 0.5 * span
165
+ return scene_minima, scene_maxima
166
+
167
+
168
+ def unproject_frustum_corners(
169
+ extrinsics: Float[Tensor, "batch 4 4"],
170
+ intrinsics: Float[Tensor, "batch 3 3"],
171
+ depth: Float[Tensor, "#batch"],
172
+ ) -> Float[Tensor, "batch 4 3"]:
173
+ device = extrinsics.device
174
+
175
+ # Get coordinates for the corners. Following them in a circle makes a rectangle.
176
+ xy = torch.linspace(0, 1, 2, device=device)
177
+ xy = torch.stack(torch.meshgrid(xy, xy, indexing="xy"), dim=-1)
178
+ xy = rearrange(xy, "i j xy -> (i j) xy")
179
+ xy = xy[torch.tensor([0, 1, 3, 2], device=device)]
180
+
181
+ # Get ray directions in camera space.
182
+ directions = unproject(
183
+ xy,
184
+ torch.ones(1, dtype=torch.float32, device=device),
185
+ rearrange(intrinsics, "b i j -> b () i j"),
186
+ )
187
+
188
+ # Divide by the z coordinate so that multiplying by depth will produce orthographic
189
+ # depth (z depth) as opposed to Euclidean depth (distance from the camera).
190
+ directions = directions / directions[..., -1:]
191
+ directions = einsum(extrinsics[..., :3, :3], directions, "b i j, b r j -> b r i")
192
+
193
+ origins = rearrange(extrinsics[:, :3, 3], "b xyz -> b () xyz")
194
+ depth = rearrange(depth, "b -> b () ()")
195
+ return origins + depth * directions
src/visualization/drawing/coordinate_conversion.py ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional, Protocol, runtime_checkable
2
+
3
+ import torch
4
+ from jaxtyping import Float
5
+ from torch import Tensor
6
+
7
+ from .types import Pair, sanitize_pair
8
+
9
+
10
+ @runtime_checkable
11
+ class ConversionFunction(Protocol):
12
+ def __call__(
13
+ self,
14
+ xy: Float[Tensor, "*batch 2"],
15
+ ) -> Float[Tensor, "*batch 2"]:
16
+ pass
17
+
18
+
19
+ def generate_conversions(
20
+ shape: tuple[int, int],
21
+ device: torch.device,
22
+ x_range: Optional[Pair] = None,
23
+ y_range: Optional[Pair] = None,
24
+ ) -> tuple[
25
+ ConversionFunction, # conversion from world coordinates to pixel coordinates
26
+ ConversionFunction, # conversion from pixel coordinates to world coordinates
27
+ ]:
28
+ h, w = shape
29
+ x_range = sanitize_pair((0, w) if x_range is None else x_range, device)
30
+ y_range = sanitize_pair((0, h) if y_range is None else y_range, device)
31
+ minima, maxima = torch.stack((x_range, y_range), dim=-1)
32
+ wh = torch.tensor((w, h), dtype=torch.float32, device=device)
33
+
34
+ def convert_world_to_pixel(
35
+ xy: Float[Tensor, "*batch 2"],
36
+ ) -> Float[Tensor, "*batch 2"]:
37
+ return (xy - minima) / (maxima - minima) * wh
38
+
39
+ def convert_pixel_to_world(
40
+ xy: Float[Tensor, "*batch 2"],
41
+ ) -> Float[Tensor, "*batch 2"]:
42
+ return xy / wh * (maxima - minima) + minima
43
+
44
+ return convert_world_to_pixel, convert_pixel_to_world
src/visualization/drawing/lines.py ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Literal, Optional
2
+
3
+ import torch
4
+ from einops import einsum, repeat
5
+ from jaxtyping import Float
6
+ from torch import Tensor
7
+
8
+ from .coordinate_conversion import generate_conversions
9
+ from .rendering import render_over_image
10
+ from .types import Pair, Scalar, Vector, sanitize_scalar, sanitize_vector
11
+
12
+
13
+ def draw_lines(
14
+ image: Float[Tensor, "3 height width"],
15
+ start: Vector,
16
+ end: Vector,
17
+ color: Vector,
18
+ width: Scalar,
19
+ cap: Literal["butt", "round", "square"] = "round",
20
+ num_msaa_passes: int = 1,
21
+ x_range: Optional[Pair] = None,
22
+ y_range: Optional[Pair] = None,
23
+ ) -> Float[Tensor, "3 height width"]:
24
+ device = image.device
25
+ start = sanitize_vector(start, 2, device)
26
+ end = sanitize_vector(end, 2, device)
27
+ color = sanitize_vector(color, 3, device)
28
+ width = sanitize_scalar(width, device)
29
+ (num_lines,) = torch.broadcast_shapes(
30
+ start.shape[0],
31
+ end.shape[0],
32
+ color.shape[0],
33
+ width.shape,
34
+ )
35
+
36
+ # Convert world-space points to pixel space.
37
+ _, h, w = image.shape
38
+ world_to_pixel, _ = generate_conversions((h, w), device, x_range, y_range)
39
+ start = world_to_pixel(start)
40
+ end = world_to_pixel(end)
41
+
42
+ def color_function(
43
+ xy: Float[Tensor, "point 2"],
44
+ ) -> Float[Tensor, "point 4"]:
45
+ # Define a vector between the start and end points.
46
+ delta = end - start
47
+ delta_norm = delta.norm(dim=-1, keepdim=True)
48
+ u_delta = delta / delta_norm
49
+
50
+ # Define a vector between each sample and the start point.
51
+ indicator = xy - start[:, None]
52
+
53
+ # Determine whether each sample is inside the line in the parallel direction.
54
+ extra = 0.5 * width[:, None] if cap == "square" else 0
55
+ parallel = einsum(u_delta, indicator, "l xy, l s xy -> l s")
56
+ parallel_inside_line = (parallel <= delta_norm + extra) & (parallel > -extra)
57
+
58
+ # Determine whether each sample is inside the line perpendicularly.
59
+ perpendicular = indicator - parallel[..., None] * u_delta[:, None]
60
+ perpendicular_inside_line = perpendicular.norm(dim=-1) < 0.5 * width[:, None]
61
+
62
+ inside_line = parallel_inside_line & perpendicular_inside_line
63
+
64
+ # Compute round caps.
65
+ if cap == "round":
66
+ near_start = indicator.norm(dim=-1) < 0.5 * width[:, None]
67
+ inside_line |= near_start
68
+ end_indicator = indicator = xy - end[:, None]
69
+ near_end = end_indicator.norm(dim=-1) < 0.5 * width[:, None]
70
+ inside_line |= near_end
71
+
72
+ # Determine the sample's color.
73
+ selectable_color = color.broadcast_to((num_lines, 3))
74
+ arrangement = inside_line * torch.arange(num_lines, device=device)[:, None]
75
+ top_color = selectable_color.gather(
76
+ dim=0,
77
+ index=repeat(arrangement.argmax(dim=0), "s -> s c", c=3),
78
+ )
79
+ rgba = torch.cat((top_color, inside_line.any(dim=0).float()[:, None]), dim=-1)
80
+
81
+ return rgba
82
+
83
+ return render_over_image(image, color_function, device, num_passes=num_msaa_passes)
src/visualization/drawing/points.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional
2
+
3
+ import torch
4
+ from einops import repeat
5
+ from jaxtyping import Float
6
+ from torch import Tensor
7
+
8
+ from .coordinate_conversion import generate_conversions
9
+ from .rendering import render_over_image
10
+ from .types import Pair, Scalar, Vector, sanitize_scalar, sanitize_vector
11
+
12
+
13
+ def draw_points(
14
+ image: Float[Tensor, "3 height width"],
15
+ points: Vector,
16
+ color: Vector = [1, 1, 1],
17
+ radius: Scalar = 1,
18
+ inner_radius: Scalar = 0,
19
+ num_msaa_passes: int = 1,
20
+ x_range: Optional[Pair] = None,
21
+ y_range: Optional[Pair] = None,
22
+ ) -> Float[Tensor, "3 height width"]:
23
+ device = image.device
24
+ points = sanitize_vector(points, 2, device)
25
+ color = sanitize_vector(color, 3, device)
26
+ radius = sanitize_scalar(radius, device)
27
+ inner_radius = sanitize_scalar(inner_radius, device)
28
+ (num_points,) = torch.broadcast_shapes(
29
+ points.shape[0],
30
+ color.shape[0],
31
+ radius.shape,
32
+ inner_radius.shape,
33
+ )
34
+
35
+ # Convert world-space points to pixel space.
36
+ _, h, w = image.shape
37
+ world_to_pixel, _ = generate_conversions((h, w), device, x_range, y_range)
38
+ points = world_to_pixel(points)
39
+
40
+ def color_function(
41
+ xy: Float[Tensor, "point 2"],
42
+ ) -> Float[Tensor, "point 4"]:
43
+ # Define a vector between the start and end points.
44
+ delta = xy[:, None] - points[None]
45
+ delta_norm = delta.norm(dim=-1)
46
+ mask = (delta_norm >= inner_radius[None]) & (delta_norm <= radius[None])
47
+
48
+ # Determine the sample's color.
49
+ selectable_color = color.broadcast_to((num_points, 3))
50
+ arrangement = mask * torch.arange(num_points, device=device)
51
+ top_color = selectable_color.gather(
52
+ dim=0,
53
+ index=repeat(arrangement.argmax(dim=1), "s -> s c", c=3),
54
+ )
55
+ rgba = torch.cat((top_color, mask.any(dim=1).float()[:, None]), dim=-1)
56
+
57
+ return rgba
58
+
59
+ return render_over_image(image, color_function, device, num_passes=num_msaa_passes)
src/visualization/drawing/rendering.py ADDED
@@ -0,0 +1,152 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Protocol, runtime_checkable
2
+
3
+ import torch
4
+ from einops import rearrange, reduce
5
+ from jaxtyping import Bool, Float
6
+ from torch import Tensor
7
+
8
+
9
+ @runtime_checkable
10
+ class ColorFunction(Protocol):
11
+ def __call__(
12
+ self,
13
+ xy: Float[Tensor, "point 2"],
14
+ ) -> Float[Tensor, "point 4"]: # RGBA color
15
+ pass
16
+
17
+
18
+ def generate_sample_grid(
19
+ shape: tuple[int, int],
20
+ device: torch.device,
21
+ ) -> Float[Tensor, "height width 2"]:
22
+ h, w = shape
23
+ x = torch.arange(w, device=device) + 0.5
24
+ y = torch.arange(h, device=device) + 0.5
25
+ x, y = torch.meshgrid(x, y, indexing="xy")
26
+ return torch.stack([x, y], dim=-1)
27
+
28
+
29
+ def detect_msaa_pixels(
30
+ image: Float[Tensor, "batch 4 height width"],
31
+ ) -> Bool[Tensor, "batch height width"]:
32
+ b, _, h, w = image.shape
33
+
34
+ mask = torch.zeros((b, h, w), dtype=torch.bool, device=image.device)
35
+
36
+ # Detect horizontal differences.
37
+ horizontal = (image[:, :, :, 1:] != image[:, :, :, :-1]).any(dim=1)
38
+ mask[:, :, 1:] |= horizontal
39
+ mask[:, :, :-1] |= horizontal
40
+
41
+ # Detect vertical differences.
42
+ vertical = (image[:, :, 1:, :] != image[:, :, :-1, :]).any(dim=1)
43
+ mask[:, 1:, :] |= vertical
44
+ mask[:, :-1, :] |= vertical
45
+
46
+ # Detect diagonal (top left to bottom right) differences.
47
+ tlbr = (image[:, :, 1:, 1:] != image[:, :, :-1, :-1]).any(dim=1)
48
+ mask[:, 1:, 1:] |= tlbr
49
+ mask[:, :-1, :-1] |= tlbr
50
+
51
+ # Detect diagonal (top right to bottom left) differences.
52
+ trbl = (image[:, :, :-1, 1:] != image[:, :, 1:, :-1]).any(dim=1)
53
+ mask[:, :-1, 1:] |= trbl
54
+ mask[:, 1:, :-1] |= trbl
55
+
56
+ return mask
57
+
58
+
59
+ def reduce_straight_alpha(
60
+ rgba: Float[Tensor, "batch 4 height width"],
61
+ ) -> Float[Tensor, "batch 4"]:
62
+ color, alpha = rgba.split((3, 1), dim=1)
63
+
64
+ # Color becomes a weighted average of color (weighted by alpha).
65
+ weighted_color = reduce(color * alpha, "b c h w -> b c", "sum")
66
+ alpha_sum = reduce(alpha, "b c h w -> b c", "sum")
67
+ color = weighted_color / (alpha_sum + 1e-10)
68
+
69
+ # Alpha becomes mean alpha.
70
+ alpha = reduce(alpha, "b c h w -> b c", "mean")
71
+
72
+ return torch.cat((color, alpha), dim=-1)
73
+
74
+
75
+ @torch.no_grad()
76
+ def run_msaa_pass(
77
+ xy: Float[Tensor, "batch height width 2"],
78
+ color_function: ColorFunction,
79
+ scale: float,
80
+ subdivision: int,
81
+ remaining_passes: int,
82
+ device: torch.device,
83
+ batch_size: int = int(2**16),
84
+ ) -> Float[Tensor, "batch 4 height width"]: # color (RGBA with straight alpha)
85
+ # Sample the color function.
86
+ b, h, w, _ = xy.shape
87
+ color = [
88
+ color_function(batch)
89
+ for batch in rearrange(xy, "b h w xy -> (b h w) xy").split(batch_size)
90
+ ]
91
+ color = torch.cat(color, dim=0)
92
+ color = rearrange(color, "(b h w) c -> b c h w", b=b, h=h, w=w)
93
+
94
+ # If any MSAA passes remain, subdivide.
95
+ if remaining_passes > 0:
96
+ mask = detect_msaa_pixels(color)
97
+ batch_index, row_index, col_index = torch.where(mask)
98
+ xy = xy[batch_index, row_index, col_index]
99
+
100
+ offsets = generate_sample_grid((subdivision, subdivision), device)
101
+ offsets = (offsets / subdivision - 0.5) * scale
102
+
103
+ color_fine = run_msaa_pass(
104
+ xy[:, None, None] + offsets,
105
+ color_function,
106
+ scale / subdivision,
107
+ subdivision,
108
+ remaining_passes - 1,
109
+ device,
110
+ batch_size=batch_size,
111
+ )
112
+ color[batch_index, :, row_index, col_index] = reduce_straight_alpha(color_fine)
113
+
114
+ return color
115
+
116
+
117
+ @torch.no_grad()
118
+ def render(
119
+ shape: tuple[int, int],
120
+ color_function: ColorFunction,
121
+ device: torch.device,
122
+ subdivision: int = 8,
123
+ num_passes: int = 2,
124
+ ) -> Float[Tensor, "4 height width"]: # color (RGBA with straight alpha)
125
+ xy = generate_sample_grid(shape, device)
126
+ return run_msaa_pass(
127
+ xy[None],
128
+ color_function,
129
+ 1.0,
130
+ subdivision,
131
+ num_passes,
132
+ device,
133
+ )[0]
134
+
135
+
136
+ def render_over_image(
137
+ image: Float[Tensor, "3 height width"],
138
+ color_function: ColorFunction,
139
+ device: torch.device,
140
+ subdivision: int = 8,
141
+ num_passes: int = 1,
142
+ ) -> Float[Tensor, "3 height width"]:
143
+ _, h, w = image.shape
144
+ overlay = render(
145
+ (h, w),
146
+ color_function,
147
+ device,
148
+ subdivision=subdivision,
149
+ num_passes=num_passes,
150
+ )
151
+ color, alpha = overlay.split((3, 1), dim=0)
152
+ return image * (1 - alpha) + color * alpha
src/visualization/drawing/types.py ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Iterable, Union
2
+
3
+ import torch
4
+ from einops import repeat
5
+ from jaxtyping import Float, Shaped
6
+ from torch import Tensor
7
+
8
+ Real = Union[float, int]
9
+
10
+ Vector = Union[
11
+ Real,
12
+ Iterable[Real],
13
+ Shaped[Tensor, "dim"],
14
+ Shaped[Tensor, "batch dim"],
15
+ ]
16
+
17
+
18
+ def sanitize_vector(
19
+ vector: Vector,
20
+ dim: int,
21
+ device: torch.device,
22
+ ) -> Float[Tensor, "*#batch dim"]:
23
+ if isinstance(vector, Tensor):
24
+ vector = vector.type(torch.float32).to(device)
25
+ else:
26
+ vector = torch.tensor(vector, dtype=torch.float32, device=device)
27
+ while vector.ndim < 2:
28
+ vector = vector[None]
29
+ if vector.shape[-1] == 1:
30
+ vector = repeat(vector, "... () -> ... c", c=dim)
31
+ assert vector.shape[-1] == dim
32
+ assert vector.ndim == 2
33
+ return vector
34
+
35
+
36
+ Scalar = Union[
37
+ Real,
38
+ Iterable[Real],
39
+ Shaped[Tensor, ""],
40
+ Shaped[Tensor, " batch"],
41
+ ]
42
+
43
+
44
+ def sanitize_scalar(scalar: Scalar, device: torch.device) -> Float[Tensor, "*#batch"]:
45
+ if isinstance(scalar, Tensor):
46
+ scalar = scalar.type(torch.float32).to(device)
47
+ else:
48
+ scalar = torch.tensor(scalar, dtype=torch.float32, device=device)
49
+ while scalar.ndim < 1:
50
+ scalar = scalar[None]
51
+ assert scalar.ndim == 1
52
+ return scalar
53
+
54
+
55
+ Pair = Union[
56
+ Iterable[Real],
57
+ Shaped[Tensor, "2"],
58
+ ]
59
+
60
+
61
+ def sanitize_pair(pair: Pair, device: torch.device) -> Float[Tensor, "2"]:
62
+ if isinstance(pair, Tensor):
63
+ pair = pair.type(torch.float32).to(device)
64
+ else:
65
+ pair = torch.tensor(pair, dtype=torch.float32, device=device)
66
+ assert pair.shape == (2,)
67
+ return pair
src/visualization/layout.py ADDED
@@ -0,0 +1,228 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """This file contains useful layout utilities for images. They are:
2
+
3
+ - add_border: Add a border to an image.
4
+ - cat/hcat/vcat: Join images by arranging them in a line. If the images have different
5
+ sizes, they are aligned as specified (start, end, center). Allows you to specify a gap
6
+ between images.
7
+
8
+ Images are assumed to be float32 tensors with shape (channel, height, width).
9
+ """
10
+
11
+ from typing import Any, Generator, Iterable, Literal, Optional, Union
12
+
13
+ import torch
14
+ import torch.nn.functional as F
15
+ from jaxtyping import Float
16
+ from torch import Tensor
17
+
18
+ Alignment = Literal["start", "center", "end"]
19
+ Axis = Literal["horizontal", "vertical"]
20
+ Color = Union[
21
+ int,
22
+ float,
23
+ Iterable[int],
24
+ Iterable[float],
25
+ Float[Tensor, "#channel"],
26
+ Float[Tensor, ""],
27
+ ]
28
+
29
+
30
+ def _sanitize_color(color: Color) -> Float[Tensor, "#channel"]:
31
+ # Convert tensor to list (or individual item).
32
+ if isinstance(color, torch.Tensor):
33
+ color = color.tolist()
34
+
35
+ # Turn iterators and individual items into lists.
36
+ if isinstance(color, Iterable):
37
+ color = list(color)
38
+ else:
39
+ color = [color]
40
+
41
+ return torch.tensor(color, dtype=torch.float32)
42
+
43
+
44
+ def _intersperse(iterable: Iterable, delimiter: Any) -> Generator[Any, None, None]:
45
+ it = iter(iterable)
46
+ yield next(it)
47
+ for item in it:
48
+ yield delimiter
49
+ yield item
50
+
51
+
52
+ def _get_main_dim(main_axis: Axis) -> int:
53
+ return {
54
+ "horizontal": 2,
55
+ "vertical": 1,
56
+ }[main_axis]
57
+
58
+
59
+ def _get_cross_dim(main_axis: Axis) -> int:
60
+ return {
61
+ "horizontal": 1,
62
+ "vertical": 2,
63
+ }[main_axis]
64
+
65
+
66
+ def _compute_offset(base: int, overlay: int, align: Alignment) -> slice:
67
+ assert base >= overlay
68
+ offset = {
69
+ "start": 0,
70
+ "center": (base - overlay) // 2,
71
+ "end": base - overlay,
72
+ }[align]
73
+ return slice(offset, offset + overlay)
74
+
75
+
76
+ def overlay(
77
+ base: Float[Tensor, "channel base_height base_width"],
78
+ overlay: Float[Tensor, "channel overlay_height overlay_width"],
79
+ main_axis: Axis,
80
+ main_axis_alignment: Alignment,
81
+ cross_axis_alignment: Alignment,
82
+ ) -> Float[Tensor, "channel base_height base_width"]:
83
+ # The overlay must be smaller than the base.
84
+ _, base_height, base_width = base.shape
85
+ _, overlay_height, overlay_width = overlay.shape
86
+ assert base_height >= overlay_height and base_width >= overlay_width
87
+
88
+ # Compute spacing on the main dimension.
89
+ main_dim = _get_main_dim(main_axis)
90
+ main_slice = _compute_offset(
91
+ base.shape[main_dim], overlay.shape[main_dim], main_axis_alignment
92
+ )
93
+
94
+ # Compute spacing on the cross dimension.
95
+ cross_dim = _get_cross_dim(main_axis)
96
+ cross_slice = _compute_offset(
97
+ base.shape[cross_dim], overlay.shape[cross_dim], cross_axis_alignment
98
+ )
99
+
100
+ # Combine the slices and paste the overlay onto the base accordingly.
101
+ selector = [..., None, None]
102
+ selector[main_dim] = main_slice
103
+ selector[cross_dim] = cross_slice
104
+ result = base.clone()
105
+ result[selector] = overlay
106
+ return result
107
+
108
+
109
+ def cat(
110
+ main_axis: Axis,
111
+ *images: Iterable[Float[Tensor, "channel _ _"]],
112
+ align: Alignment = "center",
113
+ gap: int = 8,
114
+ gap_color: Color = 1,
115
+ ) -> Float[Tensor, "channel height width"]:
116
+ """Arrange images in a line. The interface resembles a CSS div with flexbox."""
117
+ device = images[0].device
118
+ gap_color = _sanitize_color(gap_color).to(device)
119
+
120
+ # Find the maximum image side length in the cross axis dimension.
121
+ cross_dim = _get_cross_dim(main_axis)
122
+ cross_axis_length = max(image.shape[cross_dim] for image in images)
123
+
124
+ # Pad the images.
125
+ padded_images = []
126
+ for image in images:
127
+ # Create an empty image with the correct size.
128
+ padded_shape = list(image.shape)
129
+ padded_shape[cross_dim] = cross_axis_length
130
+ base = torch.ones(padded_shape, dtype=torch.float32, device=device)
131
+ base = base * gap_color[:, None, None]
132
+ padded_images.append(overlay(base, image, main_axis, "start", align))
133
+
134
+ # Intersperse separators if necessary.
135
+ if gap > 0:
136
+ # Generate a separator.
137
+ c, _, _ = images[0].shape
138
+ separator_size = [gap, gap]
139
+ separator_size[cross_dim - 1] = cross_axis_length
140
+ separator = torch.ones((c, *separator_size), dtype=torch.float32, device=device)
141
+ separator = separator * gap_color[:, None, None]
142
+
143
+ # Intersperse the separator between the images.
144
+ padded_images = list(_intersperse(padded_images, separator))
145
+
146
+ return torch.cat(padded_images, dim=_get_main_dim(main_axis))
147
+
148
+
149
+ def hcat(
150
+ *images: Iterable[Float[Tensor, "channel _ _"]],
151
+ align: Literal["start", "center", "end", "top", "bottom"] = "start",
152
+ gap: int = 8,
153
+ gap_color: Color = 1,
154
+ ):
155
+ """Shorthand for a horizontal linear concatenation."""
156
+ return cat(
157
+ "horizontal",
158
+ *images,
159
+ align={
160
+ "start": "start",
161
+ "center": "center",
162
+ "end": "end",
163
+ "top": "start",
164
+ "bottom": "end",
165
+ }[align],
166
+ gap=gap,
167
+ gap_color=gap_color,
168
+ )
169
+
170
+
171
+ def vcat(
172
+ *images: Iterable[Float[Tensor, "channel _ _"]],
173
+ align: Literal["start", "center", "end", "left", "right"] = "start",
174
+ gap: int = 8,
175
+ gap_color: Color = 1,
176
+ ):
177
+ """Shorthand for a horizontal linear concatenation."""
178
+ return cat(
179
+ "vertical",
180
+ *images,
181
+ align={
182
+ "start": "start",
183
+ "center": "center",
184
+ "end": "end",
185
+ "left": "start",
186
+ "right": "end",
187
+ }[align],
188
+ gap=gap,
189
+ gap_color=gap_color,
190
+ )
191
+
192
+
193
+ def add_border(
194
+ image: Float[Tensor, "channel height width"],
195
+ border: int = 8,
196
+ color: Color = 1,
197
+ ) -> Float[Tensor, "channel new_height new_width"]:
198
+ color = _sanitize_color(color).to(image)
199
+ c, h, w = image.shape
200
+ result = torch.empty(
201
+ (c, h + 2 * border, w + 2 * border), dtype=torch.float32, device=image.device
202
+ )
203
+ result[:] = color[:, None, None]
204
+ result[:, border : h + border, border : w + border] = image
205
+ return result
206
+
207
+
208
+ def resize(
209
+ image: Float[Tensor, "channel height width"],
210
+ shape: Optional[tuple[int, int]] = None,
211
+ width: Optional[int] = None,
212
+ height: Optional[int] = None,
213
+ ) -> Float[Tensor, "channel new_height new_width"]:
214
+ assert (shape is not None) + (width is not None) + (height is not None) == 1
215
+ _, h, w = image.shape
216
+
217
+ if width is not None:
218
+ shape = (int(h * width / w), width)
219
+ elif height is not None:
220
+ shape = (height, int(w * height / h))
221
+
222
+ return F.interpolate(
223
+ image[None],
224
+ shape,
225
+ mode="bilinear",
226
+ align_corners=False,
227
+ antialias="bilinear",
228
+ )[0]
src/visualization/validation_in_3d.py ADDED
@@ -0,0 +1,115 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from jaxtyping import Float, Shaped
3
+ from torch import Tensor
4
+
5
+ from ..model.decoder.cuda_splatting import render_cuda_orthographic
6
+ from ..model.types import Gaussians
7
+ from ..visualization.annotation import add_label
8
+ from ..visualization.drawing.cameras import draw_cameras
9
+ from .drawing.cameras import compute_equal_aabb_with_margin
10
+
11
+
12
+ def pad(images: list[Shaped[Tensor, "..."]]) -> list[Shaped[Tensor, "..."]]:
13
+ shapes = torch.stack([torch.tensor(x.shape) for x in images])
14
+ padded_shape = shapes.max(dim=0)[0]
15
+ results = [
16
+ torch.ones(padded_shape.tolist(), dtype=x.dtype, device=x.device)
17
+ for x in images
18
+ ]
19
+ for image, result in zip(images, results):
20
+ slices = [slice(0, x) for x in image.shape]
21
+ result[slices] = image[slices]
22
+ return results
23
+
24
+
25
+ def render_projections(
26
+ gaussians: Gaussians,
27
+ resolution: int,
28
+ margin: float = 0.1,
29
+ draw_label: bool = True,
30
+ extra_label: str = "",
31
+ ) -> Float[Tensor, "batch 3 3 height width"]:
32
+ device = gaussians.means.device
33
+ b, _, _ = gaussians.means.shape
34
+
35
+ # Compute the minima and maxima of the scene.
36
+ minima = gaussians.means.min(dim=1).values
37
+ maxima = gaussians.means.max(dim=1).values
38
+ scene_minima, scene_maxima = compute_equal_aabb_with_margin(
39
+ minima, maxima, margin=margin
40
+ )
41
+
42
+ projections = []
43
+ for look_axis in range(3):
44
+ right_axis = (look_axis + 1) % 3
45
+ down_axis = (look_axis + 2) % 3
46
+
47
+ # Define the extrinsics for rendering.
48
+ extrinsics = torch.zeros((b, 4, 4), dtype=torch.float32, device=device)
49
+ extrinsics[:, right_axis, 0] = 1
50
+ extrinsics[:, down_axis, 1] = 1
51
+ extrinsics[:, look_axis, 2] = 1
52
+ extrinsics[:, right_axis, 3] = 0.5 * (
53
+ scene_minima[:, right_axis] + scene_maxima[:, right_axis]
54
+ )
55
+ extrinsics[:, down_axis, 3] = 0.5 * (
56
+ scene_minima[:, down_axis] + scene_maxima[:, down_axis]
57
+ )
58
+ extrinsics[:, look_axis, 3] = scene_minima[:, look_axis]
59
+ extrinsics[:, 3, 3] = 1
60
+
61
+ # Define the intrinsics for rendering.
62
+ extents = scene_maxima - scene_minima
63
+ far = extents[:, look_axis]
64
+ near = torch.zeros_like(far)
65
+ width = extents[:, right_axis]
66
+ height = extents[:, down_axis]
67
+
68
+ projection = render_cuda_orthographic(
69
+ extrinsics,
70
+ width,
71
+ height,
72
+ near,
73
+ far,
74
+ (resolution, resolution),
75
+ torch.zeros((b, 3), dtype=torch.float32, device=device),
76
+ gaussians.means,
77
+ gaussians.covariances,
78
+ gaussians.harmonics,
79
+ gaussians.opacities,
80
+ fov_degrees=10.0,
81
+ )
82
+ if draw_label:
83
+ right_axis_name = "XYZ"[right_axis]
84
+ down_axis_name = "XYZ"[down_axis]
85
+ label = f"{right_axis_name}{down_axis_name} Projection {extra_label}"
86
+ projection = torch.stack([add_label(x, label) for x in projection])
87
+
88
+ projections.append(projection)
89
+
90
+ return torch.stack(pad(projections), dim=1)
91
+
92
+
93
+ def render_cameras(batch: dict, resolution: int) -> Float[Tensor, "3 3 height width"]:
94
+ # Define colors for context and target views.
95
+ num_context_views = batch["context"]["extrinsics"].shape[1]
96
+ num_target_views = batch["target"]["extrinsics"].shape[1]
97
+ color = torch.ones(
98
+ (num_target_views + num_context_views, 3),
99
+ dtype=torch.float32,
100
+ device=batch["target"]["extrinsics"].device,
101
+ )
102
+ color[num_context_views:, 1:] = 0
103
+
104
+ return draw_cameras(
105
+ resolution,
106
+ torch.cat(
107
+ (batch["context"]["extrinsics"][0], batch["target"]["extrinsics"][0])
108
+ ),
109
+ torch.cat(
110
+ (batch["context"]["intrinsics"][0], batch["target"]["intrinsics"][0])
111
+ ),
112
+ color,
113
+ torch.cat((batch["context"]["near"][0], batch["target"]["near"][0])),
114
+ torch.cat((batch["context"]["far"][0], batch["target"]["far"][0])),
115
+ )
src/visualization/video_render.py ADDED
@@ -0,0 +1,162 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Protocol, runtime_checkable
2
+
3
+ import cv2
4
+ import torch
5
+ from einops import repeat, pack
6
+ from jaxtyping import Float
7
+ from torch import Tensor
8
+
9
+ from .camera_trajectory.interpolation import interpolate_extrinsics, interpolate_intrinsics
10
+ from .camera_trajectory.wobble import generate_wobble, generate_wobble_transformation
11
+ from .layout import vcat
12
+ from ..dataset.types import BatchedExample
13
+ from ..misc.image_io import save_video
14
+ from ..misc.utils import vis_depth_map
15
+ from ..model.decoder import Decoder
16
+ from ..model.types import Gaussians
17
+
18
+
19
+ @runtime_checkable
20
+ class TrajectoryFn(Protocol):
21
+ def __call__(
22
+ self,
23
+ t: Float[Tensor, " t"],
24
+ ) -> tuple[
25
+ Float[Tensor, "batch view 4 4"], # extrinsics
26
+ Float[Tensor, "batch view 3 3"], # intrinsics
27
+ ]:
28
+ pass
29
+
30
+
31
+ def render_video_wobble(
32
+ gaussians: Gaussians,
33
+ decoder: Decoder,
34
+ batch: BatchedExample,
35
+ num_frames: int = 60,
36
+ smooth: bool = True,
37
+ loop_reverse: bool = True,
38
+ add_depth: bool = False,
39
+ ) -> Tensor:
40
+ # Two views are needed to get the wobble radius,use the first and the last view
41
+ _, v, _, _ = batch["context"]["extrinsics"].shape
42
+
43
+ def trajectory_fn(t):
44
+ origin_a = batch["context"]["extrinsics"][:, 0, :3, 3]
45
+ origin_b = batch["context"]["extrinsics"][:, -1, :3, 3]
46
+ delta = (origin_a - origin_b).norm(dim=-1)
47
+ extrinsics = generate_wobble(
48
+ batch["context"]["extrinsics"][:, 0],
49
+ delta * 0.25,
50
+ t,
51
+ )
52
+ intrinsics = repeat(
53
+ batch["context"]["intrinsics"][:, 0],
54
+ "b i j -> b v i j",
55
+ v=t.shape[0],
56
+ )
57
+ return extrinsics, intrinsics
58
+
59
+ return render_video_generic(gaussians, decoder, batch, trajectory_fn, num_frames, smooth, loop_reverse, add_depth)
60
+
61
+
62
+ def render_video_interpolation(
63
+ gaussians: Gaussians,
64
+ decoder: Decoder,
65
+ batch: BatchedExample,
66
+ num_frames: int = 60,
67
+ smooth: bool = True,
68
+ loop_reverse: bool = True,
69
+ add_depth: bool = False,
70
+ ) -> Tensor:
71
+ _, v, _, _ = batch["context"]["extrinsics"].shape
72
+
73
+ def trajectory_fn(t):
74
+ extrinsics = interpolate_extrinsics(
75
+ batch["context"]["extrinsics"][0, 0],
76
+ batch["context"]["extrinsics"][0, -1],
77
+ t,
78
+ )
79
+ intrinsics = interpolate_intrinsics(
80
+ batch["context"]["intrinsics"][0, 0],
81
+ batch["context"]["intrinsics"][0, -1],
82
+ t,
83
+ )
84
+ return extrinsics[None], intrinsics[None]
85
+
86
+ return render_video_generic(gaussians, decoder, batch, trajectory_fn, num_frames, smooth, loop_reverse, add_depth)
87
+
88
+
89
+ def render_video_interpolation_exaggerated(
90
+ gaussians: Gaussians,
91
+ decoder: Decoder,
92
+ batch: BatchedExample,
93
+ num_frames: int = 300,
94
+ smooth: bool = False,
95
+ loop_reverse: bool = False,
96
+ add_depth: bool = False,
97
+ ) -> Tensor:
98
+ # Two views are needed to get the wobble radius.
99
+ _, v, _, _ = batch["context"]["extrinsics"].shape
100
+
101
+ def trajectory_fn(t):
102
+ origin_a = batch["context"]["extrinsics"][:, 0, :3, 3]
103
+ origin_b = batch["context"]["extrinsics"][:, -1, :3, 3]
104
+ delta = (origin_a - origin_b).norm(dim=-1)
105
+ tf = generate_wobble_transformation(
106
+ delta * 0.5,
107
+ t,
108
+ 5,
109
+ scale_radius_with_t=False,
110
+ )
111
+ extrinsics = interpolate_extrinsics(
112
+ batch["context"]["extrinsics"][0, 0],
113
+ batch["context"]["extrinsics"][0, -1],
114
+ t * 5 - 2,
115
+ )
116
+ intrinsics = interpolate_intrinsics(
117
+ batch["context"]["intrinsics"][0, 0],
118
+ batch["context"]["extrinsics"][0, -1],
119
+ t * 5 - 2,
120
+ )
121
+ return extrinsics @ tf, intrinsics[None]
122
+
123
+ return render_video_generic(gaussians, decoder, batch, trajectory_fn, num_frames, smooth, loop_reverse, add_depth)
124
+
125
+
126
+ def render_video_generic(
127
+ gaussians: Gaussians,
128
+ decoder: Decoder,
129
+ batch: BatchedExample,
130
+ trajectory_fn: TrajectoryFn,
131
+ num_frames: int = 30,
132
+ smooth: bool = True,
133
+ loop_reverse: bool = True,
134
+ add_depth: bool = False,
135
+ ) -> Tensor:
136
+ device = gaussians.means.device
137
+
138
+ t = torch.linspace(0, 1, num_frames, dtype=torch.float32, device=device)
139
+ if smooth:
140
+ t = (torch.cos(torch.pi * (t + 1)) + 1) / 2
141
+
142
+ extrinsics, intrinsics = trajectory_fn(t)
143
+
144
+ _, _, _, h, w = batch["context"]["image"].shape
145
+
146
+ near = repeat(batch["context"]["near"][:, 0], "b -> b v", v=num_frames)
147
+ far = repeat(batch["context"]["far"][:, 0], "b -> b v", v=num_frames)
148
+ output = decoder.forward(
149
+ gaussians, extrinsics, intrinsics, near, far, (h, w), "depth"
150
+ )
151
+ images = [
152
+ vcat(rgb, depth) if add_depth else rgb
153
+ for rgb, depth in zip(output.color[0], vis_depth_map(output.depth[0]))
154
+ ]
155
+
156
+ video = torch.stack(images)
157
+ # video = (video.clip(min=0, max=1) * 255).type(torch.uint8).cpu().numpy()
158
+ if loop_reverse:
159
+ # video = pack([video, video[::-1][1:-1]], "* c h w")[0]
160
+ video = pack([video, video.flip(dims=(0,))[1:-1]], "* c h w")[0]
161
+
162
+ return video