Spaces:
Running on Zero
Running on Zero
Initial commit for brand new AnySplat ZeroGPU Space (part 2)
Browse files- src/dataset/dataset_scannetpp.py +443 -0
- src/dataset/shims/augmentation_shim.py +219 -0
- src/dataset/shims/bounds_shim.py +80 -0
- src/dataset/shims/crop_shim.py +196 -0
- src/dataset/shims/geometry_shim.py +383 -0
- src/dataset/shims/load_shim.py +12 -0
- src/dataset/shims/normalize_shim.py +27 -0
- src/dataset/shims/patch_shim.py +38 -0
- src/dataset/types.py +51 -0
- src/dataset/validation_wrapper.py +34 -0
- src/dataset/view_sampler/__init__.py +40 -0
- src/dataset/view_sampler/three_view_hack.py +10 -0
- src/dataset/view_sampler/view_sampler.py +61 -0
- src/dataset/view_sampler/view_sampler_all.py +37 -0
- src/dataset/view_sampler/view_sampler_arbitrary.py +77 -0
- src/dataset/view_sampler/view_sampler_bounded.py +151 -0
- src/dataset/view_sampler/view_sampler_evaluation.py +70 -0
- src/dataset/view_sampler/view_sampler_rank.py +263 -0
- src/eval_nvs.py +115 -0
- src/eval_pose.py +339 -0
- src/evaluation/evaluation_cfg.py +24 -0
- src/evaluation/evaluation_index_generator.py +160 -0
- src/evaluation/metrics.py +134 -0
- src/geometry/camera_emb.py +29 -0
- src/geometry/epipolar_lines.py +292 -0
src/dataset/dataset_scannetpp.py
ADDED
|
@@ -0,0 +1,443 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from dataclasses import dataclass
|
| 3 |
+
from functools import cached_property
|
| 4 |
+
from io import BytesIO
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
import random
|
| 7 |
+
from typing import Literal
|
| 8 |
+
import os
|
| 9 |
+
import cv2
|
| 10 |
+
import numpy as np
|
| 11 |
+
import torch
|
| 12 |
+
import torchvision.transforms as tf
|
| 13 |
+
from einops import rearrange, repeat
|
| 14 |
+
from jaxtyping import Float, UInt8
|
| 15 |
+
from PIL import Image
|
| 16 |
+
import torchvision
|
| 17 |
+
from torch import Tensor
|
| 18 |
+
from torch.utils.data import Dataset
|
| 19 |
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
| 20 |
+
import copy
|
| 21 |
+
from .shims.geometry_shim import depthmap_to_absolute_camera_coordinates
|
| 22 |
+
|
| 23 |
+
from .shims.load_shim import imread_cv2
|
| 24 |
+
|
| 25 |
+
from ..geometry.projection import get_fov
|
| 26 |
+
from .dataset import DatasetCfgCommon
|
| 27 |
+
from .shims.augmentation_shim import apply_augmentation_shim
|
| 28 |
+
from .shims.crop_shim import apply_crop_shim
|
| 29 |
+
from .types import Stage
|
| 30 |
+
from .view_sampler import ViewSampler
|
| 31 |
+
from ..misc.cam_utils import camera_normalization
|
| 32 |
+
|
| 33 |
+
@dataclass
|
| 34 |
+
class DatasetScannetppCfg(DatasetCfgCommon):
|
| 35 |
+
name: str
|
| 36 |
+
roots: list[Path]
|
| 37 |
+
baseline_min: float
|
| 38 |
+
baseline_max: float
|
| 39 |
+
max_fov: float
|
| 40 |
+
make_baseline_1: bool
|
| 41 |
+
augment: bool
|
| 42 |
+
relative_pose: bool
|
| 43 |
+
skip_bad_shape: bool
|
| 44 |
+
metric_thre: float
|
| 45 |
+
intr_augment: bool
|
| 46 |
+
make_baseline_1: bool
|
| 47 |
+
rescale_to_1cube: bool
|
| 48 |
+
normalize_by_pts3d: bool
|
| 49 |
+
|
| 50 |
+
@dataclass
|
| 51 |
+
class DatasetScannetppCfgWrapper:
|
| 52 |
+
scannetpp: DatasetScannetppCfg
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class DatasetScannetpp(Dataset):
|
| 56 |
+
cfg: DatasetScannetppCfgWrapper
|
| 57 |
+
stage: Stage
|
| 58 |
+
view_sampler: ViewSampler
|
| 59 |
+
|
| 60 |
+
to_tensor: tf.ToTensor
|
| 61 |
+
chunks: list[Path]
|
| 62 |
+
near: float = 0.1
|
| 63 |
+
far: float = 100.0
|
| 64 |
+
|
| 65 |
+
def __init__(
|
| 66 |
+
self,
|
| 67 |
+
cfg: DatasetScannetppCfgWrapper,
|
| 68 |
+
stage: Stage,
|
| 69 |
+
view_sampler: ViewSampler,
|
| 70 |
+
) -> None:
|
| 71 |
+
super().__init__()
|
| 72 |
+
self.cfg = cfg
|
| 73 |
+
self.stage = stage
|
| 74 |
+
self.view_sampler = view_sampler
|
| 75 |
+
self.to_tensor = tf.ToTensor()
|
| 76 |
+
|
| 77 |
+
# load data
|
| 78 |
+
self.data_root = cfg.roots[0]
|
| 79 |
+
self.data_list = [] # we use dslr rather than iphone
|
| 80 |
+
data_index = os.listdir(f"{self.data_root}") # we train all the scenes
|
| 81 |
+
|
| 82 |
+
if self.stage != "train":
|
| 83 |
+
with open(f"{self.data_root}/valid.json", "r") as file:
|
| 84 |
+
data_index = json.load(file)[:10]
|
| 85 |
+
data_index = data_index * 100
|
| 86 |
+
random.shuffle(data_index)
|
| 87 |
+
else:
|
| 88 |
+
with open(f"{self.data_root}/valid.json", "r") as file:
|
| 89 |
+
data_index = json.load(file)[10:]
|
| 90 |
+
|
| 91 |
+
self.data_list = [
|
| 92 |
+
os.path.join(self.data_root, item) for item in data_index
|
| 93 |
+
]
|
| 94 |
+
|
| 95 |
+
self.scene_ids = {}
|
| 96 |
+
self.scenes = {}
|
| 97 |
+
index = 0
|
| 98 |
+
with ThreadPoolExecutor(max_workers=32) as executor:
|
| 99 |
+
futures = [executor.submit(self.load_metadata, scene_path) for scene_path in self.data_list]
|
| 100 |
+
for future in as_completed(futures):
|
| 101 |
+
scene_frames, scene_id = future.result()
|
| 102 |
+
self.scenes[scene_id] = scene_frames
|
| 103 |
+
self.scene_ids[index] = scene_id
|
| 104 |
+
index += 1
|
| 105 |
+
|
| 106 |
+
# if self.stage != "train":
|
| 107 |
+
# self.scene_ids = self.scene_ids
|
| 108 |
+
# random.shuffle(self.scene_ids)
|
| 109 |
+
print(f"Scannetpp: {self.stage}: loaded {len(self.scene_ids)} scenes")
|
| 110 |
+
|
| 111 |
+
def shuffle(self, lst: list) -> list:
|
| 112 |
+
indices = torch.randperm(len(lst))
|
| 113 |
+
return [lst[x] for x in indices]
|
| 114 |
+
|
| 115 |
+
def load_metadata(self, scene_path):
|
| 116 |
+
metadata_path = os.path.join(scene_path, "scene_metadata.npz")
|
| 117 |
+
metadata = np.load(metadata_path, allow_pickle=True)
|
| 118 |
+
intrinsics = metadata["intrinsics"]
|
| 119 |
+
trajectories = metadata["trajectories"]
|
| 120 |
+
images = metadata["images"]
|
| 121 |
+
|
| 122 |
+
scene_id = scene_path.split("/")[-1].split(".")[0]
|
| 123 |
+
scene_frames = [
|
| 124 |
+
{
|
| 125 |
+
"file_path": os.path.join(scene_path, "images", images[i].split(".")[0] + ".jpg"),
|
| 126 |
+
"depth_path": os.path.join(scene_path, "depth", images[i].split(".")[0] + ".png"),
|
| 127 |
+
"intrinsics": self.convert_intrinsics(intrinsics[i]),
|
| 128 |
+
"extrinsics": trajectories[i],
|
| 129 |
+
}
|
| 130 |
+
for i in range(len(images))
|
| 131 |
+
]
|
| 132 |
+
scene_frames.sort(key=lambda x: x["file_path"]) # sort by file path to ensure correct order
|
| 133 |
+
return scene_frames, scene_id
|
| 134 |
+
|
| 135 |
+
def convert_intrinsics(self, intrinsics):
|
| 136 |
+
w = intrinsics[0, 2] * 2
|
| 137 |
+
h = intrinsics[1, 2] * 2
|
| 138 |
+
intrinsics[0, 0] = intrinsics[0, 0] / w
|
| 139 |
+
intrinsics[1, 1] = intrinsics[1, 1] / h
|
| 140 |
+
intrinsics[0, 2] = intrinsics[0, 2] / w
|
| 141 |
+
intrinsics[1, 2] = intrinsics[1, 2] / h
|
| 142 |
+
return intrinsics
|
| 143 |
+
|
| 144 |
+
def blender2opencv_c2w(self, pose):
|
| 145 |
+
blender2opencv = np.array(
|
| 146 |
+
[[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]]
|
| 147 |
+
)
|
| 148 |
+
opencv_c2w = np.array(pose) @ blender2opencv
|
| 149 |
+
return opencv_c2w.tolist()
|
| 150 |
+
|
| 151 |
+
def load_frames(self, frames):
|
| 152 |
+
with ThreadPoolExecutor(max_workers=1) as executor:
|
| 153 |
+
# Create a list to store futures with their original indices
|
| 154 |
+
futures_with_idx = []
|
| 155 |
+
for idx, file_path in enumerate(frames):
|
| 156 |
+
file_path = file_path["file_path"]
|
| 157 |
+
futures_with_idx.append(
|
| 158 |
+
(
|
| 159 |
+
idx,
|
| 160 |
+
executor.submit(
|
| 161 |
+
lambda p: self.to_tensor(Image.open(p).convert("RGB")),
|
| 162 |
+
file_path,
|
| 163 |
+
),
|
| 164 |
+
)
|
| 165 |
+
)
|
| 166 |
+
|
| 167 |
+
# Pre-allocate list with correct size to maintain order
|
| 168 |
+
torch_images = [None] * len(frames)
|
| 169 |
+
for idx, future in futures_with_idx:
|
| 170 |
+
torch_images[idx] = future.result()
|
| 171 |
+
# Check if all images have the same size
|
| 172 |
+
sizes = set(img.shape for img in torch_images)
|
| 173 |
+
if len(sizes) == 1:
|
| 174 |
+
torch_images = torch.stack(torch_images)
|
| 175 |
+
# Return as list if images have different sizes
|
| 176 |
+
return torch_images
|
| 177 |
+
|
| 178 |
+
def load_depths(self, frames):
|
| 179 |
+
torch_depths = []
|
| 180 |
+
for idx, frame in enumerate(frames):
|
| 181 |
+
depthmap = imread_cv2(frame["depth_path"], cv2.IMREAD_UNCHANGED)
|
| 182 |
+
depthmap = depthmap.astype(np.float32) / 1000
|
| 183 |
+
depthmap[~np.isfinite(depthmap)] = 0
|
| 184 |
+
torch_depths.append(torch.from_numpy(depthmap))
|
| 185 |
+
return torch.stack(torch_depths) # [N, H, W]
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def getitem(self, index: int, num_context_views: int, patchsize: tuple) -> dict:
|
| 189 |
+
# import time
|
| 190 |
+
# start_time = time.time()
|
| 191 |
+
|
| 192 |
+
scene = self.scene_ids[index]
|
| 193 |
+
example = self.scenes[scene]
|
| 194 |
+
# load poses
|
| 195 |
+
extrinsics = []
|
| 196 |
+
intrinsics = []
|
| 197 |
+
for frame in example:
|
| 198 |
+
extrinsic = frame["extrinsics"]
|
| 199 |
+
intrinsic = frame["intrinsics"]
|
| 200 |
+
extrinsics.append(extrinsic)
|
| 201 |
+
intrinsics.append(intrinsic)
|
| 202 |
+
|
| 203 |
+
extrinsics = np.array(extrinsics)
|
| 204 |
+
intrinsics = np.array(intrinsics)
|
| 205 |
+
extrinsics = torch.tensor(extrinsics, dtype=torch.float32)
|
| 206 |
+
intrinsics = torch.tensor(intrinsics, dtype=torch.float32)
|
| 207 |
+
|
| 208 |
+
try:
|
| 209 |
+
context_indices, target_indices, overlap = self.view_sampler.sample(
|
| 210 |
+
"scannetpp_"+scene,
|
| 211 |
+
num_context_views,
|
| 212 |
+
extrinsics,
|
| 213 |
+
intrinsics,
|
| 214 |
+
)
|
| 215 |
+
except ValueError:
|
| 216 |
+
# Skip because the example doesn't have enough frames.
|
| 217 |
+
raise Exception("Not enough frames")
|
| 218 |
+
|
| 219 |
+
# Skip the example if the field of view is too wide.
|
| 220 |
+
if (get_fov(intrinsics).rad2deg() > self.cfg.max_fov).any():
|
| 221 |
+
raise Exception("Field of view too wide")
|
| 222 |
+
|
| 223 |
+
# Load the images.
|
| 224 |
+
input_frames = [example[i] for i in context_indices]
|
| 225 |
+
target_frame = [example[i] for i in target_indices]
|
| 226 |
+
|
| 227 |
+
context_images = self.load_frames(input_frames)
|
| 228 |
+
target_images = self.load_frames(target_frame)
|
| 229 |
+
|
| 230 |
+
context_depths = self.load_depths(input_frames)
|
| 231 |
+
target_depths = self.load_depths(target_frame)
|
| 232 |
+
|
| 233 |
+
# Skip the example if the images don't have the right shape.
|
| 234 |
+
context_image_invalid = context_images.shape[1:] != (3, *self.cfg.original_image_shape)
|
| 235 |
+
target_image_invalid = target_images.shape[1:] != (3, *self.cfg.original_image_shape)
|
| 236 |
+
if self.cfg.skip_bad_shape and (context_image_invalid or target_image_invalid):
|
| 237 |
+
print(
|
| 238 |
+
f"Skipped bad example {example['key']}. Context shape was "
|
| 239 |
+
f"{context_images.shape} and target shape was "
|
| 240 |
+
f"{target_images.shape}."
|
| 241 |
+
)
|
| 242 |
+
raise Exception("Bad example image shape")
|
| 243 |
+
|
| 244 |
+
context_extrinsics = extrinsics[context_indices]
|
| 245 |
+
|
| 246 |
+
if self.cfg.make_baseline_1:
|
| 247 |
+
a, b = context_extrinsics[0, :3, 3], context_extrinsics[-1, :3, 3]
|
| 248 |
+
scale = (a - b).norm()
|
| 249 |
+
if scale < self.cfg.baseline_min or scale > self.cfg.baseline_max:
|
| 250 |
+
print(
|
| 251 |
+
f"Skipped {scene} because of baseline out of range: "
|
| 252 |
+
f"{scale:.6f}"
|
| 253 |
+
)
|
| 254 |
+
raise Exception("baseline out of range")
|
| 255 |
+
extrinsics[:, :3, 3] /= scale
|
| 256 |
+
else:
|
| 257 |
+
scale = 1
|
| 258 |
+
|
| 259 |
+
if self.cfg.relative_pose:
|
| 260 |
+
extrinsics = camera_normalization(extrinsics[context_indices][0:1], extrinsics)
|
| 261 |
+
|
| 262 |
+
if self.cfg.rescale_to_1cube:
|
| 263 |
+
scene_scale = torch.max(torch.abs(extrinsics[context_indices][:, :3, 3])) # target pose is not included
|
| 264 |
+
rescale_factor = 1 * scene_scale
|
| 265 |
+
extrinsics[:, :3, 3] /= rescale_factor
|
| 266 |
+
|
| 267 |
+
if torch.isnan(extrinsics).any() or torch.isinf(extrinsics).any():
|
| 268 |
+
raise Exception("encounter nan or inf in input poses")
|
| 269 |
+
|
| 270 |
+
example = {
|
| 271 |
+
"context": {
|
| 272 |
+
"extrinsics": extrinsics[context_indices],
|
| 273 |
+
"intrinsics": intrinsics[context_indices],
|
| 274 |
+
"image": context_images,
|
| 275 |
+
"depth": context_depths,
|
| 276 |
+
"near": self.get_bound("near", len(context_indices)) / scale,
|
| 277 |
+
"far": self.get_bound("far", len(context_indices)) / scale,
|
| 278 |
+
"index": context_indices,
|
| 279 |
+
"overlap": overlap,
|
| 280 |
+
},
|
| 281 |
+
"target": {
|
| 282 |
+
"extrinsics": extrinsics[target_indices],
|
| 283 |
+
"intrinsics": intrinsics[target_indices],
|
| 284 |
+
"image": target_images,
|
| 285 |
+
"depth": target_depths,
|
| 286 |
+
"near": self.get_bound("near", len(target_indices)) / scale,
|
| 287 |
+
"far": self.get_bound("far", len(target_indices)) / scale,
|
| 288 |
+
"index": target_indices,
|
| 289 |
+
},
|
| 290 |
+
"scene": f"Scannetpp {scene}",
|
| 291 |
+
}
|
| 292 |
+
if self.stage == "train" and self.cfg.augment:
|
| 293 |
+
example = apply_augmentation_shim(example)
|
| 294 |
+
|
| 295 |
+
if self.stage == "train" and self.cfg.intr_augment:
|
| 296 |
+
intr_aug = True
|
| 297 |
+
else:
|
| 298 |
+
intr_aug = False
|
| 299 |
+
|
| 300 |
+
example = apply_crop_shim(example, (patchsize[0] * 14, patchsize[1] * 14), intr_aug=intr_aug)
|
| 301 |
+
|
| 302 |
+
# world pts
|
| 303 |
+
image_size = example["context"]["image"].shape[2:]
|
| 304 |
+
context_intrinsics = example["context"]["intrinsics"].clone().detach().numpy()
|
| 305 |
+
context_intrinsics[:, 0] = context_intrinsics[:, 0] * image_size[1]
|
| 306 |
+
context_intrinsics[:, 1] = context_intrinsics[:, 1] * image_size[0]
|
| 307 |
+
|
| 308 |
+
target_intrinsics = example["target"]["intrinsics"].clone().detach().numpy()
|
| 309 |
+
target_intrinsics[:, 0] = target_intrinsics[:, 0] * image_size[1]
|
| 310 |
+
target_intrinsics[:, 1] = target_intrinsics[:, 1] * image_size[0]
|
| 311 |
+
|
| 312 |
+
context_pts3d_list, context_valid_mask_list = [], []
|
| 313 |
+
target_pts3d_list, target_valid_mask_list = [], []
|
| 314 |
+
|
| 315 |
+
for i in range(len(example["context"]["depth"])):
|
| 316 |
+
context_pts3d, context_valid_mask = depthmap_to_absolute_camera_coordinates(example["context"]["depth"][i].numpy(), context_intrinsics[i], example["context"]["extrinsics"][i].numpy())
|
| 317 |
+
context_pts3d_list.append(torch.from_numpy(context_pts3d).to(torch.float32))
|
| 318 |
+
context_valid_mask_list.append(torch.from_numpy(context_valid_mask))
|
| 319 |
+
|
| 320 |
+
context_pts3d = torch.stack(context_pts3d_list, dim=0)
|
| 321 |
+
context_valid_mask = torch.stack(context_valid_mask_list, dim=0)
|
| 322 |
+
|
| 323 |
+
for i in range(len(example["target"]["depth"])):
|
| 324 |
+
target_pts3d, target_valid_mask = depthmap_to_absolute_camera_coordinates(example["target"]["depth"][i].numpy(), target_intrinsics[i], example["target"]["extrinsics"][i].numpy())
|
| 325 |
+
target_pts3d_list.append(torch.from_numpy(target_pts3d).to(torch.float32))
|
| 326 |
+
target_valid_mask_list.append(torch.from_numpy(target_valid_mask))
|
| 327 |
+
|
| 328 |
+
target_pts3d = torch.stack(target_pts3d_list, dim=0)
|
| 329 |
+
target_valid_mask = torch.stack(target_valid_mask_list, dim=0)
|
| 330 |
+
|
| 331 |
+
# normalize by context pts3d
|
| 332 |
+
if self.cfg.normalize_by_pts3d:
|
| 333 |
+
transformed_pts3d = context_pts3d[context_valid_mask]
|
| 334 |
+
scene_factor = transformed_pts3d.norm(dim=-1).mean().clip(min=1e-8)
|
| 335 |
+
context_pts3d /= scene_factor
|
| 336 |
+
example["context"]["depth"] /= scene_factor
|
| 337 |
+
example["context"]["extrinsics"][:, :3, 3] /= scene_factor
|
| 338 |
+
|
| 339 |
+
target_pts3d /= scene_factor
|
| 340 |
+
example["target"]["depth"] /= scene_factor
|
| 341 |
+
example["target"]["extrinsics"][:, :3, 3] /= scene_factor
|
| 342 |
+
|
| 343 |
+
example["context"]["pts3d"] = context_pts3d
|
| 344 |
+
example["target"]["pts3d"] = target_pts3d
|
| 345 |
+
example["context"]["valid_mask"] = context_valid_mask
|
| 346 |
+
example["target"]["valid_mask"] = target_valid_mask
|
| 347 |
+
|
| 348 |
+
if torch.isnan(example["context"]["depth"]).any() or torch.isinf(example["context"]["depth"]).any() or \
|
| 349 |
+
torch.isnan(example["context"]["extrinsics"]).any() or torch.isinf(example["context"]["extrinsics"]).any() or \
|
| 350 |
+
torch.isnan(example["context"]["intrinsics"]).any() or torch.isinf(example["context"]["intrinsics"]).any() or \
|
| 351 |
+
torch.isnan(example["target"]["depth"]).any() or torch.isinf(example["target"]["depth"]).any() or \
|
| 352 |
+
torch.isnan(example["target"]["extrinsics"]).any() or torch.isinf(example["target"]["extrinsics"]).any() or \
|
| 353 |
+
torch.isnan(example["target"]["intrinsics"]).any() or torch.isinf(example["target"]["intrinsics"]).any():
|
| 354 |
+
raise Exception("encounter nan or inf in context depth")
|
| 355 |
+
|
| 356 |
+
for key in ["context", "target"]:
|
| 357 |
+
example[key]["valid_mask"] = (torch.ones_like(example[key]["valid_mask"]) * -1).type(torch.int32)
|
| 358 |
+
|
| 359 |
+
return example
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
def __getitem__(self, index_tuple: tuple) -> dict:
|
| 363 |
+
index, num_context_views, patchsize_h = index_tuple
|
| 364 |
+
# generate a random patch size
|
| 365 |
+
patchsize_w = (self.cfg.input_image_shape[1] // 14)
|
| 366 |
+
try:
|
| 367 |
+
return self.getitem(index, num_context_views, (patchsize_h, patchsize_w))
|
| 368 |
+
except Exception as e:
|
| 369 |
+
print(f"Error: {e}")
|
| 370 |
+
index = np.random.randint(len(self))
|
| 371 |
+
return self.__getitem__((index, num_context_views, patchsize_h))
|
| 372 |
+
|
| 373 |
+
def convert_poses(
|
| 374 |
+
self,
|
| 375 |
+
poses: Float[Tensor, "batch 18"],
|
| 376 |
+
) -> tuple[
|
| 377 |
+
Float[Tensor, "batch 4 4"], # extrinsics
|
| 378 |
+
Float[Tensor, "batch 3 3"], # intrinsics
|
| 379 |
+
]:
|
| 380 |
+
b, _ = poses.shape
|
| 381 |
+
|
| 382 |
+
# Convert the intrinsics to a 3x3 normalized K matrix.
|
| 383 |
+
intrinsics = torch.eye(3, dtype=torch.float32)
|
| 384 |
+
intrinsics = repeat(intrinsics, "h w -> b h w", b=b).clone()
|
| 385 |
+
fx, fy, cx, cy = poses[:, :4].T
|
| 386 |
+
intrinsics[:, 0, 0] = fx
|
| 387 |
+
intrinsics[:, 1, 1] = fy
|
| 388 |
+
intrinsics[:, 0, 2] = cx
|
| 389 |
+
intrinsics[:, 1, 2] = cy
|
| 390 |
+
|
| 391 |
+
# Convert the extrinsics to a 4x4 OpenCV-style W2C matrix.
|
| 392 |
+
w2c = repeat(torch.eye(4, dtype=torch.float32), "h w -> b h w", b=b).clone()
|
| 393 |
+
w2c[:, :3] = rearrange(poses[:, 6:], "b (h w) -> b h w", h=3, w=4)
|
| 394 |
+
return w2c.inverse(), intrinsics
|
| 395 |
+
|
| 396 |
+
def convert_images(
|
| 397 |
+
self,
|
| 398 |
+
images: list[UInt8[Tensor, "..."]],
|
| 399 |
+
) -> Float[Tensor, "batch 3 height width"]:
|
| 400 |
+
torch_images = []
|
| 401 |
+
for image in images:
|
| 402 |
+
image = Image.open(BytesIO(image.numpy().tobytes()))
|
| 403 |
+
torch_images.append(self.to_tensor(image))
|
| 404 |
+
return torch.stack(torch_images)
|
| 405 |
+
|
| 406 |
+
def get_bound(
|
| 407 |
+
self,
|
| 408 |
+
bound: Literal["near", "far"],
|
| 409 |
+
num_views: int,
|
| 410 |
+
) -> Float[Tensor, " view"]:
|
| 411 |
+
value = torch.tensor(getattr(self, bound), dtype=torch.float32)
|
| 412 |
+
return repeat(value, "-> v", v=num_views)
|
| 413 |
+
|
| 414 |
+
@property
|
| 415 |
+
def data_stage(self) -> Stage:
|
| 416 |
+
if self.cfg.overfit_to_scene is not None:
|
| 417 |
+
return "test"
|
| 418 |
+
if self.stage == "val":
|
| 419 |
+
return "test"
|
| 420 |
+
return self.stage
|
| 421 |
+
|
| 422 |
+
@cached_property
|
| 423 |
+
def index(self) -> dict[str, Path]:
|
| 424 |
+
merged_index = {}
|
| 425 |
+
data_stages = [self.data_stage]
|
| 426 |
+
if self.cfg.overfit_to_scene is not None:
|
| 427 |
+
data_stages = ("test", "train")
|
| 428 |
+
for data_stage in data_stages:
|
| 429 |
+
for root in self.cfg.roots:
|
| 430 |
+
# Load the root's index.
|
| 431 |
+
with (root / data_stage / "index.json").open("r") as f:
|
| 432 |
+
index = json.load(f)
|
| 433 |
+
index = {k: Path(root / data_stage / v) for k, v in index.items()}
|
| 434 |
+
|
| 435 |
+
# The constituent datasets should have unique keys.
|
| 436 |
+
assert not (set(merged_index.keys()) & set(index.keys()))
|
| 437 |
+
|
| 438 |
+
# Merge the root's index into the main index.
|
| 439 |
+
merged_index = {**merged_index, **index}
|
| 440 |
+
return merged_index
|
| 441 |
+
|
| 442 |
+
def __len__(self) -> int:
|
| 443 |
+
return len(self.data_list)
|
src/dataset/shims/augmentation_shim.py
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import copy
|
| 2 |
+
import random
|
| 3 |
+
import numpy as np
|
| 4 |
+
import torch
|
| 5 |
+
from jaxtyping import Float
|
| 6 |
+
from torch import Tensor
|
| 7 |
+
|
| 8 |
+
from ..types import AnyExample, AnyViews
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def reflect_extrinsics(
|
| 12 |
+
extrinsics: Float[Tensor, "*batch 4 4"],
|
| 13 |
+
) -> Float[Tensor, "*batch 4 4"]:
|
| 14 |
+
reflect = torch.eye(4, dtype=torch.float32, device=extrinsics.device)
|
| 15 |
+
reflect[0, 0] = -1
|
| 16 |
+
return reflect @ extrinsics @ reflect
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def reflect_views(views: AnyViews) -> AnyViews:
|
| 20 |
+
if "depth" in views.keys():
|
| 21 |
+
return {
|
| 22 |
+
**views,
|
| 23 |
+
"image": views["image"].flip(-1),
|
| 24 |
+
"extrinsics": reflect_extrinsics(views["extrinsics"]),
|
| 25 |
+
"depth": views["depth"].flip(-1),
|
| 26 |
+
}
|
| 27 |
+
else:
|
| 28 |
+
return {
|
| 29 |
+
**views,
|
| 30 |
+
"image": views["image"].flip(-1),
|
| 31 |
+
"extrinsics": reflect_extrinsics(views["extrinsics"]),
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def apply_augmentation_shim(
|
| 36 |
+
example: AnyExample,
|
| 37 |
+
generator: torch.Generator | None = None,
|
| 38 |
+
) -> AnyExample:
|
| 39 |
+
"""Randomly augment the training images."""
|
| 40 |
+
# Do not augment with 50% chance.
|
| 41 |
+
if torch.rand(tuple(), generator=generator) < 0.5:
|
| 42 |
+
return example
|
| 43 |
+
|
| 44 |
+
return {
|
| 45 |
+
**example,
|
| 46 |
+
"context": reflect_views(example["context"]),
|
| 47 |
+
"target": reflect_views(example["target"]),
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
def rotate_90_degrees(
|
| 51 |
+
image: torch.Tensor, depth_map: torch.Tensor | None, extri_opencv: torch.Tensor, intri_opencv: torch.Tensor, clockwise=True
|
| 52 |
+
):
|
| 53 |
+
"""
|
| 54 |
+
Rotates the input image, depth map, and camera parameters by 90 degrees.
|
| 55 |
+
|
| 56 |
+
Applies one of two 90-degree rotations:
|
| 57 |
+
- Clockwise
|
| 58 |
+
- Counterclockwise (if clockwise=False)
|
| 59 |
+
|
| 60 |
+
The extrinsic and intrinsic matrices are adjusted accordingly to maintain
|
| 61 |
+
correct camera geometry.
|
| 62 |
+
|
| 63 |
+
Args:
|
| 64 |
+
image (torch.Tensor):
|
| 65 |
+
Input image tensor of shape (C, H, W).
|
| 66 |
+
depth_map (torch.Tensor or None):
|
| 67 |
+
Depth map tensor of shape (H, W), or None if not available.
|
| 68 |
+
extri_opencv (torch.Tensor):
|
| 69 |
+
Extrinsic matrix (3x4) in OpenCV convention.
|
| 70 |
+
intri_opencv (torch.Tensor):
|
| 71 |
+
Intrinsic matrix (3x3).
|
| 72 |
+
clockwise (bool):
|
| 73 |
+
If True, rotates the image 90 degrees clockwise; else 90 degrees counterclockwise.
|
| 74 |
+
|
| 75 |
+
Returns:
|
| 76 |
+
tuple:
|
| 77 |
+
(
|
| 78 |
+
rotated_image,
|
| 79 |
+
rotated_depth_map,
|
| 80 |
+
new_extri_opencv,
|
| 81 |
+
new_intri_opencv
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
Where each is the updated version after the rotation.
|
| 85 |
+
"""
|
| 86 |
+
image_height, image_width = image.shape[-2:]
|
| 87 |
+
|
| 88 |
+
# Rotate the image and depth map
|
| 89 |
+
rotated_image, rotated_depth_map = rotate_image_and_depth_rot90(image, depth_map, clockwise)
|
| 90 |
+
# Adjust the intrinsic matrix
|
| 91 |
+
new_intri_opencv = adjust_intrinsic_matrix_rot90(intri_opencv, image_width, image_height, clockwise)
|
| 92 |
+
# Adjust the extrinsic matrix
|
| 93 |
+
new_extri_opencv = adjust_extrinsic_matrix_rot90(extri_opencv, clockwise)
|
| 94 |
+
|
| 95 |
+
return (
|
| 96 |
+
rotated_image,
|
| 97 |
+
rotated_depth_map,
|
| 98 |
+
new_extri_opencv,
|
| 99 |
+
new_intri_opencv,
|
| 100 |
+
)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def rotate_image_and_depth_rot90(image: torch.Tensor, depth_map: torch.Tensor | None, clockwise: bool):
|
| 104 |
+
"""
|
| 105 |
+
Rotates the given image and depth map by 90 degrees (clockwise or counterclockwise).
|
| 106 |
+
|
| 107 |
+
Args:
|
| 108 |
+
image (torch.Tensor):
|
| 109 |
+
Input image tensor of shape (C, H, W).
|
| 110 |
+
depth_map (torch.Tensor or None):
|
| 111 |
+
Depth map tensor of shape (H, W), or None if not available.
|
| 112 |
+
clockwise (bool):
|
| 113 |
+
If True, rotate 90 degrees clockwise; else 90 degrees counterclockwise.
|
| 114 |
+
|
| 115 |
+
Returns:
|
| 116 |
+
tuple:
|
| 117 |
+
(rotated_image, rotated_depth_map)
|
| 118 |
+
"""
|
| 119 |
+
rotated_depth_map = None
|
| 120 |
+
if clockwise:
|
| 121 |
+
rotated_image = torch.rot90(image, k=-1, dims=[-2, -1])
|
| 122 |
+
if depth_map is not None:
|
| 123 |
+
rotated_depth_map = torch.rot90(depth_map, k=-1, dims=[-2, -1])
|
| 124 |
+
else:
|
| 125 |
+
rotated_image = torch.rot90(image, k=1, dims=[-2, -1])
|
| 126 |
+
if depth_map is not None:
|
| 127 |
+
rotated_depth_map = torch.rot90(depth_map, k=1, dims=[-2, -1])
|
| 128 |
+
return rotated_image, rotated_depth_map
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def adjust_extrinsic_matrix_rot90(extri_opencv: torch.Tensor, clockwise: bool):
|
| 132 |
+
"""
|
| 133 |
+
Adjusts the extrinsic matrix (3x4) for a 90-degree rotation of the image.
|
| 134 |
+
|
| 135 |
+
The rotation is in the image plane. This modifies the camera orientation
|
| 136 |
+
accordingly. The function applies either a clockwise or counterclockwise
|
| 137 |
+
90-degree rotation.
|
| 138 |
+
|
| 139 |
+
Args:
|
| 140 |
+
extri_opencv (torch.Tensor):
|
| 141 |
+
Extrinsic matrix (3x4) in OpenCV convention.
|
| 142 |
+
clockwise (bool):
|
| 143 |
+
If True, rotate extrinsic for a 90-degree clockwise image rotation;
|
| 144 |
+
otherwise, counterclockwise.
|
| 145 |
+
|
| 146 |
+
Returns:
|
| 147 |
+
torch.Tensor:
|
| 148 |
+
A new 3x4 extrinsic matrix after the rotation.
|
| 149 |
+
"""
|
| 150 |
+
R = extri_opencv[:3, :3]
|
| 151 |
+
t = extri_opencv[:3, 3]
|
| 152 |
+
|
| 153 |
+
if clockwise:
|
| 154 |
+
R_rotation = torch.tensor([
|
| 155 |
+
[0, -1, 0],
|
| 156 |
+
[1, 0, 0],
|
| 157 |
+
[0, 0, 1]
|
| 158 |
+
], dtype=extri_opencv.dtype, device=extri_opencv.device)
|
| 159 |
+
else:
|
| 160 |
+
R_rotation = torch.tensor([
|
| 161 |
+
[0, 1, 0],
|
| 162 |
+
[-1, 0, 0],
|
| 163 |
+
[0, 0, 1]
|
| 164 |
+
], dtype=extri_opencv.dtype, device=extri_opencv.device)
|
| 165 |
+
|
| 166 |
+
new_R = torch.matmul(R_rotation, R)
|
| 167 |
+
new_t = torch.matmul(R_rotation, t)
|
| 168 |
+
new_extri_opencv = torch.cat((new_R, new_t.reshape(-1, 1)), dim=1)
|
| 169 |
+
new_extri_opencv = torch.cat((new_extri_opencv,
|
| 170 |
+
torch.tensor([[0, 0, 0, 1]],
|
| 171 |
+
dtype=extri_opencv.dtype, device=extri_opencv.device)), dim=0)
|
| 172 |
+
return new_extri_opencv
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def adjust_intrinsic_matrix_rot90(intri_opencv: torch.Tensor, image_width: int, image_height: int, clockwise: bool):
|
| 176 |
+
"""
|
| 177 |
+
Adjusts the intrinsic matrix (3x3) for a 90-degree rotation of the image in the image plane.
|
| 178 |
+
|
| 179 |
+
Args:
|
| 180 |
+
intri_opencv (torch.Tensor):
|
| 181 |
+
Intrinsic matrix (3x3).
|
| 182 |
+
image_width (int):
|
| 183 |
+
Original width of the image.
|
| 184 |
+
image_height (int):
|
| 185 |
+
Original height of the image.
|
| 186 |
+
clockwise (bool):
|
| 187 |
+
If True, rotate 90 degrees clockwise; else 90 degrees counterclockwise.
|
| 188 |
+
|
| 189 |
+
Returns:
|
| 190 |
+
torch.Tensor:
|
| 191 |
+
A new 3x3 intrinsic matrix after the rotation.
|
| 192 |
+
"""
|
| 193 |
+
intri_opencv = copy.deepcopy(intri_opencv)
|
| 194 |
+
intri_opencv[0, :] *= image_width
|
| 195 |
+
intri_opencv[1, :] *= image_height
|
| 196 |
+
|
| 197 |
+
fx, fy, cx, cy = (
|
| 198 |
+
intri_opencv[0, 0],
|
| 199 |
+
intri_opencv[1, 1],
|
| 200 |
+
intri_opencv[0, 2],
|
| 201 |
+
intri_opencv[1, 2],
|
| 202 |
+
)
|
| 203 |
+
|
| 204 |
+
new_intri_opencv = torch.eye(3, dtype=intri_opencv.dtype, device=intri_opencv.device)
|
| 205 |
+
if clockwise:
|
| 206 |
+
new_intri_opencv[0, 0] = fy
|
| 207 |
+
new_intri_opencv[1, 1] = fx
|
| 208 |
+
new_intri_opencv[0, 2] = image_height - cy
|
| 209 |
+
new_intri_opencv[1, 2] = cx
|
| 210 |
+
else:
|
| 211 |
+
new_intri_opencv[0, 0] = fy
|
| 212 |
+
new_intri_opencv[1, 1] = fx
|
| 213 |
+
new_intri_opencv[0, 2] = cy
|
| 214 |
+
new_intri_opencv[1, 2] = image_width - cx
|
| 215 |
+
|
| 216 |
+
new_intri_opencv[0, :] /= image_height
|
| 217 |
+
new_intri_opencv[1, :] /= image_width
|
| 218 |
+
|
| 219 |
+
return new_intri_opencv
|
src/dataset/shims/bounds_shim.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from einops import einsum, reduce, repeat
|
| 3 |
+
from jaxtyping import Float
|
| 4 |
+
from torch import Tensor
|
| 5 |
+
|
| 6 |
+
from ..types import BatchedExample
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def compute_depth_for_disparity(
|
| 10 |
+
extrinsics: Float[Tensor, "batch view 4 4"],
|
| 11 |
+
intrinsics: Float[Tensor, "batch view 3 3"],
|
| 12 |
+
image_shape: tuple[int, int],
|
| 13 |
+
disparity: float,
|
| 14 |
+
delta_min: float = 1e-6, # This prevents motionless scenes from lacking depth.
|
| 15 |
+
) -> Float[Tensor, " batch"]:
|
| 16 |
+
"""Compute the depth at which moving the maximum distance between cameras
|
| 17 |
+
corresponds to the specified disparity (in pixels).
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
# Use the furthest distance between cameras as the baseline.
|
| 21 |
+
origins = extrinsics[:, :, :3, 3]
|
| 22 |
+
deltas = (origins[:, None, :, :] - origins[:, :, None, :]).norm(dim=-1)
|
| 23 |
+
deltas = deltas.clip(min=delta_min)
|
| 24 |
+
baselines = reduce(deltas, "b v ov -> b", "max")
|
| 25 |
+
|
| 26 |
+
# Compute a single pixel's size at depth 1.
|
| 27 |
+
h, w = image_shape
|
| 28 |
+
pixel_size = 1 / torch.tensor((w, h), dtype=torch.float32, device=extrinsics.device)
|
| 29 |
+
pixel_size = einsum(
|
| 30 |
+
intrinsics[..., :2, :2].inverse(), pixel_size, "... i j, j -> ... i"
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
# This wouldn't make sense with non-square pixels, but then again, non-square pixels
|
| 34 |
+
# don't make much sense anyway.
|
| 35 |
+
mean_pixel_size = reduce(pixel_size, "b v xy -> b", "mean")
|
| 36 |
+
|
| 37 |
+
return baselines / (disparity * mean_pixel_size)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def apply_bounds_shim(
|
| 41 |
+
batch: BatchedExample,
|
| 42 |
+
near_disparity: float,
|
| 43 |
+
far_disparity: float,
|
| 44 |
+
) -> BatchedExample:
|
| 45 |
+
"""Compute reasonable near and far planes (lower and upper bounds on depth). This
|
| 46 |
+
assumes that all of an example's views are of roughly the same thing.
|
| 47 |
+
"""
|
| 48 |
+
|
| 49 |
+
context = batch["context"]
|
| 50 |
+
_, cv, _, h, w = context["image"].shape
|
| 51 |
+
|
| 52 |
+
# Compute near and far planes using the context views.
|
| 53 |
+
near = compute_depth_for_disparity(
|
| 54 |
+
context["extrinsics"],
|
| 55 |
+
context["intrinsics"],
|
| 56 |
+
(h, w),
|
| 57 |
+
near_disparity,
|
| 58 |
+
)
|
| 59 |
+
far = compute_depth_for_disparity(
|
| 60 |
+
context["extrinsics"],
|
| 61 |
+
context["intrinsics"],
|
| 62 |
+
(h, w),
|
| 63 |
+
far_disparity,
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
target = batch["target"]
|
| 67 |
+
_, tv, _, _, _ = target["image"].shape
|
| 68 |
+
return {
|
| 69 |
+
**batch,
|
| 70 |
+
"context": {
|
| 71 |
+
**context,
|
| 72 |
+
"near": repeat(near, "b -> b v", v=cv),
|
| 73 |
+
"far": repeat(far, "b -> b v", v=cv),
|
| 74 |
+
},
|
| 75 |
+
"target": {
|
| 76 |
+
**target,
|
| 77 |
+
"near": repeat(near, "b -> b v", v=tv),
|
| 78 |
+
"far": repeat(far, "b -> b v", v=tv),
|
| 79 |
+
},
|
| 80 |
+
}
|
src/dataset/shims/crop_shim.py
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import random
|
| 2 |
+
import numpy as np
|
| 3 |
+
import torch
|
| 4 |
+
from einops import rearrange
|
| 5 |
+
from jaxtyping import Float
|
| 6 |
+
from PIL import Image
|
| 7 |
+
from torch import Tensor
|
| 8 |
+
import torchvision.transforms.functional as F
|
| 9 |
+
import cv2
|
| 10 |
+
|
| 11 |
+
from ..types import AnyExample, AnyViews
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def rescale(
|
| 15 |
+
image: Float[Tensor, "3 h_in w_in"],
|
| 16 |
+
shape: tuple[int, int],
|
| 17 |
+
) -> Float[Tensor, "3 h_out w_out"]:
|
| 18 |
+
h, w = shape
|
| 19 |
+
image_new = (image * 255).clip(min=0, max=255).type(torch.uint8)
|
| 20 |
+
image_new = rearrange(image_new, "c h w -> h w c").detach().cpu().numpy()
|
| 21 |
+
image_new = Image.fromarray(image_new)
|
| 22 |
+
image_new = image_new.resize((w, h), Image.LANCZOS)
|
| 23 |
+
image_new = np.array(image_new) / 255
|
| 24 |
+
image_new = torch.tensor(image_new, dtype=image.dtype, device=image.device)
|
| 25 |
+
return rearrange(image_new, "h w c -> c h w")
|
| 26 |
+
|
| 27 |
+
def rescale_depth(
|
| 28 |
+
depth: Float[Tensor, "1 h w"],
|
| 29 |
+
shape: tuple[int, int],
|
| 30 |
+
) -> Float[Tensor, "1 h_out w_out"]:
|
| 31 |
+
h, w = shape
|
| 32 |
+
depth_new = depth.detach().cpu().numpy()
|
| 33 |
+
depth_new = cv2.resize(depth_new, (w,h), interpolation=cv2.INTER_NEAREST)
|
| 34 |
+
depth_new = torch.from_numpy(depth_new).to(depth.device)
|
| 35 |
+
return depth_new
|
| 36 |
+
|
| 37 |
+
def center_crop(
|
| 38 |
+
images: Float[Tensor, "*#batch c h w"],
|
| 39 |
+
intrinsics: Float[Tensor, "*#batch 3 3"],
|
| 40 |
+
shape: tuple[int, int],
|
| 41 |
+
depths: Float[Tensor, "*#batch 1 h w"] | None = None,
|
| 42 |
+
) -> tuple[
|
| 43 |
+
Float[Tensor, "*#batch c h_out w_out"], # updated images
|
| 44 |
+
Float[Tensor, "*#batch 3 3"], # updated intrinsics
|
| 45 |
+
Float[Tensor, "*#batch 1 h_out w_out"] | None, # updated depths
|
| 46 |
+
]:
|
| 47 |
+
*_, h_in, w_in = images.shape
|
| 48 |
+
h_out, w_out = shape
|
| 49 |
+
|
| 50 |
+
# Note that odd input dimensions induce half-pixel misalignments.
|
| 51 |
+
row = (h_in - h_out) // 2
|
| 52 |
+
col = (w_in - w_out) // 2
|
| 53 |
+
|
| 54 |
+
# Center-crop the image.
|
| 55 |
+
images = images[..., :, row : row + h_out, col : col + w_out]
|
| 56 |
+
|
| 57 |
+
if depths is not None:
|
| 58 |
+
depths = depths[..., row : row + h_out, col : col + w_out]
|
| 59 |
+
|
| 60 |
+
# Adjust the intrinsics to account for the cropping.
|
| 61 |
+
intrinsics = intrinsics.clone()
|
| 62 |
+
intrinsics[..., 0, 0] *= w_in / w_out # fx
|
| 63 |
+
intrinsics[..., 1, 1] *= h_in / h_out # fy
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
if depths is not None:
|
| 67 |
+
return images, intrinsics, depths
|
| 68 |
+
else:
|
| 69 |
+
return images, intrinsics
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def rescale_and_crop(
|
| 73 |
+
images: Float[Tensor, "*#batch c h w"],
|
| 74 |
+
intrinsics: Float[Tensor, "*#batch 3 3"],
|
| 75 |
+
shape: tuple[int, int],
|
| 76 |
+
intr_aug: bool = False,
|
| 77 |
+
scale_range: tuple[float, float] = (0.77, 1.0),
|
| 78 |
+
depths: Float[Tensor, "*#batch 1 h w"] | None = None,
|
| 79 |
+
) -> tuple[
|
| 80 |
+
Float[Tensor, "*#batch c h_out w_out"], # updated images
|
| 81 |
+
Float[Tensor, "*#batch 3 3"], # updated intrinsics
|
| 82 |
+
Float[Tensor, "*#batch 1 h_out w_out"] | None, # updated depths
|
| 83 |
+
]:
|
| 84 |
+
if type(images) == list:
|
| 85 |
+
images_new = []
|
| 86 |
+
intrinsics_new = []
|
| 87 |
+
for i in range(len(images)):
|
| 88 |
+
image = images[i]
|
| 89 |
+
intrinsic = intrinsics[i]
|
| 90 |
+
|
| 91 |
+
*_, h_in, w_in = image.shape
|
| 92 |
+
h_out, w_out = shape
|
| 93 |
+
|
| 94 |
+
scale_factor = max(h_out / h_in, w_out / w_in)
|
| 95 |
+
h_scaled = round(h_in * scale_factor)
|
| 96 |
+
w_scaled = round(w_in * scale_factor)
|
| 97 |
+
image = F.resize(image, (h_scaled, w_scaled))
|
| 98 |
+
image = F.center_crop(image, (h_out, w_out))
|
| 99 |
+
images_new.append(image)
|
| 100 |
+
|
| 101 |
+
intrinsic_new = intrinsic.clone()
|
| 102 |
+
intrinsic_new[..., 0, 0] *= w_scaled / w_in # fx
|
| 103 |
+
intrinsic_new[..., 1, 1] *= h_scaled / h_in # fy
|
| 104 |
+
intrinsics_new.append(intrinsic_new)
|
| 105 |
+
|
| 106 |
+
if depths is not None:
|
| 107 |
+
depths_new = []
|
| 108 |
+
for i in range(len(depths)):
|
| 109 |
+
depth = depths[i]
|
| 110 |
+
depth = rescale_depth(depth, (h_out, w_out))
|
| 111 |
+
depth = F.center_crop(depth, (h_out, w_out))
|
| 112 |
+
depths_new.append(depth)
|
| 113 |
+
return torch.stack(images_new), torch.stack(intrinsics_new), torch.stack(depths_new)
|
| 114 |
+
else:
|
| 115 |
+
return torch.stack(images_new), torch.stack(intrinsics_new)
|
| 116 |
+
|
| 117 |
+
else:
|
| 118 |
+
# we only support intr_aug for clean datasets
|
| 119 |
+
*_, h_in, w_in = images.shape
|
| 120 |
+
h_out, w_out = shape
|
| 121 |
+
# assert h_out <= h_in and w_out <= w_in # to avoid the case that the image is too small, like co3d
|
| 122 |
+
|
| 123 |
+
if intr_aug:
|
| 124 |
+
scale = random.uniform(*scale_range)
|
| 125 |
+
h_scale = round(h_out * scale)
|
| 126 |
+
w_scale = round(w_out * scale)
|
| 127 |
+
else:
|
| 128 |
+
h_scale = h_out
|
| 129 |
+
w_scale = w_out
|
| 130 |
+
|
| 131 |
+
scale_factor = max(h_scale / h_in, w_scale / w_in)
|
| 132 |
+
h_scaled = round(h_in * scale_factor)
|
| 133 |
+
w_scaled = round(w_in * scale_factor)
|
| 134 |
+
assert h_scaled == h_scale or w_scaled == w_scale
|
| 135 |
+
|
| 136 |
+
# Reshape the images to the correct size. Assume we don't have to worry about
|
| 137 |
+
# changing the intrinsics based on how the images are rounded.
|
| 138 |
+
*batch, c, h, w = images.shape
|
| 139 |
+
images = images.reshape(-1, c, h, w)
|
| 140 |
+
images = torch.stack([rescale(image, (h_scaled, w_scaled)) for image in images])
|
| 141 |
+
images = images.reshape(*batch, c, h_scaled, w_scaled)
|
| 142 |
+
|
| 143 |
+
if depths is not None:
|
| 144 |
+
if type(depths) == list:
|
| 145 |
+
depths_new = []
|
| 146 |
+
for i in range(len(depths)):
|
| 147 |
+
depth = depths[i]
|
| 148 |
+
depth = rescale_depth(depth, (h_scaled, w_scaled))
|
| 149 |
+
depths_new.append(depth)
|
| 150 |
+
depths = torch.stack(depths_new)
|
| 151 |
+
else:
|
| 152 |
+
depths = depths.reshape(-1, h, w)
|
| 153 |
+
depths = torch.stack([rescale_depth(depth, (h_scaled, w_scaled)) for depth in depths])
|
| 154 |
+
depths = depths.reshape(*batch, h_scaled, w_scaled)
|
| 155 |
+
|
| 156 |
+
images, intrinsics, depths = center_crop(images, intrinsics, (h_scale, w_scale), depths)
|
| 157 |
+
|
| 158 |
+
if intr_aug:
|
| 159 |
+
images = F.resize(images, size=(h_out, w_out), interpolation=F.InterpolationMode.BILINEAR)
|
| 160 |
+
depths = F.resize(depths, size=(h_out, w_out), interpolation=F.InterpolationMode.NEAREST)
|
| 161 |
+
|
| 162 |
+
return images, intrinsics, depths
|
| 163 |
+
else:
|
| 164 |
+
images, intrinsics = center_crop(images, intrinsics, (h_scale, w_scale))
|
| 165 |
+
|
| 166 |
+
if intr_aug:
|
| 167 |
+
images = F.resize(images, size=(h_out, w_out))
|
| 168 |
+
|
| 169 |
+
return images, intrinsics
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def apply_crop_shim_to_views(views: AnyViews, shape: tuple[int, int], intr_aug: bool = False) -> AnyViews:
|
| 173 |
+
if "depth" in views.keys():
|
| 174 |
+
images, intrinsics, depths = rescale_and_crop(views["image"], views["intrinsics"], shape, depths=views["depth"], intr_aug=intr_aug)
|
| 175 |
+
return {
|
| 176 |
+
**views,
|
| 177 |
+
"image": images,
|
| 178 |
+
"intrinsics": intrinsics,
|
| 179 |
+
"depth": depths,
|
| 180 |
+
}
|
| 181 |
+
else:
|
| 182 |
+
images, intrinsics = rescale_and_crop(views["image"], views["intrinsics"], shape, intr_aug)
|
| 183 |
+
return {
|
| 184 |
+
**views,
|
| 185 |
+
"image": images,
|
| 186 |
+
"intrinsics": intrinsics,
|
| 187 |
+
}
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def apply_crop_shim(example: AnyExample, shape: tuple[int, int], intr_aug: bool = False) -> AnyExample:
|
| 191 |
+
"""Crop images in the example."""
|
| 192 |
+
return {
|
| 193 |
+
**example,
|
| 194 |
+
"context": apply_crop_shim_to_views(example["context"], shape, intr_aug),
|
| 195 |
+
"target": apply_crop_shim_to_views(example["target"], shape, intr_aug),
|
| 196 |
+
}
|
src/dataset/shims/geometry_shim.py
ADDED
|
@@ -0,0 +1,383 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (C) 2024-present Naver Corporation. All rights reserved.
|
| 2 |
+
# Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
|
| 3 |
+
#
|
| 4 |
+
# --------------------------------------------------------
|
| 5 |
+
# geometry utilitary functions
|
| 6 |
+
# --------------------------------------------------------
|
| 7 |
+
import torch
|
| 8 |
+
import numpy as np
|
| 9 |
+
from scipy.spatial import cKDTree as KDTree
|
| 10 |
+
|
| 11 |
+
def invalid_to_nans(arr, valid_mask, ndim=999):
|
| 12 |
+
if valid_mask is not None:
|
| 13 |
+
arr = arr.clone()
|
| 14 |
+
arr[~valid_mask] = float('nan')
|
| 15 |
+
if arr.ndim > ndim:
|
| 16 |
+
arr = arr.flatten(-2 - (arr.ndim - ndim), -2)
|
| 17 |
+
return arr
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def invalid_to_zeros(arr, valid_mask, ndim=999):
|
| 21 |
+
if valid_mask is not None:
|
| 22 |
+
arr = arr.clone()
|
| 23 |
+
arr[~valid_mask] = 0
|
| 24 |
+
nnz = valid_mask.view(len(valid_mask), -1).sum(1)
|
| 25 |
+
else:
|
| 26 |
+
nnz = arr.numel() // len(arr) if len(arr) else 0 # number of point per image
|
| 27 |
+
if arr.ndim > ndim:
|
| 28 |
+
arr = arr.flatten(-2 - (arr.ndim - ndim), -2)
|
| 29 |
+
return arr, nnz
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def xy_grid(W, H, device=None, origin=(0, 0), unsqueeze=None, cat_dim=-1, homogeneous=False, **arange_kw):
|
| 33 |
+
""" Output a (H,W,2) array of int32
|
| 34 |
+
with output[j,i,0] = i + origin[0]
|
| 35 |
+
output[j,i,1] = j + origin[1]
|
| 36 |
+
"""
|
| 37 |
+
if device is None:
|
| 38 |
+
# numpy
|
| 39 |
+
arange, meshgrid, stack, ones = np.arange, np.meshgrid, np.stack, np.ones
|
| 40 |
+
else:
|
| 41 |
+
# torch
|
| 42 |
+
arange = lambda *a, **kw: torch.arange(*a, device=device, **kw)
|
| 43 |
+
meshgrid, stack = torch.meshgrid, torch.stack
|
| 44 |
+
ones = lambda *a: torch.ones(*a, device=device)
|
| 45 |
+
|
| 46 |
+
tw, th = [arange(o, o + s, **arange_kw) for s, o in zip((W, H), origin)]
|
| 47 |
+
grid = meshgrid(tw, th, indexing='xy')
|
| 48 |
+
if homogeneous:
|
| 49 |
+
grid = grid + (ones((H, W)),)
|
| 50 |
+
if unsqueeze is not None:
|
| 51 |
+
grid = (grid[0].unsqueeze(unsqueeze), grid[1].unsqueeze(unsqueeze))
|
| 52 |
+
if cat_dim is not None:
|
| 53 |
+
grid = stack(grid, cat_dim)
|
| 54 |
+
return grid
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def geotrf(Trf, pts, ncol=None, norm=False):
|
| 58 |
+
""" Apply a geometric transformation to a list of 3-D points.
|
| 59 |
+
|
| 60 |
+
H: 3x3 or 4x4 projection matrix (typically a Homography)
|
| 61 |
+
p: numpy/torch/tuple of coordinates. Shape must be (...,2) or (...,3)
|
| 62 |
+
|
| 63 |
+
ncol: int. number of columns of the result (2 or 3)
|
| 64 |
+
norm: float. if != 0, the resut is projected on the z=norm plane.
|
| 65 |
+
|
| 66 |
+
Returns an array of projected 2d points.
|
| 67 |
+
"""
|
| 68 |
+
assert Trf.ndim >= 2
|
| 69 |
+
if isinstance(Trf, np.ndarray):
|
| 70 |
+
pts = np.asarray(pts)
|
| 71 |
+
elif isinstance(Trf, torch.Tensor):
|
| 72 |
+
pts = torch.as_tensor(pts, dtype=Trf.dtype)
|
| 73 |
+
|
| 74 |
+
# adapt shape if necessary
|
| 75 |
+
output_reshape = pts.shape[:-1]
|
| 76 |
+
ncol = ncol or pts.shape[-1]
|
| 77 |
+
|
| 78 |
+
# optimized code
|
| 79 |
+
if (isinstance(Trf, torch.Tensor) and isinstance(pts, torch.Tensor) and
|
| 80 |
+
Trf.ndim == 3 and pts.ndim == 4):
|
| 81 |
+
d = pts.shape[3]
|
| 82 |
+
if Trf.shape[-1] == d:
|
| 83 |
+
pts = torch.einsum("bij, bhwj -> bhwi", Trf, pts)
|
| 84 |
+
elif Trf.shape[-1] == d + 1:
|
| 85 |
+
pts = torch.einsum("bij, bhwj -> bhwi", Trf[:, :d, :d], pts) + Trf[:, None, None, :d, d]
|
| 86 |
+
else:
|
| 87 |
+
raise ValueError(f'bad shape, not ending with 3 or 4, for {pts.shape=}')
|
| 88 |
+
else:
|
| 89 |
+
if Trf.ndim >= 3:
|
| 90 |
+
n = Trf.ndim - 2
|
| 91 |
+
assert Trf.shape[:n] == pts.shape[:n], 'batch size does not match'
|
| 92 |
+
Trf = Trf.reshape(-1, Trf.shape[-2], Trf.shape[-1])
|
| 93 |
+
|
| 94 |
+
if pts.ndim > Trf.ndim:
|
| 95 |
+
# Trf == (B,d,d) & pts == (B,H,W,d) --> (B, H*W, d)
|
| 96 |
+
pts = pts.reshape(Trf.shape[0], -1, pts.shape[-1])
|
| 97 |
+
elif pts.ndim == 2:
|
| 98 |
+
# Trf == (B,d,d) & pts == (B,d) --> (B, 1, d)
|
| 99 |
+
pts = pts[:, None, :]
|
| 100 |
+
|
| 101 |
+
if pts.shape[-1] + 1 == Trf.shape[-1]:
|
| 102 |
+
Trf = Trf.swapaxes(-1, -2) # transpose Trf
|
| 103 |
+
pts = pts @ Trf[..., :-1, :] + Trf[..., -1:, :]
|
| 104 |
+
elif pts.shape[-1] == Trf.shape[-1]:
|
| 105 |
+
Trf = Trf.swapaxes(-1, -2) # transpose Trf
|
| 106 |
+
pts = pts @ Trf
|
| 107 |
+
else:
|
| 108 |
+
pts = Trf @ pts.T
|
| 109 |
+
if pts.ndim >= 2:
|
| 110 |
+
pts = pts.swapaxes(-1, -2)
|
| 111 |
+
|
| 112 |
+
if norm:
|
| 113 |
+
pts = pts / pts[..., -1:] # DONT DO /= BECAUSE OF WEIRD PYTORCH BUG
|
| 114 |
+
if norm != 1:
|
| 115 |
+
pts *= norm
|
| 116 |
+
|
| 117 |
+
res = pts[..., :ncol].reshape(*output_reshape, ncol)
|
| 118 |
+
return res
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def inv(mat):
|
| 122 |
+
""" Invert a torch or numpy matrix
|
| 123 |
+
"""
|
| 124 |
+
if isinstance(mat, torch.Tensor):
|
| 125 |
+
return torch.linalg.inv(mat)
|
| 126 |
+
if isinstance(mat, np.ndarray):
|
| 127 |
+
return np.linalg.inv(mat)
|
| 128 |
+
raise ValueError(f'bad matrix type = {type(mat)}')
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def depthmap_to_pts3d(depth, pseudo_focal, pp=None, **_):
|
| 132 |
+
"""
|
| 133 |
+
Args:
|
| 134 |
+
- depthmap (BxHxW array):
|
| 135 |
+
- pseudo_focal: [B,H,W] ; [B,2,H,W] or [B,1,H,W]
|
| 136 |
+
Returns:
|
| 137 |
+
pointmap of absolute coordinates (BxHxWx3 array)
|
| 138 |
+
"""
|
| 139 |
+
|
| 140 |
+
if len(depth.shape) == 4:
|
| 141 |
+
B, H, W, n = depth.shape
|
| 142 |
+
else:
|
| 143 |
+
B, H, W = depth.shape
|
| 144 |
+
n = None
|
| 145 |
+
|
| 146 |
+
if len(pseudo_focal.shape) == 3: # [B,H,W]
|
| 147 |
+
pseudo_focalx = pseudo_focaly = pseudo_focal
|
| 148 |
+
elif len(pseudo_focal.shape) == 4: # [B,2,H,W] or [B,1,H,W]
|
| 149 |
+
pseudo_focalx = pseudo_focal[:, 0]
|
| 150 |
+
if pseudo_focal.shape[1] == 2:
|
| 151 |
+
pseudo_focaly = pseudo_focal[:, 1]
|
| 152 |
+
else:
|
| 153 |
+
pseudo_focaly = pseudo_focalx
|
| 154 |
+
else:
|
| 155 |
+
raise NotImplementedError("Error, unknown input focal shape format.")
|
| 156 |
+
|
| 157 |
+
assert pseudo_focalx.shape == depth.shape[:3]
|
| 158 |
+
assert pseudo_focaly.shape == depth.shape[:3]
|
| 159 |
+
grid_x, grid_y = xy_grid(W, H, cat_dim=0, device=depth.device)[:, None]
|
| 160 |
+
|
| 161 |
+
# set principal point
|
| 162 |
+
if pp is None:
|
| 163 |
+
grid_x = grid_x - (W - 1) / 2
|
| 164 |
+
grid_y = grid_y - (H - 1) / 2
|
| 165 |
+
else:
|
| 166 |
+
grid_x = grid_x.expand(B, -1, -1) - pp[:, 0, None, None]
|
| 167 |
+
grid_y = grid_y.expand(B, -1, -1) - pp[:, 1, None, None]
|
| 168 |
+
|
| 169 |
+
if n is None:
|
| 170 |
+
pts3d = torch.empty((B, H, W, 3), device=depth.device)
|
| 171 |
+
pts3d[..., 0] = depth * grid_x / pseudo_focalx
|
| 172 |
+
pts3d[..., 1] = depth * grid_y / pseudo_focaly
|
| 173 |
+
pts3d[..., 2] = depth
|
| 174 |
+
else:
|
| 175 |
+
pts3d = torch.empty((B, H, W, 3, n), device=depth.device)
|
| 176 |
+
pts3d[..., 0, :] = depth * (grid_x / pseudo_focalx)[..., None]
|
| 177 |
+
pts3d[..., 1, :] = depth * (grid_y / pseudo_focaly)[..., None]
|
| 178 |
+
pts3d[..., 2, :] = depth
|
| 179 |
+
return pts3d
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def depthmap_to_camera_coordinates(depthmap, camera_intrinsics, pseudo_focal=None):
|
| 183 |
+
"""
|
| 184 |
+
Args:
|
| 185 |
+
- depthmap (HxW array):
|
| 186 |
+
- camera_intrinsics: a 3x3 matrix
|
| 187 |
+
Returns:
|
| 188 |
+
pointmap of absolute coordinates (HxWx3 array), and a mask specifying valid pixels.
|
| 189 |
+
"""
|
| 190 |
+
camera_intrinsics = np.float32(camera_intrinsics)
|
| 191 |
+
H, W = depthmap.shape
|
| 192 |
+
|
| 193 |
+
# Compute 3D ray associated with each pixel
|
| 194 |
+
# Strong assumption: there are no skew terms
|
| 195 |
+
assert camera_intrinsics[0, 1] == 0.0
|
| 196 |
+
assert camera_intrinsics[1, 0] == 0.0
|
| 197 |
+
if pseudo_focal is None:
|
| 198 |
+
fu = camera_intrinsics[0, 0]
|
| 199 |
+
fv = camera_intrinsics[1, 1]
|
| 200 |
+
else:
|
| 201 |
+
assert pseudo_focal.shape == (H, W)
|
| 202 |
+
fu = fv = pseudo_focal
|
| 203 |
+
cu = camera_intrinsics[0, 2]
|
| 204 |
+
cv = camera_intrinsics[1, 2]
|
| 205 |
+
|
| 206 |
+
u, v = np.meshgrid(np.arange(W), np.arange(H))
|
| 207 |
+
z_cam = depthmap
|
| 208 |
+
x_cam = (u - cu) * z_cam / fu
|
| 209 |
+
y_cam = (v - cv) * z_cam / fv
|
| 210 |
+
X_cam = np.stack((x_cam, y_cam, z_cam), axis=-1).astype(np.float32)
|
| 211 |
+
|
| 212 |
+
# Mask for valid coordinates
|
| 213 |
+
valid_mask = (depthmap > 0.0)
|
| 214 |
+
return X_cam, valid_mask
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def depthmap_to_absolute_camera_coordinates(depthmap, camera_intrinsics, camera_pose=None, **kw):
|
| 218 |
+
"""
|
| 219 |
+
Args:
|
| 220 |
+
- depthmap (HxW array):
|
| 221 |
+
- camera_intrinsics: a 3x3 matrix
|
| 222 |
+
- camera_pose: a 4x3 or 4x4 cam2world matrix
|
| 223 |
+
Returns:
|
| 224 |
+
pointmap of absolute coordinates (HxWx3 array), and a mask specifying valid pixels."""
|
| 225 |
+
X_cam, valid_mask = depthmap_to_camera_coordinates(depthmap, camera_intrinsics)
|
| 226 |
+
|
| 227 |
+
X_world = X_cam # default
|
| 228 |
+
if camera_pose is not None:
|
| 229 |
+
# R_cam2world = np.float32(camera_params["R_cam2world"])
|
| 230 |
+
# t_cam2world = np.float32(camera_params["t_cam2world"]).squeeze()
|
| 231 |
+
R_cam2world = camera_pose[:3, :3]
|
| 232 |
+
t_cam2world = camera_pose[:3, 3]
|
| 233 |
+
|
| 234 |
+
# Express in absolute coordinates (invalid depth values)
|
| 235 |
+
X_world = np.einsum("ik, vuk -> vui", R_cam2world, X_cam) + t_cam2world[None, None, :]
|
| 236 |
+
|
| 237 |
+
return X_world, valid_mask
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def colmap_to_opencv_intrinsics(K):
|
| 241 |
+
"""
|
| 242 |
+
Modify camera intrinsics to follow a different convention.
|
| 243 |
+
Coordinates of the center of the top-left pixels are by default:
|
| 244 |
+
- (0.5, 0.5) in Colmap
|
| 245 |
+
- (0,0) in OpenCV
|
| 246 |
+
"""
|
| 247 |
+
K = K.copy()
|
| 248 |
+
K[0, 2] -= 0.5
|
| 249 |
+
K[1, 2] -= 0.5
|
| 250 |
+
return K
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def opencv_to_colmap_intrinsics(K):
|
| 254 |
+
"""
|
| 255 |
+
Modify camera intrinsics to follow a different convention.
|
| 256 |
+
Coordinates of the center of the top-left pixels are by default:
|
| 257 |
+
- (0.5, 0.5) in Colmap
|
| 258 |
+
- (0,0) in OpenCV
|
| 259 |
+
"""
|
| 260 |
+
K = K.copy()
|
| 261 |
+
K[0, 2] += 0.5
|
| 262 |
+
K[1, 2] += 0.5
|
| 263 |
+
return K
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
def normalize_pointcloud(pts1, pts2, norm_mode='avg_dis', valid1=None, valid2=None, ret_factor=False):
|
| 267 |
+
""" renorm pointmaps pts1, pts2 with norm_mode
|
| 268 |
+
"""
|
| 269 |
+
assert pts1.ndim >= 3 and pts1.shape[-1] == 3
|
| 270 |
+
assert pts2 is None or (pts2.ndim >= 3 and pts2.shape[-1] == 3)
|
| 271 |
+
norm_mode, dis_mode = norm_mode.split('_')
|
| 272 |
+
|
| 273 |
+
if norm_mode == 'avg':
|
| 274 |
+
# gather all points together (joint normalization)
|
| 275 |
+
nan_pts1, nnz1 = invalid_to_zeros(pts1, valid1, ndim=3)
|
| 276 |
+
nan_pts2, nnz2 = invalid_to_zeros(pts2, valid2, ndim=3) if pts2 is not None else (None, 0)
|
| 277 |
+
all_pts = torch.cat((nan_pts1, nan_pts2), dim=1) if pts2 is not None else nan_pts1
|
| 278 |
+
|
| 279 |
+
# compute distance to origin
|
| 280 |
+
all_dis = all_pts.norm(dim=-1)
|
| 281 |
+
if dis_mode == 'dis':
|
| 282 |
+
pass # do nothing
|
| 283 |
+
elif dis_mode == 'log1p':
|
| 284 |
+
all_dis = torch.log1p(all_dis)
|
| 285 |
+
elif dis_mode == 'warp-log1p':
|
| 286 |
+
# actually warp input points before normalizing them
|
| 287 |
+
log_dis = torch.log1p(all_dis)
|
| 288 |
+
warp_factor = log_dis / all_dis.clip(min=1e-8)
|
| 289 |
+
H1, W1 = pts1.shape[1:-1]
|
| 290 |
+
pts1 = pts1 * warp_factor[:, :W1 * H1].view(-1, H1, W1, 1)
|
| 291 |
+
if pts2 is not None:
|
| 292 |
+
H2, W2 = pts2.shape[1:-1]
|
| 293 |
+
pts2 = pts2 * warp_factor[:, W1 * H1:].view(-1, H2, W2, 1)
|
| 294 |
+
all_dis = log_dis # this is their true distance afterwards
|
| 295 |
+
else:
|
| 296 |
+
raise ValueError(f'bad {dis_mode=}')
|
| 297 |
+
|
| 298 |
+
norm_factor = all_dis.sum(dim=1) / (nnz1 + nnz2 + 1e-8)
|
| 299 |
+
else:
|
| 300 |
+
# gather all points together (joint normalization)
|
| 301 |
+
nan_pts1 = invalid_to_nans(pts1, valid1, ndim=3)
|
| 302 |
+
nan_pts2 = invalid_to_nans(pts2, valid2, ndim=3) if pts2 is not None else None
|
| 303 |
+
all_pts = torch.cat((nan_pts1, nan_pts2), dim=1) if pts2 is not None else nan_pts1
|
| 304 |
+
|
| 305 |
+
# compute distance to origin
|
| 306 |
+
all_dis = all_pts.norm(dim=-1)
|
| 307 |
+
|
| 308 |
+
if norm_mode == 'avg':
|
| 309 |
+
norm_factor = all_dis.nanmean(dim=1)
|
| 310 |
+
elif norm_mode == 'median':
|
| 311 |
+
norm_factor = all_dis.nanmedian(dim=1).values.detach()
|
| 312 |
+
elif norm_mode == 'sqrt':
|
| 313 |
+
norm_factor = all_dis.sqrt().nanmean(dim=1)**2
|
| 314 |
+
else:
|
| 315 |
+
raise ValueError(f'bad {norm_mode=}')
|
| 316 |
+
|
| 317 |
+
norm_factor = norm_factor.clip(min=1e-8)
|
| 318 |
+
while norm_factor.ndim < pts1.ndim:
|
| 319 |
+
norm_factor.unsqueeze_(-1)
|
| 320 |
+
|
| 321 |
+
res = pts1 / norm_factor
|
| 322 |
+
if pts2 is not None:
|
| 323 |
+
res = (res, pts2 / norm_factor)
|
| 324 |
+
if ret_factor:
|
| 325 |
+
res = res + (norm_factor,)
|
| 326 |
+
return res
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
@torch.no_grad()
|
| 330 |
+
def get_joint_pointcloud_depth(z1, z2, valid_mask1, valid_mask2=None, quantile=0.5):
|
| 331 |
+
# set invalid points to NaN
|
| 332 |
+
_z1 = invalid_to_nans(z1, valid_mask1).reshape(len(z1), -1)
|
| 333 |
+
_z2 = invalid_to_nans(z2, valid_mask2).reshape(len(z2), -1) if z2 is not None else None
|
| 334 |
+
_z = torch.cat((_z1, _z2), dim=-1) if z2 is not None else _z1
|
| 335 |
+
|
| 336 |
+
# compute median depth overall (ignoring nans)
|
| 337 |
+
if quantile == 0.5:
|
| 338 |
+
shift_z = torch.nanmedian(_z, dim=-1).values
|
| 339 |
+
else:
|
| 340 |
+
shift_z = torch.nanquantile(_z, quantile, dim=-1)
|
| 341 |
+
return shift_z # (B,)
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
@torch.no_grad()
|
| 345 |
+
def get_joint_pointcloud_center_scale(pts1, pts2, valid_mask1=None, valid_mask2=None, z_only=False, center=True):
|
| 346 |
+
# set invalid points to NaN
|
| 347 |
+
_pts1 = invalid_to_nans(pts1, valid_mask1).reshape(len(pts1), -1, 3)
|
| 348 |
+
_pts2 = invalid_to_nans(pts2, valid_mask2).reshape(len(pts2), -1, 3) if pts2 is not None else None
|
| 349 |
+
_pts = torch.cat((_pts1, _pts2), dim=1) if pts2 is not None else _pts1
|
| 350 |
+
|
| 351 |
+
# compute median center
|
| 352 |
+
_center = torch.nanmedian(_pts, dim=1, keepdim=True).values # (B,1,3)
|
| 353 |
+
if z_only:
|
| 354 |
+
_center[..., :2] = 0 # do not center X and Y
|
| 355 |
+
|
| 356 |
+
# compute median norm
|
| 357 |
+
_norm = ((_pts - _center) if center else _pts).norm(dim=-1)
|
| 358 |
+
scale = torch.nanmedian(_norm, dim=1).values
|
| 359 |
+
return _center[:, None, :, :], scale[:, None, None, None]
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
def find_reciprocal_matches(P1, P2):
|
| 363 |
+
"""
|
| 364 |
+
returns 3 values:
|
| 365 |
+
1 - reciprocal_in_P2: a boolean array of size P2.shape[0], a "True" value indicates a match
|
| 366 |
+
2 - nn2_in_P1: a int array of size P2.shape[0], it contains the indexes of the closest points in P1
|
| 367 |
+
3 - reciprocal_in_P2.sum(): the number of matches
|
| 368 |
+
"""
|
| 369 |
+
tree1 = KDTree(P1)
|
| 370 |
+
tree2 = KDTree(P2)
|
| 371 |
+
|
| 372 |
+
_, nn1_in_P2 = tree2.query(P1, workers=8)
|
| 373 |
+
_, nn2_in_P1 = tree1.query(P2, workers=8)
|
| 374 |
+
|
| 375 |
+
reciprocal_in_P1 = (nn2_in_P1[nn1_in_P2] == np.arange(len(nn1_in_P2)))
|
| 376 |
+
reciprocal_in_P2 = (nn1_in_P2[nn2_in_P1] == np.arange(len(nn2_in_P1)))
|
| 377 |
+
assert reciprocal_in_P1.sum() == reciprocal_in_P2.sum()
|
| 378 |
+
return reciprocal_in_P2, nn2_in_P1, reciprocal_in_P2.sum()
|
| 379 |
+
|
| 380 |
+
|
| 381 |
+
def get_med_dist_between_poses(poses):
|
| 382 |
+
from scipy.spatial.distance import pdist
|
| 383 |
+
return np.median(pdist([p[:3, 3].detach().cpu().numpy() for p in poses]))
|
src/dataset/shims/load_shim.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import cv2
|
| 2 |
+
|
| 3 |
+
def imread_cv2(path, options=cv2.IMREAD_COLOR):
|
| 4 |
+
"""Open an image or a depthmap with opencv-python."""
|
| 5 |
+
if path.endswith((".exr", "EXR")):
|
| 6 |
+
options = cv2.IMREAD_ANYDEPTH
|
| 7 |
+
img = cv2.imread(path, options)
|
| 8 |
+
if img is None:
|
| 9 |
+
raise IOError(f"Could not load image={path} with {options=}")
|
| 10 |
+
if img.ndim == 3:
|
| 11 |
+
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
| 12 |
+
return img
|
src/dataset/shims/normalize_shim.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from einops import einsum, reduce, repeat
|
| 3 |
+
from jaxtyping import Float
|
| 4 |
+
from torch import Tensor
|
| 5 |
+
|
| 6 |
+
from ..types import BatchedExample
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def inverse_normalize_image(tensor, mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)):
|
| 10 |
+
mean = torch.as_tensor(mean, dtype=tensor.dtype, device=tensor.device).view(-1, 1, 1)
|
| 11 |
+
std = torch.as_tensor(std, dtype=tensor.dtype, device=tensor.device).view(-1, 1, 1)
|
| 12 |
+
return tensor * std + mean
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def normalize_image(tensor, mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)):
|
| 16 |
+
mean = torch.as_tensor(mean, dtype=tensor.dtype, device=tensor.device).view(-1, 1, 1)
|
| 17 |
+
std = torch.as_tensor(std, dtype=tensor.dtype, device=tensor.device).view(-1, 1, 1)
|
| 18 |
+
return (tensor - mean) / std
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def apply_normalize_shim(
|
| 22 |
+
batch: BatchedExample,
|
| 23 |
+
mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
|
| 24 |
+
std: tuple[float, float, float] = (0.5, 0.5, 0.5),
|
| 25 |
+
) -> BatchedExample:
|
| 26 |
+
batch["context"]["image"] = normalize_image(batch["context"]["image"], mean, std)
|
| 27 |
+
return batch
|
src/dataset/shims/patch_shim.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from ..types import BatchedExample, BatchedViews
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def apply_patch_shim_to_views(views: BatchedViews, patch_size: int) -> BatchedViews:
|
| 5 |
+
_, _, _, h, w = views["image"].shape
|
| 6 |
+
|
| 7 |
+
# Image size must be even so that naive center-cropping does not cause misalignment.
|
| 8 |
+
assert h % 2 == 0 and w % 2 == 0
|
| 9 |
+
|
| 10 |
+
h_new = (h // patch_size) * patch_size
|
| 11 |
+
row = (h - h_new) // 2
|
| 12 |
+
w_new = (w // patch_size) * patch_size
|
| 13 |
+
col = (w - w_new) // 2
|
| 14 |
+
|
| 15 |
+
# Center-crop the image.
|
| 16 |
+
image = views["image"][:, :, :, row : row + h_new, col : col + w_new]
|
| 17 |
+
|
| 18 |
+
# Adjust the intrinsics to account for the cropping.
|
| 19 |
+
intrinsics = views["intrinsics"].clone()
|
| 20 |
+
intrinsics[:, :, 0, 0] *= w / w_new # fx
|
| 21 |
+
intrinsics[:, :, 1, 1] *= h / h_new # fy
|
| 22 |
+
|
| 23 |
+
return {
|
| 24 |
+
**views,
|
| 25 |
+
"image": image,
|
| 26 |
+
"intrinsics": intrinsics,
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def apply_patch_shim(batch: BatchedExample, patch_size: int) -> BatchedExample:
|
| 31 |
+
"""Crop images in the batch so that their dimensions are cleanly divisible by the
|
| 32 |
+
specified patch size.
|
| 33 |
+
"""
|
| 34 |
+
return {
|
| 35 |
+
**batch,
|
| 36 |
+
"context": apply_patch_shim_to_views(batch["context"], patch_size),
|
| 37 |
+
"target": apply_patch_shim_to_views(batch["target"], patch_size),
|
| 38 |
+
}
|
src/dataset/types.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Callable, Literal, TypedDict
|
| 2 |
+
|
| 3 |
+
from jaxtyping import Float, Int64
|
| 4 |
+
from torch import Tensor
|
| 5 |
+
|
| 6 |
+
Stage = Literal["train", "val", "test"]
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
# The following types mainly exist to make type-hinted keys show up in VS Code. Some
|
| 10 |
+
# dimensions are annotated as "_" because either:
|
| 11 |
+
# 1. They're expected to change as part of a function call (e.g., resizing the dataset).
|
| 12 |
+
# 2. They're expected to vary within the same function call (e.g., the number of views,
|
| 13 |
+
# which differs between context and target BatchedViews).
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class BatchedViews(TypedDict, total=False):
|
| 17 |
+
extrinsics: Float[Tensor, "batch _ 4 4"] # batch view 4 4
|
| 18 |
+
intrinsics: Float[Tensor, "batch _ 3 3"] # batch view 3 3
|
| 19 |
+
image: Float[Tensor, "batch _ _ _ _"] # batch view channel height width
|
| 20 |
+
near: Float[Tensor, "batch _"] # batch view
|
| 21 |
+
far: Float[Tensor, "batch _"] # batch view
|
| 22 |
+
index: Int64[Tensor, "batch _"] # batch view
|
| 23 |
+
overlap: Float[Tensor, "batch _"] # batch view
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class BatchedExample(TypedDict, total=False):
|
| 27 |
+
target: BatchedViews
|
| 28 |
+
context: BatchedViews
|
| 29 |
+
scene: list[str]
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
class UnbatchedViews(TypedDict, total=False):
|
| 33 |
+
extrinsics: Float[Tensor, "_ 4 4"]
|
| 34 |
+
intrinsics: Float[Tensor, "_ 3 3"]
|
| 35 |
+
image: Float[Tensor, "_ 3 height width"]
|
| 36 |
+
near: Float[Tensor, " _"]
|
| 37 |
+
far: Float[Tensor, " _"]
|
| 38 |
+
index: Int64[Tensor, " _"]
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class UnbatchedExample(TypedDict, total=False):
|
| 42 |
+
target: UnbatchedViews
|
| 43 |
+
context: UnbatchedViews
|
| 44 |
+
scene: str
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
# A data shim modifies the example after it's been returned from the data loader.
|
| 48 |
+
DataShim = Callable[[BatchedExample], BatchedExample]
|
| 49 |
+
|
| 50 |
+
AnyExample = BatchedExample | UnbatchedExample
|
| 51 |
+
AnyViews = BatchedViews | UnbatchedViews
|
src/dataset/validation_wrapper.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Iterator, Optional
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from torch.utils.data import Dataset, IterableDataset
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class ValidationWrapper(Dataset):
|
| 8 |
+
"""Wraps a dataset so that PyTorch Lightning's validation step can be turned into a
|
| 9 |
+
visualization step.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
dataset: Dataset
|
| 13 |
+
dataset_iterator: Optional[Iterator]
|
| 14 |
+
length: int
|
| 15 |
+
|
| 16 |
+
def __init__(self, dataset: Dataset, length: int) -> None:
|
| 17 |
+
super().__init__()
|
| 18 |
+
self.dataset = dataset
|
| 19 |
+
self.length = length
|
| 20 |
+
self.dataset_iterator = None
|
| 21 |
+
|
| 22 |
+
def __len__(self):
|
| 23 |
+
return self.length
|
| 24 |
+
|
| 25 |
+
def __getitem__(self, index: tuple):
|
| 26 |
+
if isinstance(self.dataset, IterableDataset):
|
| 27 |
+
if self.dataset_iterator is None:
|
| 28 |
+
self.dataset_iterator = iter(self.dataset)
|
| 29 |
+
return next(self.dataset_iterator)
|
| 30 |
+
|
| 31 |
+
random_index = torch.randint(0, len(self.dataset), tuple())
|
| 32 |
+
random_context_num = torch.randint(2, self.dataset.view_sampler.num_context_views + 1, tuple())
|
| 33 |
+
# breakpoint()
|
| 34 |
+
return self.dataset[random_index.item(), random_context_num.item()]
|
src/dataset/view_sampler/__init__.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any
|
| 2 |
+
|
| 3 |
+
from ...misc.step_tracker import StepTracker
|
| 4 |
+
from ..types import Stage
|
| 5 |
+
from .view_sampler import ViewSampler
|
| 6 |
+
from .view_sampler_all import ViewSamplerAll, ViewSamplerAllCfg
|
| 7 |
+
from .view_sampler_arbitrary import ViewSamplerArbitrary, ViewSamplerArbitraryCfg
|
| 8 |
+
from .view_sampler_bounded import ViewSamplerBounded, ViewSamplerBoundedCfg
|
| 9 |
+
from .view_sampler_evaluation import ViewSamplerEvaluation, ViewSamplerEvaluationCfg
|
| 10 |
+
from .view_sampler_rank import ViewSamplerRank, ViewSamplerRankCfg
|
| 11 |
+
VIEW_SAMPLERS: dict[str, ViewSampler[Any]] = {
|
| 12 |
+
"all": ViewSamplerAll,
|
| 13 |
+
"arbitrary": ViewSamplerArbitrary,
|
| 14 |
+
"bounded": ViewSamplerBounded,
|
| 15 |
+
"evaluation": ViewSamplerEvaluation,
|
| 16 |
+
"rank": ViewSamplerRank,
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
ViewSamplerCfg = (
|
| 20 |
+
ViewSamplerArbitraryCfg
|
| 21 |
+
| ViewSamplerBoundedCfg
|
| 22 |
+
| ViewSamplerEvaluationCfg
|
| 23 |
+
| ViewSamplerAllCfg
|
| 24 |
+
| ViewSamplerRankCfg
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
def get_view_sampler(
|
| 28 |
+
cfg: ViewSamplerCfg,
|
| 29 |
+
stage: Stage,
|
| 30 |
+
overfit: bool,
|
| 31 |
+
cameras_are_circular: bool,
|
| 32 |
+
step_tracker: StepTracker | None,
|
| 33 |
+
) -> ViewSampler[Any]:
|
| 34 |
+
return VIEW_SAMPLERS[cfg.name](
|
| 35 |
+
cfg,
|
| 36 |
+
stage,
|
| 37 |
+
overfit,
|
| 38 |
+
cameras_are_circular,
|
| 39 |
+
step_tracker,
|
| 40 |
+
)
|
src/dataset/view_sampler/three_view_hack.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from jaxtyping import Int
|
| 3 |
+
from torch import Tensor
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def add_third_context_index(
|
| 7 |
+
indices: Int[Tensor, "*batch 2"]
|
| 8 |
+
) -> Int[Tensor, "*batch 3"]:
|
| 9 |
+
left, right = indices.unbind(dim=-1)
|
| 10 |
+
return torch.stack((left, (left + right) // 2, right), dim=-1)
|
src/dataset/view_sampler/view_sampler.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from abc import ABC, abstractmethod
|
| 2 |
+
from typing import Generic, TypeVar
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
from jaxtyping import Float, Int64
|
| 6 |
+
from torch import Tensor
|
| 7 |
+
|
| 8 |
+
from ...misc.step_tracker import StepTracker
|
| 9 |
+
from ..types import Stage
|
| 10 |
+
|
| 11 |
+
T = TypeVar("T")
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class ViewSampler(ABC, Generic[T]):
|
| 15 |
+
cfg: T
|
| 16 |
+
stage: Stage
|
| 17 |
+
is_overfitting: bool
|
| 18 |
+
cameras_are_circular: bool
|
| 19 |
+
step_tracker: StepTracker | None
|
| 20 |
+
|
| 21 |
+
def __init__(
|
| 22 |
+
self,
|
| 23 |
+
cfg: T,
|
| 24 |
+
stage: Stage,
|
| 25 |
+
is_overfitting: bool,
|
| 26 |
+
cameras_are_circular: bool,
|
| 27 |
+
step_tracker: StepTracker | None,
|
| 28 |
+
) -> None:
|
| 29 |
+
self.cfg = cfg
|
| 30 |
+
self.stage = stage
|
| 31 |
+
self.is_overfitting = is_overfitting
|
| 32 |
+
self.cameras_are_circular = cameras_are_circular
|
| 33 |
+
self.step_tracker = step_tracker
|
| 34 |
+
|
| 35 |
+
@abstractmethod
|
| 36 |
+
def sample(
|
| 37 |
+
self,
|
| 38 |
+
scene: str,
|
| 39 |
+
extrinsics: Float[Tensor, "view 4 4"],
|
| 40 |
+
intrinsics: Float[Tensor, "view 3 3"],
|
| 41 |
+
device: torch.device = torch.device("cpu"),
|
| 42 |
+
) -> tuple[
|
| 43 |
+
Int64[Tensor, " context_view"], # indices for context views
|
| 44 |
+
Int64[Tensor, " target_view"], # indices for target views
|
| 45 |
+
Float[Tensor, " overlap"], # overlap
|
| 46 |
+
]:
|
| 47 |
+
pass
|
| 48 |
+
|
| 49 |
+
@property
|
| 50 |
+
@abstractmethod
|
| 51 |
+
def num_target_views(self) -> int:
|
| 52 |
+
pass
|
| 53 |
+
|
| 54 |
+
@property
|
| 55 |
+
@abstractmethod
|
| 56 |
+
def num_context_views(self) -> int:
|
| 57 |
+
pass
|
| 58 |
+
|
| 59 |
+
@property
|
| 60 |
+
def global_step(self) -> int:
|
| 61 |
+
return 0 if self.step_tracker is None else self.step_tracker.get_step()
|
src/dataset/view_sampler/view_sampler_all.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass
|
| 2 |
+
from typing import Literal
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
from jaxtyping import Float, Int64
|
| 6 |
+
from torch import Tensor
|
| 7 |
+
|
| 8 |
+
from .view_sampler import ViewSampler
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
@dataclass
|
| 12 |
+
class ViewSamplerAllCfg:
|
| 13 |
+
name: Literal["all"]
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class ViewSamplerAll(ViewSampler[ViewSamplerAllCfg]):
|
| 17 |
+
def sample(
|
| 18 |
+
self,
|
| 19 |
+
scene: str,
|
| 20 |
+
extrinsics: Float[Tensor, "view 4 4"],
|
| 21 |
+
intrinsics: Float[Tensor, "view 3 3"],
|
| 22 |
+
device: torch.device = torch.device("cpu"),
|
| 23 |
+
) -> tuple[
|
| 24 |
+
Int64[Tensor, " context_view"], # indices for context views
|
| 25 |
+
Int64[Tensor, " target_view"], # indices for target views
|
| 26 |
+
]:
|
| 27 |
+
v, _, _ = extrinsics.shape
|
| 28 |
+
all_frames = torch.arange(v, device=device)
|
| 29 |
+
return all_frames, all_frames
|
| 30 |
+
|
| 31 |
+
@property
|
| 32 |
+
def num_context_views(self) -> int:
|
| 33 |
+
return 0
|
| 34 |
+
|
| 35 |
+
@property
|
| 36 |
+
def num_target_views(self) -> int:
|
| 37 |
+
return 0
|
src/dataset/view_sampler/view_sampler_arbitrary.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass
|
| 2 |
+
from typing import Literal
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
from jaxtyping import Float, Int64
|
| 6 |
+
from torch import Tensor
|
| 7 |
+
|
| 8 |
+
from .three_view_hack import add_third_context_index
|
| 9 |
+
from .view_sampler import ViewSampler
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@dataclass
|
| 13 |
+
class ViewSamplerArbitraryCfg:
|
| 14 |
+
name: Literal["arbitrary"]
|
| 15 |
+
num_context_views: int
|
| 16 |
+
num_target_views: int
|
| 17 |
+
context_views: list[int] | None
|
| 18 |
+
target_views: list[int] | None
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class ViewSamplerArbitrary(ViewSampler[ViewSamplerArbitraryCfg]):
|
| 22 |
+
def sample(
|
| 23 |
+
self,
|
| 24 |
+
scene: str,
|
| 25 |
+
extrinsics: Float[Tensor, "view 4 4"],
|
| 26 |
+
intrinsics: Float[Tensor, "view 3 3"],
|
| 27 |
+
device: torch.device = torch.device("cpu"),
|
| 28 |
+
) -> tuple[
|
| 29 |
+
Int64[Tensor, " context_view"], # indices for context views
|
| 30 |
+
Int64[Tensor, " target_view"], # indices for target views
|
| 31 |
+
Float[Tensor, " overlap"], # overlap
|
| 32 |
+
]:
|
| 33 |
+
"""Arbitrarily sample context and target views."""
|
| 34 |
+
num_views, _, _ = extrinsics.shape
|
| 35 |
+
|
| 36 |
+
index_context = torch.randint(
|
| 37 |
+
0,
|
| 38 |
+
num_views,
|
| 39 |
+
size=(self.cfg.num_context_views,),
|
| 40 |
+
device=device,
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
# Allow the context views to be fixed.
|
| 44 |
+
if self.cfg.context_views is not None:
|
| 45 |
+
index_context = torch.tensor(
|
| 46 |
+
self.cfg.context_views, dtype=torch.int64, device=device
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
if self.cfg.num_context_views == 3 and len(self.cfg.context_views) == 2:
|
| 50 |
+
index_context = add_third_context_index(index_context)
|
| 51 |
+
else:
|
| 52 |
+
assert len(self.cfg.context_views) == self.cfg.num_context_views
|
| 53 |
+
index_target = torch.randint(
|
| 54 |
+
0,
|
| 55 |
+
num_views,
|
| 56 |
+
size=(self.cfg.num_target_views,),
|
| 57 |
+
device=device,
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
# Allow the target views to be fixed.
|
| 61 |
+
if self.cfg.target_views is not None:
|
| 62 |
+
assert len(self.cfg.target_views) == self.cfg.num_target_views
|
| 63 |
+
index_target = torch.tensor(
|
| 64 |
+
self.cfg.target_views, dtype=torch.int64, device=device
|
| 65 |
+
)
|
| 66 |
+
|
| 67 |
+
overlap = torch.tensor([0.5], dtype=torch.float32, device=device) # dummy
|
| 68 |
+
|
| 69 |
+
return index_context, index_target, overlap
|
| 70 |
+
|
| 71 |
+
@property
|
| 72 |
+
def num_context_views(self) -> int:
|
| 73 |
+
return self.cfg.num_context_views
|
| 74 |
+
|
| 75 |
+
@property
|
| 76 |
+
def num_target_views(self) -> int:
|
| 77 |
+
return self.cfg.num_target_views
|
src/dataset/view_sampler/view_sampler_bounded.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass
|
| 2 |
+
from typing import Literal
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
from jaxtyping import Float, Int64
|
| 6 |
+
from torch import Tensor
|
| 7 |
+
|
| 8 |
+
from .view_sampler import ViewSampler
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
@dataclass
|
| 12 |
+
class ViewSamplerBoundedCfg:
|
| 13 |
+
name: Literal["bounded"]
|
| 14 |
+
num_context_views: int
|
| 15 |
+
num_target_views: int
|
| 16 |
+
min_distance_between_context_views: int
|
| 17 |
+
max_distance_between_context_views: int
|
| 18 |
+
min_distance_to_context_views: int
|
| 19 |
+
warm_up_steps: int
|
| 20 |
+
initial_min_distance_between_context_views: int
|
| 21 |
+
initial_max_distance_between_context_views: int
|
| 22 |
+
max_img_per_gpu: int
|
| 23 |
+
min_gap_multiplier: int
|
| 24 |
+
max_gap_multiplier: int
|
| 25 |
+
|
| 26 |
+
class ViewSamplerBounded(ViewSampler[ViewSamplerBoundedCfg]):
|
| 27 |
+
def schedule(self, initial: int, final: int) -> int:
|
| 28 |
+
fraction = self.global_step / self.cfg.warm_up_steps
|
| 29 |
+
return min(initial + int((final - initial) * fraction), final)
|
| 30 |
+
|
| 31 |
+
def sample(
|
| 32 |
+
self,
|
| 33 |
+
scene: str,
|
| 34 |
+
num_context_views: int,
|
| 35 |
+
extrinsics: Float[Tensor, "view 4 4"],
|
| 36 |
+
intrinsics: Float[Tensor, "view 3 3"],
|
| 37 |
+
device: torch.device = torch.device("cpu"),
|
| 38 |
+
) -> tuple[
|
| 39 |
+
Int64[Tensor, " context_view"], # indices for context views
|
| 40 |
+
Int64[Tensor, " target_view"], # indices for target views
|
| 41 |
+
Float[Tensor, " overlap"], # overlap
|
| 42 |
+
]:
|
| 43 |
+
num_views, _, _ = extrinsics.shape
|
| 44 |
+
|
| 45 |
+
# Compute the context view spacing based on the current global step.
|
| 46 |
+
if self.stage == "test":
|
| 47 |
+
# When testing, always use the full gap.
|
| 48 |
+
max_gap = self.cfg.max_distance_between_context_views
|
| 49 |
+
min_gap = self.cfg.max_distance_between_context_views
|
| 50 |
+
# elif self.cfg.warm_up_steps > 0:
|
| 51 |
+
# max_gap = self.schedule(
|
| 52 |
+
# self.cfg.initial_max_distance_between_context_views,
|
| 53 |
+
# self.cfg.max_distance_between_context_views,
|
| 54 |
+
# )
|
| 55 |
+
# min_gap = self.schedule(
|
| 56 |
+
# self.cfg.initial_min_distance_between_context_views,
|
| 57 |
+
# self.cfg.min_distance_between_context_views,
|
| 58 |
+
# )
|
| 59 |
+
# else:
|
| 60 |
+
# max_gap = self.cfg.max_distance_between_context_views
|
| 61 |
+
# min_gap = self.cfg.min_distance_between_context_views
|
| 62 |
+
|
| 63 |
+
min_gap, max_gap = self.num_ctxt_gap_mapping[num_context_views]
|
| 64 |
+
max_gap = min(max_gap, num_views-1)
|
| 65 |
+
# Pick the gap between the context views.
|
| 66 |
+
if not self.cameras_are_circular:
|
| 67 |
+
max_gap = min(num_views - 1, max_gap)
|
| 68 |
+
min_gap = max(2 * self.cfg.min_distance_to_context_views, min_gap)
|
| 69 |
+
if max_gap < min_gap:
|
| 70 |
+
raise ValueError("Example does not have enough frames!")
|
| 71 |
+
context_gap = torch.randint(
|
| 72 |
+
min_gap,
|
| 73 |
+
max_gap + 1,
|
| 74 |
+
size=tuple(),
|
| 75 |
+
device=device,
|
| 76 |
+
).item()
|
| 77 |
+
|
| 78 |
+
# Pick the left and right context indices.
|
| 79 |
+
index_context_left = torch.randint(
|
| 80 |
+
num_views if self.cameras_are_circular else num_views - context_gap,
|
| 81 |
+
size=tuple(),
|
| 82 |
+
device=device,
|
| 83 |
+
).item()
|
| 84 |
+
if self.stage == "test":
|
| 85 |
+
index_context_left = index_context_left * 0
|
| 86 |
+
index_context_right = index_context_left + context_gap
|
| 87 |
+
|
| 88 |
+
if self.is_overfitting:
|
| 89 |
+
index_context_left *= 0
|
| 90 |
+
index_context_right *= 0
|
| 91 |
+
index_context_right += max_gap
|
| 92 |
+
|
| 93 |
+
# Pick the target view indices.
|
| 94 |
+
if self.stage == "test":
|
| 95 |
+
# When testing, pick all.
|
| 96 |
+
index_target = torch.arange(
|
| 97 |
+
index_context_left,
|
| 98 |
+
index_context_right + 1,
|
| 99 |
+
device=device,
|
| 100 |
+
)
|
| 101 |
+
else:
|
| 102 |
+
# When training or validating (visualizing), pick at random.
|
| 103 |
+
index_target = torch.randint(
|
| 104 |
+
index_context_left + self.cfg.min_distance_to_context_views,
|
| 105 |
+
index_context_right + 1 - self.cfg.min_distance_to_context_views,
|
| 106 |
+
size=(self.cfg.num_target_views,),
|
| 107 |
+
device=device,
|
| 108 |
+
)
|
| 109 |
+
|
| 110 |
+
# Apply modulo for circular datasets.
|
| 111 |
+
if self.cameras_are_circular:
|
| 112 |
+
index_target %= num_views
|
| 113 |
+
index_context_right %= num_views
|
| 114 |
+
|
| 115 |
+
# If more than two context views are desired, pick extra context views between
|
| 116 |
+
# the left and right ones.
|
| 117 |
+
if num_context_views > 2:
|
| 118 |
+
num_extra_views = num_context_views - 2
|
| 119 |
+
extra_views = []
|
| 120 |
+
while len(set(extra_views)) != num_extra_views:
|
| 121 |
+
extra_views = torch.randint(
|
| 122 |
+
index_context_left + 1,
|
| 123 |
+
index_context_right,
|
| 124 |
+
(num_extra_views,),
|
| 125 |
+
).tolist()
|
| 126 |
+
else:
|
| 127 |
+
extra_views = []
|
| 128 |
+
|
| 129 |
+
overlap = torch.tensor([0.5], dtype=torch.float32, device=device) # dummy
|
| 130 |
+
|
| 131 |
+
return (
|
| 132 |
+
torch.tensor((index_context_left, *extra_views, index_context_right)),
|
| 133 |
+
index_target,
|
| 134 |
+
overlap
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
@property
|
| 138 |
+
def num_context_views(self) -> int:
|
| 139 |
+
return self.cfg.num_context_views
|
| 140 |
+
|
| 141 |
+
@property
|
| 142 |
+
def num_target_views(self) -> int:
|
| 143 |
+
return self.cfg.num_target_views
|
| 144 |
+
|
| 145 |
+
@property
|
| 146 |
+
def num_ctxt_gap_mapping(self) -> dict:
|
| 147 |
+
mapping = dict()
|
| 148 |
+
for num_ctxt in range(2, self.cfg.num_context_views + 1):
|
| 149 |
+
mapping[num_ctxt] = [min(num_ctxt * self.cfg.min_gap_multiplier, self.cfg.min_distance_between_context_views),
|
| 150 |
+
min(max(num_ctxt * self.cfg.max_gap_multiplier, num_ctxt ** 2), self.cfg.max_distance_between_context_views)]
|
| 151 |
+
return mapping
|
src/dataset/view_sampler/view_sampler_evaluation.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from dataclasses import dataclass
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from typing import Literal
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from dacite import Config, from_dict
|
| 8 |
+
from jaxtyping import Float, Int64
|
| 9 |
+
from torch import Tensor
|
| 10 |
+
|
| 11 |
+
from ...evaluation.evaluation_index_generator import IndexEntry
|
| 12 |
+
from ...global_cfg import get_cfg
|
| 13 |
+
from ...misc.step_tracker import StepTracker
|
| 14 |
+
from ..types import Stage
|
| 15 |
+
from .three_view_hack import add_third_context_index
|
| 16 |
+
from .view_sampler import ViewSampler
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@dataclass
|
| 20 |
+
class ViewSamplerEvaluationCfg:
|
| 21 |
+
name: Literal["evaluation"]
|
| 22 |
+
index_path: Path
|
| 23 |
+
num_context_views: int
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class ViewSamplerEvaluation(ViewSampler[ViewSamplerEvaluationCfg]):
|
| 27 |
+
index: dict[str, IndexEntry | None]
|
| 28 |
+
|
| 29 |
+
def __init__(
|
| 30 |
+
self,
|
| 31 |
+
cfg: ViewSamplerEvaluationCfg,
|
| 32 |
+
stage: Stage,
|
| 33 |
+
is_overfitting: bool,
|
| 34 |
+
cameras_are_circular: bool,
|
| 35 |
+
step_tracker: StepTracker | None,
|
| 36 |
+
) -> None:
|
| 37 |
+
super().__init__(cfg, stage, is_overfitting, cameras_are_circular, step_tracker)
|
| 38 |
+
|
| 39 |
+
dacite_config = Config(cast=[tuple])
|
| 40 |
+
with cfg.index_path.open("r") as f:
|
| 41 |
+
self.index = {
|
| 42 |
+
k: None if v is None else from_dict(IndexEntry, v, dacite_config)
|
| 43 |
+
for k, v in json.load(f).items()
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
def sample(
|
| 47 |
+
self,
|
| 48 |
+
scene: str,
|
| 49 |
+
extrinsics: Float[Tensor, "view 4 4"],
|
| 50 |
+
intrinsics: Float[Tensor, "view 3 3"],
|
| 51 |
+
device: torch.device = torch.device("cpu"),
|
| 52 |
+
**kwargs,
|
| 53 |
+
) -> tuple[
|
| 54 |
+
Int64[Tensor, " context_view"], # indices for context views
|
| 55 |
+
Int64[Tensor, " target_view"], # indices for target views
|
| 56 |
+
]:
|
| 57 |
+
entry = self.index.get(scene)
|
| 58 |
+
if entry is None:
|
| 59 |
+
raise ValueError(f"No indices available for scene {scene}.")
|
| 60 |
+
context_indices = torch.tensor(entry.context, dtype=torch.int64, device=device)
|
| 61 |
+
target_indices = torch.tensor(entry.target, dtype=torch.int64, device=device)
|
| 62 |
+
return context_indices, target_indices, torch.zeros(1)
|
| 63 |
+
|
| 64 |
+
@property
|
| 65 |
+
def num_context_views(self) -> int:
|
| 66 |
+
return 0
|
| 67 |
+
|
| 68 |
+
@property
|
| 69 |
+
def num_target_views(self) -> int:
|
| 70 |
+
return 0
|
src/dataset/view_sampler/view_sampler_rank.py
ADDED
|
@@ -0,0 +1,263 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
from dataclasses import dataclass
|
| 3 |
+
from typing import Literal
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import copy
|
| 7 |
+
from jaxtyping import Float, Int64
|
| 8 |
+
from torch import Tensor
|
| 9 |
+
import random
|
| 10 |
+
from .view_sampler import ViewSampler
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@dataclass
|
| 14 |
+
class ViewSamplerRankCfg:
|
| 15 |
+
name: Literal["rank"]
|
| 16 |
+
num_context_views: int # max number of context views
|
| 17 |
+
num_target_views: int
|
| 18 |
+
min_distance_between_context_views: int
|
| 19 |
+
max_distance_between_context_views: int
|
| 20 |
+
min_distance_to_context_views: int
|
| 21 |
+
warm_up_steps: int
|
| 22 |
+
initial_min_distance_between_context_views: int
|
| 23 |
+
initial_max_distance_between_context_views: int
|
| 24 |
+
max_img_per_gpu: int
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def rotation_angle(R1, R2):
|
| 28 |
+
# R1 and R2 are 3x3 rotation matrices
|
| 29 |
+
R = R1.T @ R2
|
| 30 |
+
# Numerical stability: clamp values into [-1,1]
|
| 31 |
+
val = (torch.trace(R) - 1) / 2
|
| 32 |
+
val = torch.clamp(val, -1.0, 1.0)
|
| 33 |
+
angle_rad = torch.acos(val)
|
| 34 |
+
angle_deg = angle_rad * 180 / torch.pi # Convert radians to degrees
|
| 35 |
+
return angle_deg
|
| 36 |
+
|
| 37 |
+
def extrinsic_distance(extrinsic1, extrinsic2, lambda_t=1.0):
|
| 38 |
+
R1, t1 = extrinsic1[:3, :3], extrinsic1[:3, 3]
|
| 39 |
+
R2, t2 = extrinsic2[:3, :3], extrinsic2[:3, 3]
|
| 40 |
+
rot_diff = rotation_angle(R1, R2) / 180
|
| 41 |
+
|
| 42 |
+
center_diff = torch.norm(t1 - t2)
|
| 43 |
+
return rot_diff + lambda_t * center_diff
|
| 44 |
+
|
| 45 |
+
def rotation_angle_batch(R1, R2):
|
| 46 |
+
# R1, R2: shape (N, 3, 3)
|
| 47 |
+
# We want a matrix of rotation angles for all pairs.
|
| 48 |
+
# We'll get R1^T R2 for each pair.
|
| 49 |
+
# Expand dimensions to broadcast:
|
| 50 |
+
# R1^T: (N,3,3) -> (N,1,3,3)
|
| 51 |
+
# R2: (N,3,3) -> (1,N,3,3)
|
| 52 |
+
R1_t = R1.transpose(-2, -1)[:, None, :, :] # shape (N,1,3,3)
|
| 53 |
+
R2_b = R2[None, :, :, :] # shape (1,N,3,3)
|
| 54 |
+
R_mult = torch.matmul(R1_t, R2_b) # shape (N,N,3,3)
|
| 55 |
+
# trace(R) for each pair
|
| 56 |
+
trace_vals = R_mult[..., 0, 0] + R_mult[..., 1, 1] + R_mult[..., 2, 2] # (N,N)
|
| 57 |
+
val = (trace_vals - 1) / 2
|
| 58 |
+
val = torch.clamp(val, -1.0, 1.0)
|
| 59 |
+
angle_rad = torch.acos(val)
|
| 60 |
+
angle_deg = angle_rad * 180 / torch.pi
|
| 61 |
+
return angle_deg / 180.0 # normalized rotation difference
|
| 62 |
+
|
| 63 |
+
def extrinsic_distance_batch(extrinsics, lambda_t=1.0):
|
| 64 |
+
# extrinsics: (N,4,4)
|
| 65 |
+
# Extract rotation and translation
|
| 66 |
+
R = extrinsics[:, :3, :3] # (N,3,3)
|
| 67 |
+
t = extrinsics[:, :3, 3] # (N,3)
|
| 68 |
+
# Compute all pairwise rotation differences
|
| 69 |
+
rot_diff = rotation_angle_batch(R, R) # (N,N)
|
| 70 |
+
# Compute all pairwise translation differences
|
| 71 |
+
# For t, shape (N,3). We want all pair differences: t[i] - t[j].
|
| 72 |
+
# t_i: (N,1,3), t_j: (1,N,3)
|
| 73 |
+
t_i = t[:, None, :] # (N,1,3)
|
| 74 |
+
t_j = t[None, :, :] # (1,N,3)
|
| 75 |
+
trans_diff = torch.norm(t_i - t_j, dim=2) # (N,N)
|
| 76 |
+
dists = rot_diff + lambda_t * trans_diff
|
| 77 |
+
return dists
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def compute_ranking(extrinsics, lambda_t=1.0, normalize=True, batched=True):
|
| 81 |
+
|
| 82 |
+
if normalize:
|
| 83 |
+
extrinsics = copy.deepcopy(extrinsics)
|
| 84 |
+
camera_center = copy.deepcopy(extrinsics[:, :3, 3])
|
| 85 |
+
camera_center_scale = torch.norm(camera_center, dim=1)
|
| 86 |
+
avg_scale = torch.mean(camera_center_scale)
|
| 87 |
+
extrinsics[:, :3, 3] = extrinsics[:, :3, 3] / avg_scale
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
if batched:
|
| 91 |
+
dists = extrinsic_distance_batch(extrinsics, lambda_t=lambda_t)
|
| 92 |
+
else:
|
| 93 |
+
N = extrinsics.shape[0]
|
| 94 |
+
dists = torch.zeros((N, N), device=extrinsics.device)
|
| 95 |
+
for i in range(N):
|
| 96 |
+
for j in range(N):
|
| 97 |
+
dists[i,j] = extrinsic_distance(extrinsics[i], extrinsics[j], lambda_t=lambda_t)
|
| 98 |
+
ranking = torch.argsort(dists, dim=1)
|
| 99 |
+
return ranking, dists
|
| 100 |
+
|
| 101 |
+
# class ViewSamplerRank(ViewSampler[ViewSamplerRankCfg]):
|
| 102 |
+
# def schedule(self, initial: int, final: int) -> int:
|
| 103 |
+
# fraction = self.global_step / self.cfg.warm_up_steps
|
| 104 |
+
# return min(initial + int((final - initial) * fraction), final)
|
| 105 |
+
|
| 106 |
+
# def sample(
|
| 107 |
+
# self,
|
| 108 |
+
# scene: str,
|
| 109 |
+
# extrinsics: Float[Tensor, "view 4 4"],
|
| 110 |
+
# intrinsics: Float[Tensor, "view 3 3"],
|
| 111 |
+
# device: torch.device = torch.device("cpu"),
|
| 112 |
+
# ) -> tuple[
|
| 113 |
+
# Int64[Tensor, " context_view"], # indices for context views
|
| 114 |
+
# Int64[Tensor, " target_view"], # indices for target views
|
| 115 |
+
# Float[Tensor, " overlap"], # overlap
|
| 116 |
+
# ]:
|
| 117 |
+
# num_views, _, _ = extrinsics.shape
|
| 118 |
+
# # breakpoint()
|
| 119 |
+
# # Compute the context view spacing based on the current global step.
|
| 120 |
+
# ranking, dists = compute_ranking(extrinsics, lambda_t=1.0, normalize=True, batched=True)
|
| 121 |
+
# reference_view = random.sample(range(num_views), 1)[0]
|
| 122 |
+
|
| 123 |
+
# refview_ranking = ranking[reference_view]
|
| 124 |
+
# # if self.cfg.warm_up_steps > 0:
|
| 125 |
+
# # max_gap = self.schedule(
|
| 126 |
+
# # self.cfg.initial_max_distance_between_context_views,
|
| 127 |
+
# # self.cfg.max_distance_between_context_views,
|
| 128 |
+
# # )
|
| 129 |
+
# # min_gap = self.schedule(
|
| 130 |
+
# # self.cfg.initial_min_distance_between_context_views,
|
| 131 |
+
# # self.cfg.min_distance_between_context_views,
|
| 132 |
+
# # )
|
| 133 |
+
# # else:
|
| 134 |
+
# max_gap = self.cfg.max_distance_between_context_views
|
| 135 |
+
# min_gap = self.cfg.min_distance_between_context_views
|
| 136 |
+
|
| 137 |
+
# index_context_left = reference_view
|
| 138 |
+
# rightmost_index = random.sample(range(min_gap, max_gap + 1), 1)[0] + 1
|
| 139 |
+
# index_context_right = refview_ranking[rightmost_index].item()
|
| 140 |
+
|
| 141 |
+
# middle_indices = refview_ranking[1: rightmost_index].tolist()
|
| 142 |
+
# index_target = random.sample(middle_indices, self.num_target_views)
|
| 143 |
+
|
| 144 |
+
# remaining_indices = [idx for idx in middle_indices if idx not in index_target]
|
| 145 |
+
|
| 146 |
+
# # Sample extra context views if needed
|
| 147 |
+
# extra_views = []
|
| 148 |
+
# num_extra_views = self.num_context_views - 2 # subtract left and right context views
|
| 149 |
+
# if num_extra_views > 0 and remaining_indices:
|
| 150 |
+
# extra_views = random.sample(remaining_indices, min(num_extra_views, len(remaining_indices)))
|
| 151 |
+
# else:
|
| 152 |
+
# extra_views = []
|
| 153 |
+
|
| 154 |
+
# overlap = torch.zeros(1)
|
| 155 |
+
|
| 156 |
+
# return (
|
| 157 |
+
# torch.tensor((index_context_left, *extra_views, index_context_right)),
|
| 158 |
+
# torch.tensor(index_target),
|
| 159 |
+
# overlap
|
| 160 |
+
# )
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
# @property
|
| 164 |
+
# def num_context_views(self) -> int:
|
| 165 |
+
# return self.cfg.num_context_views
|
| 166 |
+
|
| 167 |
+
# @property
|
| 168 |
+
# def num_target_views(self) -> int:
|
| 169 |
+
# return self.cfg.num_target_views
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
class ViewSamplerRank(ViewSampler[ViewSamplerRankCfg]):
|
| 173 |
+
|
| 174 |
+
def sample(
|
| 175 |
+
self,
|
| 176 |
+
scene: str,
|
| 177 |
+
num_context_views: int,
|
| 178 |
+
extrinsics: Float[Tensor, "view 4 4"],
|
| 179 |
+
intrinsics: Float[Tensor, "view 3 3"],
|
| 180 |
+
device: torch.device = torch.device("cpu"),
|
| 181 |
+
) -> tuple[
|
| 182 |
+
Int64[Tensor, " context_view"], # indices for context views
|
| 183 |
+
Int64[Tensor, " target_view"], # indices for target views
|
| 184 |
+
Float[Tensor, " overlap"], # overlap
|
| 185 |
+
]:
|
| 186 |
+
num_views, _, _ = extrinsics.shape
|
| 187 |
+
# breakpoint()
|
| 188 |
+
extrinsics = extrinsics.clone()
|
| 189 |
+
# Compute the context view spacing based on the current global step.
|
| 190 |
+
ranking, dists = compute_ranking(extrinsics, lambda_t=1.0, normalize=True, batched=True)
|
| 191 |
+
reference_view = random.sample(range(num_views), 1)[0]
|
| 192 |
+
|
| 193 |
+
refview_ranking = ranking[reference_view]
|
| 194 |
+
# if self.cfg.warm_up_steps > 0:
|
| 195 |
+
# max_gap = self.schedule(
|
| 196 |
+
# self.cfg.initial_max_distance_between_context_views,
|
| 197 |
+
# self.cfg.max_distance_between_context_views,
|
| 198 |
+
# )
|
| 199 |
+
# min_gap = self.schedule(
|
| 200 |
+
# self.cfg.initial_min_distance_between_context_views,
|
| 201 |
+
# self.cfg.min_distance_between_context_views,
|
| 202 |
+
# )
|
| 203 |
+
# else:
|
| 204 |
+
min_gap, max_gap = self.num_ctxt_gap_mapping[num_context_views]
|
| 205 |
+
|
| 206 |
+
# min_gap = self.cfg.min_distance_between_context_views
|
| 207 |
+
# max_gap = self.cfg.max_distance_between_context_views
|
| 208 |
+
|
| 209 |
+
max_gap = min(max_gap, num_views-1)
|
| 210 |
+
# print(f"num_context_views: {num_context_views}, min_gap: {min_gap}, max_gap: {max_gap}")
|
| 211 |
+
index_context_left = reference_view
|
| 212 |
+
rightmost_index = random.sample(range(min_gap, max_gap + 1), 1)[0]
|
| 213 |
+
|
| 214 |
+
# #! hard code for visualization
|
| 215 |
+
# rightmost_index = self.cfg.max_distance_between_context_views
|
| 216 |
+
|
| 217 |
+
index_context_right = refview_ranking[rightmost_index].item()
|
| 218 |
+
|
| 219 |
+
middle_indices = refview_ranking[1: rightmost_index].tolist()
|
| 220 |
+
index_target = random.sample(middle_indices, self.num_target_views)
|
| 221 |
+
|
| 222 |
+
remaining_indices = [idx for idx in middle_indices if idx not in index_target]
|
| 223 |
+
|
| 224 |
+
# Sample extra context views if needed
|
| 225 |
+
extra_views = []
|
| 226 |
+
num_extra_views = num_context_views - 2 # subtract left and right context views
|
| 227 |
+
if num_extra_views > 0 and remaining_indices:
|
| 228 |
+
extra_views = random.sample(remaining_indices, min(num_extra_views, len(remaining_indices)))
|
| 229 |
+
else:
|
| 230 |
+
extra_views = []
|
| 231 |
+
|
| 232 |
+
overlap = torch.zeros(1)
|
| 233 |
+
|
| 234 |
+
return (
|
| 235 |
+
torch.tensor((index_context_left, *extra_views, index_context_right)),
|
| 236 |
+
torch.tensor(index_target),
|
| 237 |
+
overlap
|
| 238 |
+
)
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
@property
|
| 242 |
+
def num_context_views(self) -> int:
|
| 243 |
+
return self.cfg.num_context_views
|
| 244 |
+
|
| 245 |
+
@property
|
| 246 |
+
def num_target_views(self) -> int:
|
| 247 |
+
return self.cfg.num_target_views
|
| 248 |
+
|
| 249 |
+
@property
|
| 250 |
+
def num_ctxt_gap_mapping_target(self) -> dict:
|
| 251 |
+
mapping = dict()
|
| 252 |
+
for num_ctxt in range(2, self.cfg.num_context_views + 1):
|
| 253 |
+
mapping[num_ctxt] = [max(num_ctxt * 2, self.cfg.num_target_views + num_ctxt), max(self.cfg.num_target_views + num_ctxt, min(num_ctxt ** 2, self.cfg.max_distance_between_context_views))]
|
| 254 |
+
return mapping
|
| 255 |
+
|
| 256 |
+
@property
|
| 257 |
+
def num_ctxt_gap_mapping(self) -> dict:
|
| 258 |
+
mapping = dict()
|
| 259 |
+
for num_ctxt in range(2, self.cfg.num_context_views + 1):
|
| 260 |
+
mapping[num_ctxt] = [min(num_ctxt * 3, self.cfg.min_distance_between_context_views), min(max(num_ctxt * 5, num_ctxt ** 2), self.cfg.max_distance_between_context_views)]
|
| 261 |
+
return mapping
|
| 262 |
+
|
| 263 |
+
|
src/eval_nvs.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
import sys
|
| 4 |
+
import json
|
| 5 |
+
import gzip
|
| 6 |
+
import argparse
|
| 7 |
+
import numpy as np
|
| 8 |
+
from PIL import Image
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
import torch.nn as nn
|
| 12 |
+
import torchvision
|
| 13 |
+
from einops import rearrange
|
| 14 |
+
|
| 15 |
+
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| 16 |
+
|
| 17 |
+
from src.evaluation.metrics import compute_lpips, compute_psnr, compute_ssim
|
| 18 |
+
from misc.image_io import save_image, save_interpolated_video
|
| 19 |
+
from src.utils.image import process_image
|
| 20 |
+
|
| 21 |
+
from src.model.model.anysplat import AnySplat
|
| 22 |
+
from src.model.encoder.vggt.utils.pose_enc import pose_encoding_to_extri_intri
|
| 23 |
+
|
| 24 |
+
def setup_args():
|
| 25 |
+
"""Set up command-line arguments for the eval NVS script."""
|
| 26 |
+
parser = argparse.ArgumentParser(description='Test AnySplat on NVS evaluation')
|
| 27 |
+
parser.add_argument('--data_dir', type=str, required=True, help='Path to NVS dataset')
|
| 28 |
+
parser.add_argument('--llffhold', type=int, default=8, help='LLFF holdout')
|
| 29 |
+
parser.add_argument('--output_path', type=str, default="outputs/nvs", help='Path to output directory')
|
| 30 |
+
return parser.parse_args()
|
| 31 |
+
|
| 32 |
+
def compute_metrics(pred_image, image):
|
| 33 |
+
psnr = compute_psnr(pred_image, image)
|
| 34 |
+
ssim = compute_ssim(pred_image, image)
|
| 35 |
+
lpips = compute_lpips(pred_image, image)
|
| 36 |
+
return psnr, ssim, lpips
|
| 37 |
+
|
| 38 |
+
def evaluate(args: argparse.Namespace):
|
| 39 |
+
model = AnySplat.from_pretrained("lhjiang/anysplat")
|
| 40 |
+
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 41 |
+
model.to(device)
|
| 42 |
+
model.eval()
|
| 43 |
+
for param in model.parameters():
|
| 44 |
+
param.requires_grad = False
|
| 45 |
+
|
| 46 |
+
os.makedirs(args.output_path, exist_ok=True)
|
| 47 |
+
|
| 48 |
+
# load images
|
| 49 |
+
image_folder = args.data_dir
|
| 50 |
+
image_names = sorted([os.path.join(image_folder, f) for f in os.listdir(image_folder) if f.lower().endswith(('.png', '.jpg', '.jpeg'))])
|
| 51 |
+
images = [process_image(img_path) for img_path in image_names]
|
| 52 |
+
ctx_indices = [idx for idx, name in enumerate(image_names) if idx % args.llffhold != 0]
|
| 53 |
+
tgt_indices = [idx for idx, name in enumerate(image_names) if idx % args.llffhold == 0]
|
| 54 |
+
|
| 55 |
+
ctx_images = torch.stack([images[i] for i in ctx_indices], dim=0).unsqueeze(0).to(device)
|
| 56 |
+
tgt_images = torch.stack([images[i] for i in tgt_indices], dim=0).unsqueeze(0).to(device)
|
| 57 |
+
ctx_images = (ctx_images+1)*0.5
|
| 58 |
+
tgt_images = (tgt_images+1)*0.5
|
| 59 |
+
b, v, _, h, w = tgt_images.shape
|
| 60 |
+
|
| 61 |
+
# run inference
|
| 62 |
+
encoder_output = model.encoder(
|
| 63 |
+
ctx_images,
|
| 64 |
+
global_step=0,
|
| 65 |
+
visualization_dump={},
|
| 66 |
+
)
|
| 67 |
+
gaussians, pred_context_pose = encoder_output.gaussians, encoder_output.pred_context_pose
|
| 68 |
+
|
| 69 |
+
num_context_view = ctx_images.shape[1]
|
| 70 |
+
vggt_input_image = torch.cat((ctx_images, tgt_images), dim=1).to(torch.bfloat16)
|
| 71 |
+
with torch.no_grad(), torch.cuda.amp.autocast(enabled=False, dtype=torch.bfloat16):
|
| 72 |
+
aggregated_tokens_list, patch_start_idx = model.encoder.aggregator(vggt_input_image, intermediate_layer_idx=model.encoder.cfg.intermediate_layer_idx)
|
| 73 |
+
with torch.cuda.amp.autocast(enabled=False):
|
| 74 |
+
fp32_tokens = [token.float() for token in aggregated_tokens_list]
|
| 75 |
+
pred_all_pose_enc = model.encoder.camera_head(fp32_tokens)[-1]
|
| 76 |
+
pred_all_extrinsic, pred_all_intrinsic = pose_encoding_to_extri_intri(pred_all_pose_enc, vggt_input_image.shape[-2:])
|
| 77 |
+
|
| 78 |
+
extrinsic_padding = torch.tensor([0, 0, 0, 1], device=pred_all_extrinsic.device, dtype=pred_all_extrinsic.dtype).view(1, 1, 1, 4).repeat(b, vggt_input_image.shape[1], 1, 1)
|
| 79 |
+
pred_all_extrinsic = torch.cat([pred_all_extrinsic, extrinsic_padding], dim=2).inverse()
|
| 80 |
+
|
| 81 |
+
pred_all_intrinsic[:, :, 0] = pred_all_intrinsic[:, :, 0] / w
|
| 82 |
+
pred_all_intrinsic[:, :, 1] = pred_all_intrinsic[:, :, 1] / h
|
| 83 |
+
pred_all_context_extrinsic, pred_all_target_extrinsic = pred_all_extrinsic[:, :num_context_view], pred_all_extrinsic[:, num_context_view:]
|
| 84 |
+
pred_all_context_intrinsic, pred_all_target_intrinsic = pred_all_intrinsic[:, :num_context_view], pred_all_intrinsic[:, num_context_view:]
|
| 85 |
+
|
| 86 |
+
scale_factor = pred_context_pose['extrinsic'][:, :, :3, 3].mean() / pred_all_context_extrinsic[:, :, :3, 3].mean()
|
| 87 |
+
pred_all_target_extrinsic[..., :3, 3] = pred_all_target_extrinsic[..., :3, 3] * scale_factor
|
| 88 |
+
pred_all_context_extrinsic[..., :3, 3] = pred_all_context_extrinsic[..., :3, 3] * scale_factor
|
| 89 |
+
print("scale_factor:", scale_factor)
|
| 90 |
+
|
| 91 |
+
output = model.decoder.forward(
|
| 92 |
+
gaussians,
|
| 93 |
+
pred_all_target_extrinsic,
|
| 94 |
+
pred_all_target_intrinsic.float(),
|
| 95 |
+
torch.ones(1, v, device=device) * 0.01,
|
| 96 |
+
torch.ones(1, v, device=device) * 100,
|
| 97 |
+
(h, w)
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
save_interpolated_video(pred_all_context_extrinsic, pred_all_context_intrinsic, b, h, w, gaussians, args.output_path, model.decoder)
|
| 101 |
+
|
| 102 |
+
# Save original images
|
| 103 |
+
save_path = Path(args.output_path)
|
| 104 |
+
# os.makedirs(save_path, exist_ok=True)
|
| 105 |
+
for idx, (gt_image, pred_image) in enumerate(zip(tgt_images[0], output.color[0])):
|
| 106 |
+
save_image(gt_image, save_path / "gt" / f"{idx:0>6}.jpg")
|
| 107 |
+
save_image(pred_image, save_path / "pred" / f"{idx:0>6}.jpg")
|
| 108 |
+
|
| 109 |
+
# compute metrics
|
| 110 |
+
psnr, ssim, lpips = compute_metrics(output.color[0], tgt_images[0])
|
| 111 |
+
print(f"PSNR: {psnr.mean():.2f}, SSIM: {ssim.mean():.3f}, LPIPS: {lpips.mean():.3f}")
|
| 112 |
+
|
| 113 |
+
if __name__ == "__main__":
|
| 114 |
+
args = setup_args()
|
| 115 |
+
evaluate(args)
|
src/eval_pose.py
ADDED
|
@@ -0,0 +1,339 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import sys
|
| 3 |
+
import json
|
| 4 |
+
import gzip
|
| 5 |
+
import argparse
|
| 6 |
+
import numpy as np
|
| 7 |
+
from PIL import Image
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
import torch.nn as nn
|
| 11 |
+
import torchvision
|
| 12 |
+
from einops import rearrange
|
| 13 |
+
from lpips import LPIPS
|
| 14 |
+
|
| 15 |
+
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| 16 |
+
|
| 17 |
+
from src.model.model.anysplat import AnySplat
|
| 18 |
+
from src.model.encoder.vggt.utils.pose_enc import pose_encoding_to_extri_intri
|
| 19 |
+
from src.model.encoder.vggt.utils.load_fn import load_and_preprocess_images
|
| 20 |
+
from src.utils.pose import align_to_first_camera, calculate_auc_np, convert_pt3d_RT_to_opencv, se3_to_relative_pose_error
|
| 21 |
+
from src.misc.cam_utils import camera_normalization, pose_auc, rotation_6d_to_matrix, update_pose, get_pnp_pose
|
| 22 |
+
|
| 23 |
+
def setup_args():
|
| 24 |
+
"""Set up command-line arguments for the CO3D evaluation script."""
|
| 25 |
+
parser = argparse.ArgumentParser(description='Test AnySplat on CO3D dataset')
|
| 26 |
+
parser.add_argument('--debug', action='store_true', help='Enable debug mode (only test on specific category)')
|
| 27 |
+
parser.add_argument('--use_ba', action='store_true', default=False, help='Enable bundle adjustment')
|
| 28 |
+
parser.add_argument('--fast_eval', action='store_true', default=False, help='Only evaluate 10 sequences per category')
|
| 29 |
+
parser.add_argument('--min_num_images', type=int, default=50, help='Minimum number of images for a sequence')
|
| 30 |
+
parser.add_argument('--num_frames', type=int, default=10, help='Number of frames to use for testing')
|
| 31 |
+
parser.add_argument('--co3d_dir', type=str, required=True, help='Path to CO3D dataset')
|
| 32 |
+
parser.add_argument('--co3d_anno_dir', type=str, required=True, help='Path to CO3D annotations')
|
| 33 |
+
parser.add_argument('--seed', type=int, default=0, help='Random seed for reproducibility')
|
| 34 |
+
return parser.parse_args()
|
| 35 |
+
|
| 36 |
+
lpips = LPIPS(net="vgg")
|
| 37 |
+
|
| 38 |
+
def rendering_loss(pred_image, image):
|
| 39 |
+
lpips_loss = lpips.forward(rearrange(pred_image, "b v c h w -> (b v) c h w"), rearrange(image, "b v c h w -> (b v) c h w"), normalize=True)
|
| 40 |
+
delta = pred_image - (image + 1) / 2
|
| 41 |
+
mse_loss = (delta**2).mean()
|
| 42 |
+
return mse_loss + 0.05 * lpips_loss.mean()
|
| 43 |
+
|
| 44 |
+
def process_sequence(model, seq_name, seq_data, category, co3d_dir, min_num_images, num_frames, use_ba, device, dtype):
|
| 45 |
+
"""
|
| 46 |
+
Process a single sequence and compute pose errors.
|
| 47 |
+
|
| 48 |
+
Args:
|
| 49 |
+
model: AnySplat model
|
| 50 |
+
seq_name: Sequence name
|
| 51 |
+
seq_data: Sequence data
|
| 52 |
+
category: Category name
|
| 53 |
+
co3d_dir: CO3D dataset directory
|
| 54 |
+
min_num_images: Minimum number of images required
|
| 55 |
+
num_frames: Number of frames to sample
|
| 56 |
+
use_ba: Whether to use bundle adjustment
|
| 57 |
+
device: Device to run on
|
| 58 |
+
dtype: Data type for model inference
|
| 59 |
+
|
| 60 |
+
Returns:
|
| 61 |
+
rError: Rotation errors
|
| 62 |
+
tError: Translation errors
|
| 63 |
+
"""
|
| 64 |
+
if len(seq_data) < min_num_images:
|
| 65 |
+
return None, None
|
| 66 |
+
|
| 67 |
+
metadata = []
|
| 68 |
+
for data in seq_data:
|
| 69 |
+
# Make sure translations are not ridiculous
|
| 70 |
+
if data["T"][0] + data["T"][1] + data["T"][2] > 1e5:
|
| 71 |
+
return None, None
|
| 72 |
+
|
| 73 |
+
extri_opencv = convert_pt3d_RT_to_opencv(data["R"], data["T"])
|
| 74 |
+
metadata.append({
|
| 75 |
+
"filepath": data["filepath"],
|
| 76 |
+
"extri": extri_opencv,
|
| 77 |
+
})
|
| 78 |
+
|
| 79 |
+
ids = np.random.choice(len(metadata), num_frames, replace=False)
|
| 80 |
+
image_names = [os.path.join(co3d_dir, metadata[i]["filepath"]) for i in ids]
|
| 81 |
+
gt_extri = [np.array(metadata[i]["extri"]) for i in ids]
|
| 82 |
+
gt_extri = np.stack(gt_extri, axis=0)
|
| 83 |
+
|
| 84 |
+
max_size = max(Image.open(image_names[0]).size)
|
| 85 |
+
if max_size < 448:
|
| 86 |
+
return None, None
|
| 87 |
+
images = load_and_preprocess_images(image_names)[None].to(device)
|
| 88 |
+
|
| 89 |
+
batch = {
|
| 90 |
+
"context": {
|
| 91 |
+
"image": images*2.0-1,
|
| 92 |
+
"image_names": image_names,
|
| 93 |
+
"index": ids,
|
| 94 |
+
},
|
| 95 |
+
"scene": "co3d"
|
| 96 |
+
}
|
| 97 |
+
|
| 98 |
+
if use_ba:
|
| 99 |
+
try:
|
| 100 |
+
encoder_output = model.encoder(
|
| 101 |
+
batch,
|
| 102 |
+
global_step=0,
|
| 103 |
+
visualization_dump={},
|
| 104 |
+
)
|
| 105 |
+
gaussians, pred_context_pose = encoder_output.gaussians, encoder_output.pred_context_pose
|
| 106 |
+
pred_extrinsic = pred_context_pose['extrinsic']
|
| 107 |
+
pred_intrinsic = pred_context_pose['intrinsic']
|
| 108 |
+
# rendering ba
|
| 109 |
+
b, v, _, h, w = images.shape
|
| 110 |
+
with torch.set_grad_enabled(True), torch.cuda.amp.autocast(enabled=False, dtype=torch.float32):
|
| 111 |
+
cam_rot_delta = nn.Parameter(torch.zeros([b, v, 6], requires_grad=True, device=pred_extrinsic.device, dtype=torch.float32))
|
| 112 |
+
cam_trans_delta = nn.Parameter(torch.zeros([b, v, 3], requires_grad=True, device=pred_extrinsic.device, dtype=torch.float32))
|
| 113 |
+
opt_params = []
|
| 114 |
+
model.register_buffer("identity", torch.tensor([1.0, 0.0, 0.0, 0.0, 1.0, 0.0], dtype=torch.float32).to(pred_extrinsic.device))
|
| 115 |
+
opt_params.append(
|
| 116 |
+
{
|
| 117 |
+
"params": [cam_rot_delta],
|
| 118 |
+
"lr": 0.005,
|
| 119 |
+
}
|
| 120 |
+
)
|
| 121 |
+
opt_params.append(
|
| 122 |
+
{
|
| 123 |
+
"params": [cam_trans_delta],
|
| 124 |
+
"lr": 0.005,
|
| 125 |
+
}
|
| 126 |
+
)
|
| 127 |
+
pose_optimizer = torch.optim.Adam(opt_params)
|
| 128 |
+
extrinsics = pred_extrinsic.clone().float()
|
| 129 |
+
|
| 130 |
+
for i in range(100):
|
| 131 |
+
pose_optimizer.zero_grad()
|
| 132 |
+
dx, drot = cam_trans_delta, cam_rot_delta
|
| 133 |
+
rot = rotation_6d_to_matrix(
|
| 134 |
+
drot + model.identity.expand(b, v, -1)
|
| 135 |
+
) # (..., 3, 3)
|
| 136 |
+
|
| 137 |
+
transform = torch.eye(4, device=extrinsics.device).repeat((b, v, 1, 1))
|
| 138 |
+
transform[..., :3, :3] = rot
|
| 139 |
+
transform[..., :3, 3] = dx
|
| 140 |
+
|
| 141 |
+
new_extrinsics = torch.matmul(extrinsics, transform)
|
| 142 |
+
# breakpoint()
|
| 143 |
+
output = model.decoder.forward(
|
| 144 |
+
gaussians,
|
| 145 |
+
new_extrinsics,
|
| 146 |
+
pred_intrinsic.float(),
|
| 147 |
+
0.1,
|
| 148 |
+
100.0,
|
| 149 |
+
(h, w),
|
| 150 |
+
# cam_rot_delta=cam_rot_delta,
|
| 151 |
+
# cam_trans_delta=cam_trans_delta,
|
| 152 |
+
)
|
| 153 |
+
# export_ply(gaussians.means[0], gaussians.scales[0], gaussians.rotations[0], gaussians.harmonics[0], gaussians.opacities[0], Path(f"gaussians_co3d.ply"))
|
| 154 |
+
rendering_loss = rendering_loss(output.color, images*2.0-1)
|
| 155 |
+
torchvision.utils.save_image(output.color[0], f"outputs/vis/output_co3d_{i}.png")
|
| 156 |
+
print(f"Rendering loss: {rendering_loss.item()}")
|
| 157 |
+
# print(f"Rendering loss: {rendering_loss.item()}")
|
| 158 |
+
|
| 159 |
+
rendering_loss.backward()
|
| 160 |
+
pose_optimizer.step()
|
| 161 |
+
torchvision.utils.save_image(images[0], f"outputs/vis/gt_co3d.png")
|
| 162 |
+
pred_extrinsic = new_extrinsics.inverse()[0][:,:-1,:]
|
| 163 |
+
|
| 164 |
+
except Exception as e:
|
| 165 |
+
print(f"BA failed with error: {e}. Falling back to standard VGGT inference.")
|
| 166 |
+
with torch.no_grad(), torch.cuda.amp.autocast(dtype=dtype):
|
| 167 |
+
aggregated_tokens_list, patch_start_idx = model.encoder.aggregator(images, intermediate_layer_idx=model.encoder.cfg.intermediate_layer_idx)
|
| 168 |
+
with torch.cuda.amp.autocast(dtype=torch.float32):
|
| 169 |
+
fp32_tokens = [token.float() for token in aggregated_tokens_list]
|
| 170 |
+
pred_all_pose_enc = model.encoder.camera_head(fp32_tokens)[-1]
|
| 171 |
+
pred_all_extrinsic, pred_all_intrinsic = pose_encoding_to_extri_intri(pred_all_pose_enc, images.shape[-2:])
|
| 172 |
+
pred_extrinsic = pred_all_extrinsic[0]
|
| 173 |
+
else:
|
| 174 |
+
with torch.no_grad(), torch.cuda.amp.autocast(dtype=dtype):
|
| 175 |
+
aggregated_tokens_list, patch_start_idx = model.encoder.aggregator(images, intermediate_layer_idx=model.encoder.cfg.intermediate_layer_idx)
|
| 176 |
+
with torch.cuda.amp.autocast(dtype=torch.float32):
|
| 177 |
+
fp32_tokens = [token.float() for token in aggregated_tokens_list]
|
| 178 |
+
pred_all_pose_enc = model.encoder.camera_head(fp32_tokens)[-1]
|
| 179 |
+
pred_all_extrinsic, pred_all_intrinsic = pose_encoding_to_extri_intri(pred_all_pose_enc, images.shape[-2:])
|
| 180 |
+
pred_extrinsic = pred_all_extrinsic[0]
|
| 181 |
+
|
| 182 |
+
with torch.cuda.amp.autocast(dtype=torch.float32):
|
| 183 |
+
gt_extrinsic = torch.from_numpy(gt_extri).to(device)
|
| 184 |
+
add_row = torch.tensor([0, 0, 0, 1], device=device).expand(pred_extrinsic.size(0), 1, 4)
|
| 185 |
+
|
| 186 |
+
pred_se3 = torch.cat((pred_extrinsic, add_row), dim=1)
|
| 187 |
+
gt_se3 = torch.cat((gt_extrinsic, add_row), dim=1)
|
| 188 |
+
|
| 189 |
+
# Set the coordinate of the first camera as the coordinate of the world
|
| 190 |
+
# NOTE: DO NOT REMOVE THIS UNLESS YOU KNOW WHAT YOU ARE DOING
|
| 191 |
+
# pred_se3 = align_to_first_camera(pred_se3)
|
| 192 |
+
gt_se3 = align_to_first_camera(gt_se3)
|
| 193 |
+
|
| 194 |
+
rel_rangle_deg, rel_tangle_deg = se3_to_relative_pose_error(pred_se3, gt_se3, num_frames)
|
| 195 |
+
print(f"{category} sequence {seq_name} Rot Error: {rel_rangle_deg.mean().item():.4f}")
|
| 196 |
+
print(f"{category} sequence {seq_name} Trans Error: {rel_tangle_deg.mean().item():.4f}")
|
| 197 |
+
|
| 198 |
+
return rel_rangle_deg.cpu().numpy(), rel_tangle_deg.cpu().numpy()
|
| 199 |
+
|
| 200 |
+
def evaluate(args: argparse.Namespace):
|
| 201 |
+
model = AnySplat.from_pretrained("lhjiang/anysplat")
|
| 202 |
+
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 203 |
+
model.to(device)
|
| 204 |
+
model.eval()
|
| 205 |
+
for param in model.parameters():
|
| 206 |
+
param.requires_grad = False
|
| 207 |
+
|
| 208 |
+
# CO3D evaluation
|
| 209 |
+
SEEN_CATEGORIES = [
|
| 210 |
+
"apple", "backpack", "banana", "baseballbat", "baseballglove",
|
| 211 |
+
"bench", "bicycle", "bottle", "bowl", "broccoli",
|
| 212 |
+
"cake", "car", "carrot", "cellphone", "chair",
|
| 213 |
+
"cup", "donut", "hairdryer", "handbag", "hydrant",
|
| 214 |
+
"keyboard", "laptop", "microwave", "motorcycle", "mouse",
|
| 215 |
+
"orange", "parkingmeter", "pizza", "plant", "stopsign",
|
| 216 |
+
"teddybear", "toaster", "toilet", "toybus", "toyplane",
|
| 217 |
+
"toytrain", "toytruck", "tv", "umbrella", "vase", "wineglass",
|
| 218 |
+
]
|
| 219 |
+
|
| 220 |
+
if args.debug:
|
| 221 |
+
SEEN_CATEGORIES = ["apple"]
|
| 222 |
+
|
| 223 |
+
per_category_results = {}
|
| 224 |
+
|
| 225 |
+
for category in SEEN_CATEGORIES:
|
| 226 |
+
print(f"Loading annotation for {category} test set")
|
| 227 |
+
annotation_file = os.path.join(args.co3d_anno_dir, f"{category}_test.jgz")
|
| 228 |
+
|
| 229 |
+
try:
|
| 230 |
+
with gzip.open(annotation_file, "r") as fin:
|
| 231 |
+
annotation = json.loads(fin.read())
|
| 232 |
+
except FileNotFoundError:
|
| 233 |
+
print(f"Annotation file not found for {category}, skipping")
|
| 234 |
+
continue
|
| 235 |
+
|
| 236 |
+
rError = []
|
| 237 |
+
tError = []
|
| 238 |
+
|
| 239 |
+
for seq_name, seq_data in annotation.items():
|
| 240 |
+
print("-" * 50)
|
| 241 |
+
|
| 242 |
+
print(f"Processing {seq_name} for {category} test set")
|
| 243 |
+
if args.debug and not os.path.exists(os.path.join(args.co3d_dir, category, seq_name)):
|
| 244 |
+
print(f"Skipping {seq_name} (not found)")
|
| 245 |
+
continue
|
| 246 |
+
|
| 247 |
+
seq_rError, seq_tError = process_sequence(
|
| 248 |
+
model, seq_name, seq_data, category, args.co3d_dir,
|
| 249 |
+
args.min_num_images, args.num_frames, args.use_ba, device, torch.bfloat16
|
| 250 |
+
)
|
| 251 |
+
|
| 252 |
+
print("-" * 50)
|
| 253 |
+
|
| 254 |
+
if seq_rError is not None and seq_tError is not None:
|
| 255 |
+
rError.extend(seq_rError)
|
| 256 |
+
tError.extend(seq_tError)
|
| 257 |
+
|
| 258 |
+
if not rError:
|
| 259 |
+
print(f"No valid sequences found for {category}, skipping")
|
| 260 |
+
continue
|
| 261 |
+
|
| 262 |
+
rError = np.array(rError)
|
| 263 |
+
tError = np.array(tError)
|
| 264 |
+
|
| 265 |
+
thresholds = [5, 10, 20, 30]
|
| 266 |
+
Aucs = {}
|
| 267 |
+
|
| 268 |
+
for threshold in thresholds:
|
| 269 |
+
Auc, _ = calculate_auc_np(rError, tError, max_threshold=threshold)
|
| 270 |
+
Aucs[threshold] = Auc
|
| 271 |
+
|
| 272 |
+
print("="*80)
|
| 273 |
+
print(f"AUC of {category} test set: {Aucs[30]:.4f}")
|
| 274 |
+
print("="*80)
|
| 275 |
+
|
| 276 |
+
per_category_results[category] = {
|
| 277 |
+
"rError": rError,
|
| 278 |
+
"tError": tError,
|
| 279 |
+
"Auc_5": Aucs[5],
|
| 280 |
+
"Auc_10": Aucs[10],
|
| 281 |
+
"Auc_20": Aucs[20],
|
| 282 |
+
"Auc_30": Aucs[30],
|
| 283 |
+
}
|
| 284 |
+
|
| 285 |
+
# Print summary results
|
| 286 |
+
print("\nSummary of AUC results:")
|
| 287 |
+
print("-"*50)
|
| 288 |
+
for category in sorted(per_category_results.keys()):
|
| 289 |
+
print(f"{category:<15} AUC_5: {per_category_results[category]['Auc_5']:.4f}")
|
| 290 |
+
print(f"{category:<15} AUC_30: {per_category_results[category]['Auc_30']:.4f}")
|
| 291 |
+
print(f"{category:<15} AUC_20: {per_category_results[category]['Auc_20']:.4f}")
|
| 292 |
+
print(f"{category:<15} AUC_10: {per_category_results[category]['Auc_10']:.4f}")
|
| 293 |
+
|
| 294 |
+
if per_category_results:
|
| 295 |
+
mean_AUC_30 = np.mean([per_category_results[category]["Auc_30"] for category in per_category_results])
|
| 296 |
+
mean_AUC_20 = np.mean([per_category_results[category]["Auc_20"] for category in per_category_results])
|
| 297 |
+
mean_AUC_10 = np.mean([per_category_results[category]["Auc_10"] for category in per_category_results])
|
| 298 |
+
mean_AUC_5 = np.mean([per_category_results[category]["Auc_5"] for category in per_category_results])
|
| 299 |
+
print("-"*50)
|
| 300 |
+
print(f"Mean AUC_5: {mean_AUC_5:.4f}")
|
| 301 |
+
print(f"Mean AUC_30: {mean_AUC_30:.4f}")
|
| 302 |
+
print(f"Mean AUC_20: {mean_AUC_20:.4f}")
|
| 303 |
+
print(f"Mean AUC_10: {mean_AUC_10:.4f}")
|
| 304 |
+
|
| 305 |
+
# Generate a random index to avoid overwriting previous results
|
| 306 |
+
# random_index = torch.randint(0, 10000, (1,)).item()
|
| 307 |
+
# Use timestamp as index instead of random number
|
| 308 |
+
import datetime
|
| 309 |
+
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
| 310 |
+
random_index = timestamp
|
| 311 |
+
results_file = f"co3d_results_{random_index}.txt"
|
| 312 |
+
|
| 313 |
+
with open(results_file, "w") as f:
|
| 314 |
+
f.write("CO3D Evaluation Results\n")
|
| 315 |
+
f.write("=" * 50 + "\n\n")
|
| 316 |
+
|
| 317 |
+
f.write("Per-category results:\n")
|
| 318 |
+
f.write("-" * 50 + "\n")
|
| 319 |
+
for category in sorted(per_category_results.keys()):
|
| 320 |
+
f.write(f"{category:<15} AUC_30: {per_category_results[category]['Auc_30']:.4f}\n")
|
| 321 |
+
f.write(f"{category:<15} AUC_20: {per_category_results[category]['Auc_20']:.4f}\n")
|
| 322 |
+
f.write(f"{category:<15} AUC_10: {per_category_results[category]['Auc_10']:.4f}\n")
|
| 323 |
+
f.write(f"{category:<15} AUC_5: {per_category_results[category]['Auc_5']:.4f}\n")
|
| 324 |
+
f.write("\n")
|
| 325 |
+
|
| 326 |
+
if per_category_results:
|
| 327 |
+
f.write("-" * 50 + "\n")
|
| 328 |
+
f.write(f"Mean AUC_30: {mean_AUC_30:.4f}\n")
|
| 329 |
+
f.write(f"Mean AUC_20: {mean_AUC_20:.4f}\n")
|
| 330 |
+
f.write(f"Mean AUC_10: {mean_AUC_10:.4f}\n")
|
| 331 |
+
f.write(f"Mean AUC_5: {mean_AUC_5:.4f}\n")
|
| 332 |
+
f.write("\n" + "=" * 50 + "\n")
|
| 333 |
+
|
| 334 |
+
print(f"Results saved to {results_file}")
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
if __name__ == "__main__":
|
| 338 |
+
args = setup_args()
|
| 339 |
+
evaluate(args)
|
src/evaluation/evaluation_cfg.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
@dataclass
|
| 6 |
+
class MethodCfg:
|
| 7 |
+
name: str
|
| 8 |
+
key: str
|
| 9 |
+
path: Path
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@dataclass
|
| 13 |
+
class SceneCfg:
|
| 14 |
+
scene: str
|
| 15 |
+
target_index: int
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
@dataclass
|
| 19 |
+
class EvaluationCfg:
|
| 20 |
+
methods: list[MethodCfg]
|
| 21 |
+
side_by_side_path: Path | None
|
| 22 |
+
output_metrics_path: Path
|
| 23 |
+
animate_side_by_side: bool
|
| 24 |
+
highlighted: list[SceneCfg]
|
src/evaluation/evaluation_index_generator.py
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from dataclasses import asdict, dataclass
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from typing import Optional
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from einops import rearrange
|
| 8 |
+
from lightning.pytorch import LightningModule
|
| 9 |
+
from tqdm import tqdm
|
| 10 |
+
|
| 11 |
+
from ..geometry.epipolar_lines import project_rays
|
| 12 |
+
from ..geometry.projection import get_world_rays, sample_image_grid
|
| 13 |
+
from ..misc.image_io import save_image
|
| 14 |
+
from ..visualization.annotation import add_label
|
| 15 |
+
from ..visualization.layout import add_border, hcat
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
@dataclass
|
| 19 |
+
class EvaluationIndexGeneratorCfg:
|
| 20 |
+
num_target_views: int
|
| 21 |
+
min_distance: int
|
| 22 |
+
max_distance: int
|
| 23 |
+
min_overlap: float
|
| 24 |
+
max_overlap: float
|
| 25 |
+
output_path: Path
|
| 26 |
+
save_previews: bool
|
| 27 |
+
seed: int
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@dataclass
|
| 31 |
+
class IndexEntry:
|
| 32 |
+
context: tuple[int, ...]
|
| 33 |
+
target: tuple[int, ...]
|
| 34 |
+
overlap: Optional[str | float] = None # choose from ["small", "medium", "large"] or a float number indicates the overlap ratio
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class EvaluationIndexGenerator(LightningModule):
|
| 38 |
+
generator: torch.Generator
|
| 39 |
+
cfg: EvaluationIndexGeneratorCfg
|
| 40 |
+
index: dict[str, IndexEntry | None]
|
| 41 |
+
|
| 42 |
+
def __init__(self, cfg: EvaluationIndexGeneratorCfg) -> None:
|
| 43 |
+
super().__init__()
|
| 44 |
+
self.cfg = cfg
|
| 45 |
+
self.generator = torch.Generator()
|
| 46 |
+
self.generator.manual_seed(cfg.seed)
|
| 47 |
+
self.index = {}
|
| 48 |
+
|
| 49 |
+
def test_step(self, batch, batch_idx):
|
| 50 |
+
b, v, _, h, w = batch["target"]["image"].shape
|
| 51 |
+
assert b == 1
|
| 52 |
+
extrinsics = batch["target"]["extrinsics"][0]
|
| 53 |
+
intrinsics = batch["target"]["intrinsics"][0]
|
| 54 |
+
scene = batch["scene"][0]
|
| 55 |
+
|
| 56 |
+
context_indices = torch.randperm(v, generator=self.generator)
|
| 57 |
+
for context_index in tqdm(context_indices, "Finding context pair"):
|
| 58 |
+
xy, _ = sample_image_grid((h, w), self.device)
|
| 59 |
+
context_origins, context_directions = get_world_rays(
|
| 60 |
+
rearrange(xy, "h w xy -> (h w) xy"),
|
| 61 |
+
extrinsics[context_index],
|
| 62 |
+
intrinsics[context_index],
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
# Step away from context view until the minimum overlap threshold is met.
|
| 66 |
+
valid_indices = []
|
| 67 |
+
for step in (1, -1):
|
| 68 |
+
min_distance = self.cfg.min_distance
|
| 69 |
+
max_distance = self.cfg.max_distance
|
| 70 |
+
current_index = context_index + step * min_distance
|
| 71 |
+
|
| 72 |
+
while 0 <= current_index.item() < v:
|
| 73 |
+
# Compute overlap.
|
| 74 |
+
current_origins, current_directions = get_world_rays(
|
| 75 |
+
rearrange(xy, "h w xy -> (h w) xy"),
|
| 76 |
+
extrinsics[current_index],
|
| 77 |
+
intrinsics[current_index],
|
| 78 |
+
)
|
| 79 |
+
projection_onto_current = project_rays(
|
| 80 |
+
context_origins,
|
| 81 |
+
context_directions,
|
| 82 |
+
extrinsics[current_index],
|
| 83 |
+
intrinsics[current_index],
|
| 84 |
+
)
|
| 85 |
+
projection_onto_context = project_rays(
|
| 86 |
+
current_origins,
|
| 87 |
+
current_directions,
|
| 88 |
+
extrinsics[context_index],
|
| 89 |
+
intrinsics[context_index],
|
| 90 |
+
)
|
| 91 |
+
overlap_a = projection_onto_context["overlaps_image"].float().mean()
|
| 92 |
+
overlap_b = projection_onto_current["overlaps_image"].float().mean()
|
| 93 |
+
|
| 94 |
+
overlap = min(overlap_a, overlap_b)
|
| 95 |
+
delta = (current_index - context_index).abs()
|
| 96 |
+
|
| 97 |
+
min_overlap = self.cfg.min_overlap
|
| 98 |
+
max_overlap = self.cfg.max_overlap
|
| 99 |
+
if min_overlap <= overlap <= max_overlap:
|
| 100 |
+
valid_indices.append(
|
| 101 |
+
(current_index.item(), overlap_a, overlap_b)
|
| 102 |
+
)
|
| 103 |
+
|
| 104 |
+
# Stop once the camera has panned away too much.
|
| 105 |
+
if overlap < min_overlap or delta > max_distance:
|
| 106 |
+
break
|
| 107 |
+
|
| 108 |
+
current_index += step
|
| 109 |
+
|
| 110 |
+
if valid_indices:
|
| 111 |
+
# Pick a random valid view. Index the resulting views.
|
| 112 |
+
num_options = len(valid_indices)
|
| 113 |
+
chosen = torch.randint(
|
| 114 |
+
0, num_options, size=tuple(), generator=self.generator
|
| 115 |
+
)
|
| 116 |
+
chosen, overlap_a, overlap_b = valid_indices[chosen]
|
| 117 |
+
|
| 118 |
+
context_left = min(chosen, context_index.item())
|
| 119 |
+
context_right = max(chosen, context_index.item())
|
| 120 |
+
delta = context_right - context_left
|
| 121 |
+
|
| 122 |
+
# Pick non-repeated random target views.
|
| 123 |
+
while True:
|
| 124 |
+
target_views = torch.randint(
|
| 125 |
+
context_left,
|
| 126 |
+
context_right + 1,
|
| 127 |
+
(self.cfg.num_target_views,),
|
| 128 |
+
generator=self.generator,
|
| 129 |
+
)
|
| 130 |
+
if (target_views.unique(return_counts=True)[1] == 1).all():
|
| 131 |
+
break
|
| 132 |
+
|
| 133 |
+
target = tuple(sorted(target_views.tolist()))
|
| 134 |
+
self.index[scene] = IndexEntry(
|
| 135 |
+
context=(context_left, context_right),
|
| 136 |
+
target=target,
|
| 137 |
+
)
|
| 138 |
+
|
| 139 |
+
# Optionally, save a preview.
|
| 140 |
+
if self.cfg.save_previews:
|
| 141 |
+
preview_path = self.cfg.output_path / "previews"
|
| 142 |
+
preview_path.mkdir(exist_ok=True, parents=True)
|
| 143 |
+
a = batch["target"]["image"][0, chosen]
|
| 144 |
+
a = add_label(a, f"Overlap: {overlap_a * 100:.1f}%")
|
| 145 |
+
b = batch["target"]["image"][0, context_index]
|
| 146 |
+
b = add_label(b, f"Overlap: {overlap_b * 100:.1f}%")
|
| 147 |
+
vis = add_border(add_border(hcat(a, b)), 1, 0)
|
| 148 |
+
vis = add_label(vis, f"Distance: {delta} frames")
|
| 149 |
+
save_image(add_border(vis), preview_path / f"{scene}.png")
|
| 150 |
+
break
|
| 151 |
+
else:
|
| 152 |
+
# This happens if no starting frame produces a valid evaluation example.
|
| 153 |
+
self.index[scene] = None
|
| 154 |
+
|
| 155 |
+
def save_index(self) -> None:
|
| 156 |
+
self.cfg.output_path.mkdir(exist_ok=True, parents=True)
|
| 157 |
+
with (self.cfg.output_path / "evaluation_index.json").open("w") as f:
|
| 158 |
+
json.dump(
|
| 159 |
+
{k: None if v is None else asdict(v) for k, v in self.index.items()}, f
|
| 160 |
+
)
|
src/evaluation/metrics.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from functools import cache
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from einops import reduce
|
| 5 |
+
from jaxtyping import Float
|
| 6 |
+
from lpips import LPIPS
|
| 7 |
+
from skimage.metrics import structural_similarity
|
| 8 |
+
from torch import Tensor
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
@torch.no_grad()
|
| 12 |
+
def compute_psnr(
|
| 13 |
+
ground_truth: Float[Tensor, "batch channel height width"],
|
| 14 |
+
predicted: Float[Tensor, "batch channel height width"],
|
| 15 |
+
) -> Float[Tensor, " batch"]:
|
| 16 |
+
ground_truth = ground_truth.clip(min=0, max=1)
|
| 17 |
+
predicted = predicted.clip(min=0, max=1)
|
| 18 |
+
mse = reduce((ground_truth - predicted) ** 2, "b c h w -> b", "mean")
|
| 19 |
+
return -10 * mse.log10()
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@cache
|
| 23 |
+
def get_lpips(device: torch.device) -> LPIPS:
|
| 24 |
+
return LPIPS(net="vgg").to(device)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
@torch.no_grad()
|
| 28 |
+
def compute_lpips(
|
| 29 |
+
ground_truth: Float[Tensor, "batch channel height width"],
|
| 30 |
+
predicted: Float[Tensor, "batch channel height width"],
|
| 31 |
+
) -> Float[Tensor, " batch"]:
|
| 32 |
+
value = get_lpips(predicted.device).forward(ground_truth, predicted, normalize=True)
|
| 33 |
+
return value[:, 0, 0, 0]
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
@torch.no_grad()
|
| 37 |
+
def compute_ssim(
|
| 38 |
+
ground_truth: Float[Tensor, "batch channel height width"],
|
| 39 |
+
predicted: Float[Tensor, "batch channel height width"],
|
| 40 |
+
) -> Float[Tensor, " batch"]:
|
| 41 |
+
ssim = [
|
| 42 |
+
structural_similarity(
|
| 43 |
+
gt.detach().cpu().numpy(),
|
| 44 |
+
hat.detach().cpu().numpy(),
|
| 45 |
+
win_size=11,
|
| 46 |
+
gaussian_weights=True,
|
| 47 |
+
channel_axis=0,
|
| 48 |
+
data_range=1.0,
|
| 49 |
+
)
|
| 50 |
+
for gt, hat in zip(ground_truth, predicted)
|
| 51 |
+
]
|
| 52 |
+
return torch.tensor(ssim, dtype=predicted.dtype, device=predicted.device)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def compute_geodesic_distance_from_two_matrices(m1, m2):
|
| 56 |
+
batch = m1.shape[0]
|
| 57 |
+
m = torch.bmm(m1, m2.transpose(1, 2)) # batch*3*3
|
| 58 |
+
|
| 59 |
+
cos = (m[:, 0, 0] + m[:, 1, 1] + m[:, 2, 2] - 1) / 2
|
| 60 |
+
cos = torch.min(cos, torch.autograd.Variable(torch.ones(batch).to(m1.device)))
|
| 61 |
+
cos = torch.max(cos, torch.autograd.Variable(torch.ones(batch).to(m1.device)) * -1)
|
| 62 |
+
|
| 63 |
+
theta = torch.acos(cos)
|
| 64 |
+
|
| 65 |
+
# theta = torch.min(theta, 2*np.pi - theta)
|
| 66 |
+
|
| 67 |
+
return theta
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def angle_error_mat(R1, R2):
|
| 71 |
+
cos = (torch.trace(torch.mm(R1.T, R2)) - 1) / 2
|
| 72 |
+
cos = torch.clamp(cos, -1.0, 1.0) # numerical errors can make it out of bounds
|
| 73 |
+
return torch.rad2deg(torch.abs(torch.acos(cos)))
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def angle_error_vec(v1, v2):
|
| 77 |
+
n = torch.norm(v1) * torch.norm(v2)
|
| 78 |
+
cos_theta = torch.dot(v1, v2) / n
|
| 79 |
+
cos_theta = torch.clamp(cos_theta, -1.0, 1.0) # numerical errors can make it out of bounds
|
| 80 |
+
return torch.rad2deg(torch.acos(cos_theta))
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def compute_translation_error(t1, t2):
|
| 84 |
+
return torch.norm(t1 - t2)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
@torch.no_grad()
|
| 88 |
+
def compute_pose_error(pose_gt, pose_pred):
|
| 89 |
+
R_gt = pose_gt[:3, :3]
|
| 90 |
+
t_gt = pose_gt[:3, 3]
|
| 91 |
+
|
| 92 |
+
R = pose_pred[:3, :3]
|
| 93 |
+
t = pose_pred[:3, 3]
|
| 94 |
+
|
| 95 |
+
error_t = angle_error_vec(t, t_gt)
|
| 96 |
+
error_t = torch.minimum(error_t, 180 - error_t) # ambiguity of E estimation
|
| 97 |
+
error_t_scale = compute_translation_error(t, t_gt)
|
| 98 |
+
error_R = angle_error_mat(R, R_gt)
|
| 99 |
+
return error_t, error_t_scale, error_R
|
| 100 |
+
|
| 101 |
+
@torch.no_grad()
|
| 102 |
+
def abs_relative_difference(output, target, valid_mask=None):
|
| 103 |
+
actual_output = output
|
| 104 |
+
actual_target = target
|
| 105 |
+
abs_relative_diff = torch.abs(actual_output - actual_target) / actual_target
|
| 106 |
+
if valid_mask is not None:
|
| 107 |
+
abs_relative_diff[~valid_mask] = 0
|
| 108 |
+
n = valid_mask.sum((-1, -2))
|
| 109 |
+
else:
|
| 110 |
+
n = output.shape[-1] * output.shape[-2]
|
| 111 |
+
abs_relative_diff = torch.sum(abs_relative_diff, (-1, -2)) / n
|
| 112 |
+
return abs_relative_diff.mean()
|
| 113 |
+
|
| 114 |
+
# adapt from: https://github.com/imran3180/depth-map-prediction/blob/master/main.py
|
| 115 |
+
@torch.no_grad()
|
| 116 |
+
def threshold_percentage(output, target, threshold_val, valid_mask=None):
|
| 117 |
+
d1 = output / target
|
| 118 |
+
d2 = target / output
|
| 119 |
+
max_d1_d2 = torch.max(d1, d2)
|
| 120 |
+
zero = torch.zeros_like(output)
|
| 121 |
+
one = torch.ones_like(output)
|
| 122 |
+
bit_mat = torch.where(max_d1_d2 < threshold_val, one, zero)
|
| 123 |
+
if valid_mask is not None:
|
| 124 |
+
bit_mat[~valid_mask] = 0
|
| 125 |
+
n = valid_mask.sum((-1, -2))
|
| 126 |
+
else:
|
| 127 |
+
n = output.shape[-1] * output.shape[-2]
|
| 128 |
+
count_mat = torch.sum(bit_mat, (-1, -2))
|
| 129 |
+
threshold_mat = count_mat / n
|
| 130 |
+
return threshold_mat.mean()
|
| 131 |
+
|
| 132 |
+
@torch.no_grad()
|
| 133 |
+
def delta1_acc(pred, gt, valid_mask):
|
| 134 |
+
return threshold_percentage(pred, gt, 1.25, valid_mask)
|
src/geometry/camera_emb.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from einops import rearrange
|
| 2 |
+
|
| 3 |
+
from .projection import sample_image_grid, get_local_rays
|
| 4 |
+
from ..misc.sht import rsh_cart_2, rsh_cart_4, rsh_cart_6, rsh_cart_8
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def get_intrinsic_embedding(context, degree=0, downsample=1, merge_hw=False):
|
| 8 |
+
assert degree in [0, 2, 4, 8]
|
| 9 |
+
|
| 10 |
+
b, v, _, h, w = context["image"].shape
|
| 11 |
+
device = context["image"].device
|
| 12 |
+
tgt_h, tgt_w = h // downsample, w // downsample
|
| 13 |
+
xy_ray, _ = sample_image_grid((tgt_h, tgt_w), device)
|
| 14 |
+
xy_ray = xy_ray[None, None, ...].expand(b, v, -1, -1, -1) # [b, v, h, w, 2]
|
| 15 |
+
directions = get_local_rays(xy_ray, rearrange(context["intrinsics"], "b v i j -> b v () () i j"),)
|
| 16 |
+
|
| 17 |
+
if degree == 2:
|
| 18 |
+
directions = rsh_cart_2(directions)
|
| 19 |
+
elif degree == 4:
|
| 20 |
+
directions = rsh_cart_4(directions)
|
| 21 |
+
elif degree == 8:
|
| 22 |
+
directions = rsh_cart_8(directions)
|
| 23 |
+
|
| 24 |
+
if merge_hw:
|
| 25 |
+
directions = rearrange(directions, "b v h w d -> b v (h w) d")
|
| 26 |
+
else:
|
| 27 |
+
directions = rearrange(directions, "b v h w d -> b v d h w")
|
| 28 |
+
|
| 29 |
+
return directions
|
src/geometry/epipolar_lines.py
ADDED
|
@@ -0,0 +1,292 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import itertools
|
| 2 |
+
from typing import Iterable, Literal, Optional, TypedDict
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
from einops import einsum, repeat
|
| 6 |
+
from jaxtyping import Bool, Float
|
| 7 |
+
from torch import Tensor
|
| 8 |
+
from torch.utils.data.dataloader import default_collate
|
| 9 |
+
|
| 10 |
+
from .projection import (
|
| 11 |
+
get_world_rays,
|
| 12 |
+
homogenize_points,
|
| 13 |
+
homogenize_vectors,
|
| 14 |
+
intersect_rays,
|
| 15 |
+
project_camera_space,
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _is_in_bounds(
|
| 20 |
+
xy: Float[Tensor, "*batch 2"],
|
| 21 |
+
epsilon: float = 1e-6,
|
| 22 |
+
) -> Bool[Tensor, " *batch"]:
|
| 23 |
+
"""Check whether the specified XY coordinates are within the normalized image plane,
|
| 24 |
+
which has a range from 0 to 1 in each direction.
|
| 25 |
+
"""
|
| 26 |
+
return (xy >= -epsilon).all(dim=-1) & (xy <= 1 + epsilon).all(dim=-1)
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _is_in_front_of_camera(
|
| 30 |
+
xyz: Float[Tensor, "*batch 3"],
|
| 31 |
+
epsilon: float = 1e-6,
|
| 32 |
+
) -> Bool[Tensor, " *batch"]:
|
| 33 |
+
"""Check whether the specified points in camera space are in front of the camera."""
|
| 34 |
+
return xyz[..., -1] > -epsilon
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _is_positive_t(
|
| 38 |
+
t: Float[Tensor, " *batch"],
|
| 39 |
+
epsilon: float = 1e-6,
|
| 40 |
+
) -> Bool[Tensor, " *batch"]:
|
| 41 |
+
"""Check whether the specified t value is positive."""
|
| 42 |
+
return t > -epsilon
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
class PointProjection(TypedDict):
|
| 46 |
+
t: Float[Tensor, " *batch"] # ray parameter, as in xyz = origin + t * direction
|
| 47 |
+
xy: Float[Tensor, "*batch 2"] # image-space xy (normalized to 0 to 1)
|
| 48 |
+
|
| 49 |
+
# A "valid" projection satisfies two conditions:
|
| 50 |
+
# 1. It is in front of the camera (i.e., its 3D Z coordinate is positive).
|
| 51 |
+
# 2. It is within the image frame (i.e., its 2D coordinates are between 0 and 1).
|
| 52 |
+
valid: Bool[Tensor, " *batch"]
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _intersect_image_coordinate(
|
| 56 |
+
intrinsics: Float[Tensor, "*#batch 3 3"],
|
| 57 |
+
origins: Float[Tensor, "*#batch 3"],
|
| 58 |
+
directions: Float[Tensor, "*#batch 3"],
|
| 59 |
+
dimension: Literal["x", "y"],
|
| 60 |
+
coordinate_value: float,
|
| 61 |
+
) -> PointProjection:
|
| 62 |
+
"""Compute the intersection of the projection of a camera-space ray with a line
|
| 63 |
+
that's parallel to the image frame, either horizontally or vertically.
|
| 64 |
+
"""
|
| 65 |
+
|
| 66 |
+
# Define shorthands.
|
| 67 |
+
dim = "xy".index(dimension)
|
| 68 |
+
other_dim = 1 - dim
|
| 69 |
+
fs = intrinsics[..., dim, dim] # focal length, same coordinate
|
| 70 |
+
fo = intrinsics[..., other_dim, other_dim] # focal length, other coordinate
|
| 71 |
+
cs = intrinsics[..., dim, 2] # principal point, same coordinate
|
| 72 |
+
co = intrinsics[..., other_dim, 2] # principal point, other coordinate
|
| 73 |
+
os = origins[..., dim] # ray origin, same coordinate
|
| 74 |
+
oo = origins[..., other_dim] # ray origin, other coordinate
|
| 75 |
+
ds = directions[..., dim] # ray direction, same coordinate
|
| 76 |
+
do = directions[..., other_dim] # ray direction, other coordinate
|
| 77 |
+
oz = origins[..., 2] # ray origin, z coordinate
|
| 78 |
+
dz = directions[..., 2] # ray direction, z coordinate
|
| 79 |
+
c = (coordinate_value - cs) / fs # coefficient (computed once and factored out)
|
| 80 |
+
|
| 81 |
+
# Compute the value of t at the intersection.
|
| 82 |
+
# Note: Infinite values of t are fine. No need to handle division by zero.
|
| 83 |
+
t_numerator = c * oz - os
|
| 84 |
+
t_denominator = ds - c * dz
|
| 85 |
+
t = t_numerator / t_denominator
|
| 86 |
+
|
| 87 |
+
# Compute the value of the other coordinate at the intersection.
|
| 88 |
+
# Note: Infinite coordinate values are fine. No need to handle division by zero.
|
| 89 |
+
coordinate_numerator = fo * (oo * (c * dz - ds) + do * (os - c * oz))
|
| 90 |
+
coordinate_denominator = dz * os - ds * oz
|
| 91 |
+
coordinate_other = co + coordinate_numerator / coordinate_denominator
|
| 92 |
+
coordinate_same = torch.ones_like(coordinate_other) * coordinate_value
|
| 93 |
+
xy = [coordinate_same]
|
| 94 |
+
xy.insert(other_dim, coordinate_other)
|
| 95 |
+
xy = torch.stack(xy, dim=-1)
|
| 96 |
+
xyz = origins + t[..., None] * directions
|
| 97 |
+
|
| 98 |
+
# These will all have exactly the same batch shape (no broadcasting necessary). In
|
| 99 |
+
# terms of jaxtyping annotations, they all match *batch, not just *#batch.
|
| 100 |
+
return {
|
| 101 |
+
"t": t,
|
| 102 |
+
"xy": xy,
|
| 103 |
+
"valid": _is_in_bounds(xy) & _is_in_front_of_camera(xyz) & _is_positive_t(t),
|
| 104 |
+
}
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def _compare_projections(
|
| 108 |
+
intersections: Iterable[PointProjection],
|
| 109 |
+
reduction: Literal["min", "max"],
|
| 110 |
+
) -> PointProjection:
|
| 111 |
+
intersections = {k: v.clone() for k, v in default_collate(intersections).items()}
|
| 112 |
+
t = intersections["t"]
|
| 113 |
+
xy = intersections["xy"]
|
| 114 |
+
valid = intersections["valid"]
|
| 115 |
+
|
| 116 |
+
# Make sure out-of-bounds values are not chosen.
|
| 117 |
+
lowest_priority = {
|
| 118 |
+
"min": torch.inf,
|
| 119 |
+
"max": -torch.inf,
|
| 120 |
+
}[reduction]
|
| 121 |
+
t[~valid] = lowest_priority
|
| 122 |
+
|
| 123 |
+
# Run the reduction (either t.min() or t.max()).
|
| 124 |
+
reduced, selector = getattr(t, reduction)(dim=0)
|
| 125 |
+
|
| 126 |
+
# Index the results.
|
| 127 |
+
return {
|
| 128 |
+
"t": reduced,
|
| 129 |
+
"xy": xy.gather(0, repeat(selector, "... -> () ... xy", xy=2))[0],
|
| 130 |
+
"valid": valid.gather(0, selector[None])[0],
|
| 131 |
+
}
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def _compute_point_projection(
|
| 135 |
+
xyz: Float[Tensor, "*#batch 3"],
|
| 136 |
+
t: Float[Tensor, "*#batch"],
|
| 137 |
+
intrinsics: Float[Tensor, "*#batch 3 3"],
|
| 138 |
+
) -> PointProjection:
|
| 139 |
+
xy = project_camera_space(xyz, intrinsics)
|
| 140 |
+
return {
|
| 141 |
+
"t": t,
|
| 142 |
+
"xy": xy,
|
| 143 |
+
"valid": _is_in_bounds(xy) & _is_in_front_of_camera(xyz) & _is_positive_t(t),
|
| 144 |
+
}
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
class RaySegmentProjection(TypedDict):
|
| 148 |
+
t_min: Float[Tensor, " *batch"] # ray parameter
|
| 149 |
+
t_max: Float[Tensor, " *batch"] # ray parameter
|
| 150 |
+
xy_min: Float[Tensor, "*batch 2"] # image-space xy (normalized to 0 to 1)
|
| 151 |
+
xy_max: Float[Tensor, "*batch 2"] # image-space xy (normalized to 0 to 1)
|
| 152 |
+
|
| 153 |
+
# Whether the segment overlaps the image. If not, the above values are meaningless.
|
| 154 |
+
overlaps_image: Bool[Tensor, " *batch"]
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def project_rays(
|
| 158 |
+
origins: Float[Tensor, "*#batch 3"],
|
| 159 |
+
directions: Float[Tensor, "*#batch 3"],
|
| 160 |
+
extrinsics: Float[Tensor, "*#batch 4 4"],
|
| 161 |
+
intrinsics: Float[Tensor, "*#batch 3 3"],
|
| 162 |
+
near: Optional[Float[Tensor, "*#batch"]] = None,
|
| 163 |
+
far: Optional[Float[Tensor, "*#batch"]] = None,
|
| 164 |
+
epsilon: float = 1e-6,
|
| 165 |
+
) -> RaySegmentProjection:
|
| 166 |
+
# Transform the rays into camera space.
|
| 167 |
+
world_to_cam = torch.linalg.inv(extrinsics)
|
| 168 |
+
origins = homogenize_points(origins)
|
| 169 |
+
origins = einsum(world_to_cam, origins, "... i j, ... j -> ... i")
|
| 170 |
+
directions = homogenize_vectors(directions)
|
| 171 |
+
directions = einsum(world_to_cam, directions, "... i j, ... j -> ... i")
|
| 172 |
+
origins = origins[..., :3]
|
| 173 |
+
directions = directions[..., :3]
|
| 174 |
+
|
| 175 |
+
# Compute intersections with the image's frame.
|
| 176 |
+
frame_intersections = (
|
| 177 |
+
_intersect_image_coordinate(intrinsics, origins, directions, "x", 0.0),
|
| 178 |
+
_intersect_image_coordinate(intrinsics, origins, directions, "x", 1.0),
|
| 179 |
+
_intersect_image_coordinate(intrinsics, origins, directions, "y", 0.0),
|
| 180 |
+
_intersect_image_coordinate(intrinsics, origins, directions, "y", 1.0),
|
| 181 |
+
)
|
| 182 |
+
frame_intersection_min = _compare_projections(frame_intersections, "min")
|
| 183 |
+
frame_intersection_max = _compare_projections(frame_intersections, "max")
|
| 184 |
+
|
| 185 |
+
if near is None:
|
| 186 |
+
# Compute the ray's projection at zero depth. If an origin's depth (z value) is
|
| 187 |
+
# within epsilon of zero, this can mean one of two things:
|
| 188 |
+
# 1. The origin is at the camera's position. In this case, use the direction
|
| 189 |
+
# instead (the ray is probably coming from the camera).
|
| 190 |
+
# 2. The origin isn't at the camera's position, and randomly happens to be on
|
| 191 |
+
# the plane at zero depth. In this case, its projection is outside the image
|
| 192 |
+
# plane, and is thus marked as invalid.
|
| 193 |
+
origins_for_projection = origins.clone()
|
| 194 |
+
mask_depth_zero = origins_for_projection[..., -1] < epsilon
|
| 195 |
+
mask_at_camera = origins_for_projection.norm(dim=-1) < epsilon
|
| 196 |
+
origins_for_projection[mask_at_camera] = directions[mask_at_camera]
|
| 197 |
+
projection_at_zero = _compute_point_projection(
|
| 198 |
+
origins_for_projection,
|
| 199 |
+
torch.zeros_like(frame_intersection_min["t"]),
|
| 200 |
+
intrinsics,
|
| 201 |
+
)
|
| 202 |
+
projection_at_zero["valid"][mask_depth_zero & ~mask_at_camera] = False
|
| 203 |
+
else:
|
| 204 |
+
# If a near plane is specified, use it instead.
|
| 205 |
+
t_near = near.broadcast_to(frame_intersection_min["t"].shape)
|
| 206 |
+
projection_at_zero = _compute_point_projection(
|
| 207 |
+
origins + near[..., None] * directions,
|
| 208 |
+
t_near,
|
| 209 |
+
intrinsics,
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
if far is None:
|
| 213 |
+
# Compute the ray's projection at infinite depth. Using the projection function
|
| 214 |
+
# with directions (vectors) instead of points may seem wonky, but is equivalent
|
| 215 |
+
# to projecting the point at (origins + infinity * directions).
|
| 216 |
+
projection_at_infinity = _compute_point_projection(
|
| 217 |
+
directions,
|
| 218 |
+
torch.ones_like(frame_intersection_min["t"]) * torch.inf,
|
| 219 |
+
intrinsics,
|
| 220 |
+
)
|
| 221 |
+
else:
|
| 222 |
+
# If a far plane is specified, use it instead.
|
| 223 |
+
t_far = far.broadcast_to(frame_intersection_min["t"].shape)
|
| 224 |
+
projection_at_infinity = _compute_point_projection(
|
| 225 |
+
origins + far[..., None] * directions,
|
| 226 |
+
t_far,
|
| 227 |
+
intrinsics,
|
| 228 |
+
)
|
| 229 |
+
|
| 230 |
+
# Build the result by handling cases for ray intersection.
|
| 231 |
+
result = {
|
| 232 |
+
"t_min": torch.empty_like(projection_at_zero["t"]),
|
| 233 |
+
"t_max": torch.empty_like(projection_at_infinity["t"]),
|
| 234 |
+
"xy_min": torch.empty_like(projection_at_zero["xy"]),
|
| 235 |
+
"xy_max": torch.empty_like(projection_at_infinity["xy"]),
|
| 236 |
+
"overlaps_image": torch.empty_like(projection_at_zero["valid"]),
|
| 237 |
+
}
|
| 238 |
+
|
| 239 |
+
for min_valid, max_valid in itertools.product([True, False], [True, False]):
|
| 240 |
+
min_mask = projection_at_zero["valid"] ^ (not min_valid)
|
| 241 |
+
max_mask = projection_at_infinity["valid"] ^ (not max_valid)
|
| 242 |
+
mask = min_mask & max_mask
|
| 243 |
+
min_value = projection_at_zero if min_valid else frame_intersection_min
|
| 244 |
+
max_value = projection_at_infinity if max_valid else frame_intersection_max
|
| 245 |
+
result["t_min"][mask] = min_value["t"][mask]
|
| 246 |
+
result["t_max"][mask] = max_value["t"][mask]
|
| 247 |
+
result["xy_min"][mask] = min_value["xy"][mask]
|
| 248 |
+
result["xy_max"][mask] = max_value["xy"][mask]
|
| 249 |
+
result["overlaps_image"][mask] = (min_value["valid"] & max_value["valid"])[mask]
|
| 250 |
+
|
| 251 |
+
return result
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
class RaySegmentProjection(TypedDict):
|
| 255 |
+
t_min: Float[Tensor, " *batch"] # ray parameter
|
| 256 |
+
t_max: Float[Tensor, " *batch"] # ray parameter
|
| 257 |
+
xy_min: Float[Tensor, "*batch 2"] # image-space xy (normalized to 0 to 1)
|
| 258 |
+
xy_max: Float[Tensor, "*batch 2"] # image-space xy (normalized to 0 to 1)
|
| 259 |
+
|
| 260 |
+
# Whether the segment overlaps the image. If not, the above values are meaningless.
|
| 261 |
+
overlaps_image: Bool[Tensor, " *batch"]
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
def lift_to_3d(
|
| 265 |
+
origins: Float[Tensor, "*#batch 3"],
|
| 266 |
+
directions: Float[Tensor, "*#batch 3"],
|
| 267 |
+
xy: Float[Tensor, "*#batch 2"],
|
| 268 |
+
extrinsics: Float[Tensor, "*#batch 4 4"],
|
| 269 |
+
intrinsics: Float[Tensor, "*#batch 3 3"],
|
| 270 |
+
) -> Float[Tensor, "*batch 3"]:
|
| 271 |
+
"""Calculate the 3D positions that correspond to the specified 2D points on the
|
| 272 |
+
epipolar lines defined by the origins and directions. The extrinsics and intrinsics
|
| 273 |
+
are for the images the 2D points lie on.
|
| 274 |
+
"""
|
| 275 |
+
|
| 276 |
+
xy_origins, xy_directions = get_world_rays(xy, extrinsics, intrinsics)
|
| 277 |
+
return intersect_rays(origins, directions, xy_origins, xy_directions)
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
def get_depth(
|
| 281 |
+
origins: Float[Tensor, "*#batch 3"],
|
| 282 |
+
directions: Float[Tensor, "*#batch 3"],
|
| 283 |
+
xy: Float[Tensor, "*#batch 2"],
|
| 284 |
+
extrinsics: Float[Tensor, "*#batch 4 4"],
|
| 285 |
+
intrinsics: Float[Tensor, "*#batch 3 3"],
|
| 286 |
+
) -> Float[Tensor, " *batch"]:
|
| 287 |
+
"""Calculate the depths that correspond to the specified 2D points on the epipolar
|
| 288 |
+
lines defined by the origins and directions. The extrinsics and intrinsics are for
|
| 289 |
+
the images the 2D points lie on.
|
| 290 |
+
"""
|
| 291 |
+
xyz = lift_to_3d(origins, directions, xy, extrinsics, intrinsics)
|
| 292 |
+
return (xyz - origins).norm(dim=-1)
|