Aeonicc commited on
Commit
831ce99
·
verified ·
1 Parent(s): e781dd1

Initial commit for brand new AnySplat ZeroGPU Space (part 2)

Browse files
src/dataset/dataset_scannetpp.py ADDED
@@ -0,0 +1,443 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ from dataclasses import dataclass
3
+ from functools import cached_property
4
+ from io import BytesIO
5
+ from pathlib import Path
6
+ import random
7
+ from typing import Literal
8
+ import os
9
+ import cv2
10
+ import numpy as np
11
+ import torch
12
+ import torchvision.transforms as tf
13
+ from einops import rearrange, repeat
14
+ from jaxtyping import Float, UInt8
15
+ from PIL import Image
16
+ import torchvision
17
+ from torch import Tensor
18
+ from torch.utils.data import Dataset
19
+ from concurrent.futures import ThreadPoolExecutor, as_completed
20
+ import copy
21
+ from .shims.geometry_shim import depthmap_to_absolute_camera_coordinates
22
+
23
+ from .shims.load_shim import imread_cv2
24
+
25
+ from ..geometry.projection import get_fov
26
+ from .dataset import DatasetCfgCommon
27
+ from .shims.augmentation_shim import apply_augmentation_shim
28
+ from .shims.crop_shim import apply_crop_shim
29
+ from .types import Stage
30
+ from .view_sampler import ViewSampler
31
+ from ..misc.cam_utils import camera_normalization
32
+
33
+ @dataclass
34
+ class DatasetScannetppCfg(DatasetCfgCommon):
35
+ name: str
36
+ roots: list[Path]
37
+ baseline_min: float
38
+ baseline_max: float
39
+ max_fov: float
40
+ make_baseline_1: bool
41
+ augment: bool
42
+ relative_pose: bool
43
+ skip_bad_shape: bool
44
+ metric_thre: float
45
+ intr_augment: bool
46
+ make_baseline_1: bool
47
+ rescale_to_1cube: bool
48
+ normalize_by_pts3d: bool
49
+
50
+ @dataclass
51
+ class DatasetScannetppCfgWrapper:
52
+ scannetpp: DatasetScannetppCfg
53
+
54
+
55
+ class DatasetScannetpp(Dataset):
56
+ cfg: DatasetScannetppCfgWrapper
57
+ stage: Stage
58
+ view_sampler: ViewSampler
59
+
60
+ to_tensor: tf.ToTensor
61
+ chunks: list[Path]
62
+ near: float = 0.1
63
+ far: float = 100.0
64
+
65
+ def __init__(
66
+ self,
67
+ cfg: DatasetScannetppCfgWrapper,
68
+ stage: Stage,
69
+ view_sampler: ViewSampler,
70
+ ) -> None:
71
+ super().__init__()
72
+ self.cfg = cfg
73
+ self.stage = stage
74
+ self.view_sampler = view_sampler
75
+ self.to_tensor = tf.ToTensor()
76
+
77
+ # load data
78
+ self.data_root = cfg.roots[0]
79
+ self.data_list = [] # we use dslr rather than iphone
80
+ data_index = os.listdir(f"{self.data_root}") # we train all the scenes
81
+
82
+ if self.stage != "train":
83
+ with open(f"{self.data_root}/valid.json", "r") as file:
84
+ data_index = json.load(file)[:10]
85
+ data_index = data_index * 100
86
+ random.shuffle(data_index)
87
+ else:
88
+ with open(f"{self.data_root}/valid.json", "r") as file:
89
+ data_index = json.load(file)[10:]
90
+
91
+ self.data_list = [
92
+ os.path.join(self.data_root, item) for item in data_index
93
+ ]
94
+
95
+ self.scene_ids = {}
96
+ self.scenes = {}
97
+ index = 0
98
+ with ThreadPoolExecutor(max_workers=32) as executor:
99
+ futures = [executor.submit(self.load_metadata, scene_path) for scene_path in self.data_list]
100
+ for future in as_completed(futures):
101
+ scene_frames, scene_id = future.result()
102
+ self.scenes[scene_id] = scene_frames
103
+ self.scene_ids[index] = scene_id
104
+ index += 1
105
+
106
+ # if self.stage != "train":
107
+ # self.scene_ids = self.scene_ids
108
+ # random.shuffle(self.scene_ids)
109
+ print(f"Scannetpp: {self.stage}: loaded {len(self.scene_ids)} scenes")
110
+
111
+ def shuffle(self, lst: list) -> list:
112
+ indices = torch.randperm(len(lst))
113
+ return [lst[x] for x in indices]
114
+
115
+ def load_metadata(self, scene_path):
116
+ metadata_path = os.path.join(scene_path, "scene_metadata.npz")
117
+ metadata = np.load(metadata_path, allow_pickle=True)
118
+ intrinsics = metadata["intrinsics"]
119
+ trajectories = metadata["trajectories"]
120
+ images = metadata["images"]
121
+
122
+ scene_id = scene_path.split("/")[-1].split(".")[0]
123
+ scene_frames = [
124
+ {
125
+ "file_path": os.path.join(scene_path, "images", images[i].split(".")[0] + ".jpg"),
126
+ "depth_path": os.path.join(scene_path, "depth", images[i].split(".")[0] + ".png"),
127
+ "intrinsics": self.convert_intrinsics(intrinsics[i]),
128
+ "extrinsics": trajectories[i],
129
+ }
130
+ for i in range(len(images))
131
+ ]
132
+ scene_frames.sort(key=lambda x: x["file_path"]) # sort by file path to ensure correct order
133
+ return scene_frames, scene_id
134
+
135
+ def convert_intrinsics(self, intrinsics):
136
+ w = intrinsics[0, 2] * 2
137
+ h = intrinsics[1, 2] * 2
138
+ intrinsics[0, 0] = intrinsics[0, 0] / w
139
+ intrinsics[1, 1] = intrinsics[1, 1] / h
140
+ intrinsics[0, 2] = intrinsics[0, 2] / w
141
+ intrinsics[1, 2] = intrinsics[1, 2] / h
142
+ return intrinsics
143
+
144
+ def blender2opencv_c2w(self, pose):
145
+ blender2opencv = np.array(
146
+ [[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]]
147
+ )
148
+ opencv_c2w = np.array(pose) @ blender2opencv
149
+ return opencv_c2w.tolist()
150
+
151
+ def load_frames(self, frames):
152
+ with ThreadPoolExecutor(max_workers=1) as executor:
153
+ # Create a list to store futures with their original indices
154
+ futures_with_idx = []
155
+ for idx, file_path in enumerate(frames):
156
+ file_path = file_path["file_path"]
157
+ futures_with_idx.append(
158
+ (
159
+ idx,
160
+ executor.submit(
161
+ lambda p: self.to_tensor(Image.open(p).convert("RGB")),
162
+ file_path,
163
+ ),
164
+ )
165
+ )
166
+
167
+ # Pre-allocate list with correct size to maintain order
168
+ torch_images = [None] * len(frames)
169
+ for idx, future in futures_with_idx:
170
+ torch_images[idx] = future.result()
171
+ # Check if all images have the same size
172
+ sizes = set(img.shape for img in torch_images)
173
+ if len(sizes) == 1:
174
+ torch_images = torch.stack(torch_images)
175
+ # Return as list if images have different sizes
176
+ return torch_images
177
+
178
+ def load_depths(self, frames):
179
+ torch_depths = []
180
+ for idx, frame in enumerate(frames):
181
+ depthmap = imread_cv2(frame["depth_path"], cv2.IMREAD_UNCHANGED)
182
+ depthmap = depthmap.astype(np.float32) / 1000
183
+ depthmap[~np.isfinite(depthmap)] = 0
184
+ torch_depths.append(torch.from_numpy(depthmap))
185
+ return torch.stack(torch_depths) # [N, H, W]
186
+
187
+
188
+ def getitem(self, index: int, num_context_views: int, patchsize: tuple) -> dict:
189
+ # import time
190
+ # start_time = time.time()
191
+
192
+ scene = self.scene_ids[index]
193
+ example = self.scenes[scene]
194
+ # load poses
195
+ extrinsics = []
196
+ intrinsics = []
197
+ for frame in example:
198
+ extrinsic = frame["extrinsics"]
199
+ intrinsic = frame["intrinsics"]
200
+ extrinsics.append(extrinsic)
201
+ intrinsics.append(intrinsic)
202
+
203
+ extrinsics = np.array(extrinsics)
204
+ intrinsics = np.array(intrinsics)
205
+ extrinsics = torch.tensor(extrinsics, dtype=torch.float32)
206
+ intrinsics = torch.tensor(intrinsics, dtype=torch.float32)
207
+
208
+ try:
209
+ context_indices, target_indices, overlap = self.view_sampler.sample(
210
+ "scannetpp_"+scene,
211
+ num_context_views,
212
+ extrinsics,
213
+ intrinsics,
214
+ )
215
+ except ValueError:
216
+ # Skip because the example doesn't have enough frames.
217
+ raise Exception("Not enough frames")
218
+
219
+ # Skip the example if the field of view is too wide.
220
+ if (get_fov(intrinsics).rad2deg() > self.cfg.max_fov).any():
221
+ raise Exception("Field of view too wide")
222
+
223
+ # Load the images.
224
+ input_frames = [example[i] for i in context_indices]
225
+ target_frame = [example[i] for i in target_indices]
226
+
227
+ context_images = self.load_frames(input_frames)
228
+ target_images = self.load_frames(target_frame)
229
+
230
+ context_depths = self.load_depths(input_frames)
231
+ target_depths = self.load_depths(target_frame)
232
+
233
+ # Skip the example if the images don't have the right shape.
234
+ context_image_invalid = context_images.shape[1:] != (3, *self.cfg.original_image_shape)
235
+ target_image_invalid = target_images.shape[1:] != (3, *self.cfg.original_image_shape)
236
+ if self.cfg.skip_bad_shape and (context_image_invalid or target_image_invalid):
237
+ print(
238
+ f"Skipped bad example {example['key']}. Context shape was "
239
+ f"{context_images.shape} and target shape was "
240
+ f"{target_images.shape}."
241
+ )
242
+ raise Exception("Bad example image shape")
243
+
244
+ context_extrinsics = extrinsics[context_indices]
245
+
246
+ if self.cfg.make_baseline_1:
247
+ a, b = context_extrinsics[0, :3, 3], context_extrinsics[-1, :3, 3]
248
+ scale = (a - b).norm()
249
+ if scale < self.cfg.baseline_min or scale > self.cfg.baseline_max:
250
+ print(
251
+ f"Skipped {scene} because of baseline out of range: "
252
+ f"{scale:.6f}"
253
+ )
254
+ raise Exception("baseline out of range")
255
+ extrinsics[:, :3, 3] /= scale
256
+ else:
257
+ scale = 1
258
+
259
+ if self.cfg.relative_pose:
260
+ extrinsics = camera_normalization(extrinsics[context_indices][0:1], extrinsics)
261
+
262
+ if self.cfg.rescale_to_1cube:
263
+ scene_scale = torch.max(torch.abs(extrinsics[context_indices][:, :3, 3])) # target pose is not included
264
+ rescale_factor = 1 * scene_scale
265
+ extrinsics[:, :3, 3] /= rescale_factor
266
+
267
+ if torch.isnan(extrinsics).any() or torch.isinf(extrinsics).any():
268
+ raise Exception("encounter nan or inf in input poses")
269
+
270
+ example = {
271
+ "context": {
272
+ "extrinsics": extrinsics[context_indices],
273
+ "intrinsics": intrinsics[context_indices],
274
+ "image": context_images,
275
+ "depth": context_depths,
276
+ "near": self.get_bound("near", len(context_indices)) / scale,
277
+ "far": self.get_bound("far", len(context_indices)) / scale,
278
+ "index": context_indices,
279
+ "overlap": overlap,
280
+ },
281
+ "target": {
282
+ "extrinsics": extrinsics[target_indices],
283
+ "intrinsics": intrinsics[target_indices],
284
+ "image": target_images,
285
+ "depth": target_depths,
286
+ "near": self.get_bound("near", len(target_indices)) / scale,
287
+ "far": self.get_bound("far", len(target_indices)) / scale,
288
+ "index": target_indices,
289
+ },
290
+ "scene": f"Scannetpp {scene}",
291
+ }
292
+ if self.stage == "train" and self.cfg.augment:
293
+ example = apply_augmentation_shim(example)
294
+
295
+ if self.stage == "train" and self.cfg.intr_augment:
296
+ intr_aug = True
297
+ else:
298
+ intr_aug = False
299
+
300
+ example = apply_crop_shim(example, (patchsize[0] * 14, patchsize[1] * 14), intr_aug=intr_aug)
301
+
302
+ # world pts
303
+ image_size = example["context"]["image"].shape[2:]
304
+ context_intrinsics = example["context"]["intrinsics"].clone().detach().numpy()
305
+ context_intrinsics[:, 0] = context_intrinsics[:, 0] * image_size[1]
306
+ context_intrinsics[:, 1] = context_intrinsics[:, 1] * image_size[0]
307
+
308
+ target_intrinsics = example["target"]["intrinsics"].clone().detach().numpy()
309
+ target_intrinsics[:, 0] = target_intrinsics[:, 0] * image_size[1]
310
+ target_intrinsics[:, 1] = target_intrinsics[:, 1] * image_size[0]
311
+
312
+ context_pts3d_list, context_valid_mask_list = [], []
313
+ target_pts3d_list, target_valid_mask_list = [], []
314
+
315
+ for i in range(len(example["context"]["depth"])):
316
+ context_pts3d, context_valid_mask = depthmap_to_absolute_camera_coordinates(example["context"]["depth"][i].numpy(), context_intrinsics[i], example["context"]["extrinsics"][i].numpy())
317
+ context_pts3d_list.append(torch.from_numpy(context_pts3d).to(torch.float32))
318
+ context_valid_mask_list.append(torch.from_numpy(context_valid_mask))
319
+
320
+ context_pts3d = torch.stack(context_pts3d_list, dim=0)
321
+ context_valid_mask = torch.stack(context_valid_mask_list, dim=0)
322
+
323
+ for i in range(len(example["target"]["depth"])):
324
+ target_pts3d, target_valid_mask = depthmap_to_absolute_camera_coordinates(example["target"]["depth"][i].numpy(), target_intrinsics[i], example["target"]["extrinsics"][i].numpy())
325
+ target_pts3d_list.append(torch.from_numpy(target_pts3d).to(torch.float32))
326
+ target_valid_mask_list.append(torch.from_numpy(target_valid_mask))
327
+
328
+ target_pts3d = torch.stack(target_pts3d_list, dim=0)
329
+ target_valid_mask = torch.stack(target_valid_mask_list, dim=0)
330
+
331
+ # normalize by context pts3d
332
+ if self.cfg.normalize_by_pts3d:
333
+ transformed_pts3d = context_pts3d[context_valid_mask]
334
+ scene_factor = transformed_pts3d.norm(dim=-1).mean().clip(min=1e-8)
335
+ context_pts3d /= scene_factor
336
+ example["context"]["depth"] /= scene_factor
337
+ example["context"]["extrinsics"][:, :3, 3] /= scene_factor
338
+
339
+ target_pts3d /= scene_factor
340
+ example["target"]["depth"] /= scene_factor
341
+ example["target"]["extrinsics"][:, :3, 3] /= scene_factor
342
+
343
+ example["context"]["pts3d"] = context_pts3d
344
+ example["target"]["pts3d"] = target_pts3d
345
+ example["context"]["valid_mask"] = context_valid_mask
346
+ example["target"]["valid_mask"] = target_valid_mask
347
+
348
+ if torch.isnan(example["context"]["depth"]).any() or torch.isinf(example["context"]["depth"]).any() or \
349
+ torch.isnan(example["context"]["extrinsics"]).any() or torch.isinf(example["context"]["extrinsics"]).any() or \
350
+ torch.isnan(example["context"]["intrinsics"]).any() or torch.isinf(example["context"]["intrinsics"]).any() or \
351
+ torch.isnan(example["target"]["depth"]).any() or torch.isinf(example["target"]["depth"]).any() or \
352
+ torch.isnan(example["target"]["extrinsics"]).any() or torch.isinf(example["target"]["extrinsics"]).any() or \
353
+ torch.isnan(example["target"]["intrinsics"]).any() or torch.isinf(example["target"]["intrinsics"]).any():
354
+ raise Exception("encounter nan or inf in context depth")
355
+
356
+ for key in ["context", "target"]:
357
+ example[key]["valid_mask"] = (torch.ones_like(example[key]["valid_mask"]) * -1).type(torch.int32)
358
+
359
+ return example
360
+
361
+
362
+ def __getitem__(self, index_tuple: tuple) -> dict:
363
+ index, num_context_views, patchsize_h = index_tuple
364
+ # generate a random patch size
365
+ patchsize_w = (self.cfg.input_image_shape[1] // 14)
366
+ try:
367
+ return self.getitem(index, num_context_views, (patchsize_h, patchsize_w))
368
+ except Exception as e:
369
+ print(f"Error: {e}")
370
+ index = np.random.randint(len(self))
371
+ return self.__getitem__((index, num_context_views, patchsize_h))
372
+
373
+ def convert_poses(
374
+ self,
375
+ poses: Float[Tensor, "batch 18"],
376
+ ) -> tuple[
377
+ Float[Tensor, "batch 4 4"], # extrinsics
378
+ Float[Tensor, "batch 3 3"], # intrinsics
379
+ ]:
380
+ b, _ = poses.shape
381
+
382
+ # Convert the intrinsics to a 3x3 normalized K matrix.
383
+ intrinsics = torch.eye(3, dtype=torch.float32)
384
+ intrinsics = repeat(intrinsics, "h w -> b h w", b=b).clone()
385
+ fx, fy, cx, cy = poses[:, :4].T
386
+ intrinsics[:, 0, 0] = fx
387
+ intrinsics[:, 1, 1] = fy
388
+ intrinsics[:, 0, 2] = cx
389
+ intrinsics[:, 1, 2] = cy
390
+
391
+ # Convert the extrinsics to a 4x4 OpenCV-style W2C matrix.
392
+ w2c = repeat(torch.eye(4, dtype=torch.float32), "h w -> b h w", b=b).clone()
393
+ w2c[:, :3] = rearrange(poses[:, 6:], "b (h w) -> b h w", h=3, w=4)
394
+ return w2c.inverse(), intrinsics
395
+
396
+ def convert_images(
397
+ self,
398
+ images: list[UInt8[Tensor, "..."]],
399
+ ) -> Float[Tensor, "batch 3 height width"]:
400
+ torch_images = []
401
+ for image in images:
402
+ image = Image.open(BytesIO(image.numpy().tobytes()))
403
+ torch_images.append(self.to_tensor(image))
404
+ return torch.stack(torch_images)
405
+
406
+ def get_bound(
407
+ self,
408
+ bound: Literal["near", "far"],
409
+ num_views: int,
410
+ ) -> Float[Tensor, " view"]:
411
+ value = torch.tensor(getattr(self, bound), dtype=torch.float32)
412
+ return repeat(value, "-> v", v=num_views)
413
+
414
+ @property
415
+ def data_stage(self) -> Stage:
416
+ if self.cfg.overfit_to_scene is not None:
417
+ return "test"
418
+ if self.stage == "val":
419
+ return "test"
420
+ return self.stage
421
+
422
+ @cached_property
423
+ def index(self) -> dict[str, Path]:
424
+ merged_index = {}
425
+ data_stages = [self.data_stage]
426
+ if self.cfg.overfit_to_scene is not None:
427
+ data_stages = ("test", "train")
428
+ for data_stage in data_stages:
429
+ for root in self.cfg.roots:
430
+ # Load the root's index.
431
+ with (root / data_stage / "index.json").open("r") as f:
432
+ index = json.load(f)
433
+ index = {k: Path(root / data_stage / v) for k, v in index.items()}
434
+
435
+ # The constituent datasets should have unique keys.
436
+ assert not (set(merged_index.keys()) & set(index.keys()))
437
+
438
+ # Merge the root's index into the main index.
439
+ merged_index = {**merged_index, **index}
440
+ return merged_index
441
+
442
+ def __len__(self) -> int:
443
+ return len(self.data_list)
src/dataset/shims/augmentation_shim.py ADDED
@@ -0,0 +1,219 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import copy
2
+ import random
3
+ import numpy as np
4
+ import torch
5
+ from jaxtyping import Float
6
+ from torch import Tensor
7
+
8
+ from ..types import AnyExample, AnyViews
9
+
10
+
11
+ def reflect_extrinsics(
12
+ extrinsics: Float[Tensor, "*batch 4 4"],
13
+ ) -> Float[Tensor, "*batch 4 4"]:
14
+ reflect = torch.eye(4, dtype=torch.float32, device=extrinsics.device)
15
+ reflect[0, 0] = -1
16
+ return reflect @ extrinsics @ reflect
17
+
18
+
19
+ def reflect_views(views: AnyViews) -> AnyViews:
20
+ if "depth" in views.keys():
21
+ return {
22
+ **views,
23
+ "image": views["image"].flip(-1),
24
+ "extrinsics": reflect_extrinsics(views["extrinsics"]),
25
+ "depth": views["depth"].flip(-1),
26
+ }
27
+ else:
28
+ return {
29
+ **views,
30
+ "image": views["image"].flip(-1),
31
+ "extrinsics": reflect_extrinsics(views["extrinsics"]),
32
+ }
33
+
34
+
35
+ def apply_augmentation_shim(
36
+ example: AnyExample,
37
+ generator: torch.Generator | None = None,
38
+ ) -> AnyExample:
39
+ """Randomly augment the training images."""
40
+ # Do not augment with 50% chance.
41
+ if torch.rand(tuple(), generator=generator) < 0.5:
42
+ return example
43
+
44
+ return {
45
+ **example,
46
+ "context": reflect_views(example["context"]),
47
+ "target": reflect_views(example["target"]),
48
+ }
49
+
50
+ def rotate_90_degrees(
51
+ image: torch.Tensor, depth_map: torch.Tensor | None, extri_opencv: torch.Tensor, intri_opencv: torch.Tensor, clockwise=True
52
+ ):
53
+ """
54
+ Rotates the input image, depth map, and camera parameters by 90 degrees.
55
+
56
+ Applies one of two 90-degree rotations:
57
+ - Clockwise
58
+ - Counterclockwise (if clockwise=False)
59
+
60
+ The extrinsic and intrinsic matrices are adjusted accordingly to maintain
61
+ correct camera geometry.
62
+
63
+ Args:
64
+ image (torch.Tensor):
65
+ Input image tensor of shape (C, H, W).
66
+ depth_map (torch.Tensor or None):
67
+ Depth map tensor of shape (H, W), or None if not available.
68
+ extri_opencv (torch.Tensor):
69
+ Extrinsic matrix (3x4) in OpenCV convention.
70
+ intri_opencv (torch.Tensor):
71
+ Intrinsic matrix (3x3).
72
+ clockwise (bool):
73
+ If True, rotates the image 90 degrees clockwise; else 90 degrees counterclockwise.
74
+
75
+ Returns:
76
+ tuple:
77
+ (
78
+ rotated_image,
79
+ rotated_depth_map,
80
+ new_extri_opencv,
81
+ new_intri_opencv
82
+ )
83
+
84
+ Where each is the updated version after the rotation.
85
+ """
86
+ image_height, image_width = image.shape[-2:]
87
+
88
+ # Rotate the image and depth map
89
+ rotated_image, rotated_depth_map = rotate_image_and_depth_rot90(image, depth_map, clockwise)
90
+ # Adjust the intrinsic matrix
91
+ new_intri_opencv = adjust_intrinsic_matrix_rot90(intri_opencv, image_width, image_height, clockwise)
92
+ # Adjust the extrinsic matrix
93
+ new_extri_opencv = adjust_extrinsic_matrix_rot90(extri_opencv, clockwise)
94
+
95
+ return (
96
+ rotated_image,
97
+ rotated_depth_map,
98
+ new_extri_opencv,
99
+ new_intri_opencv,
100
+ )
101
+
102
+
103
+ def rotate_image_and_depth_rot90(image: torch.Tensor, depth_map: torch.Tensor | None, clockwise: bool):
104
+ """
105
+ Rotates the given image and depth map by 90 degrees (clockwise or counterclockwise).
106
+
107
+ Args:
108
+ image (torch.Tensor):
109
+ Input image tensor of shape (C, H, W).
110
+ depth_map (torch.Tensor or None):
111
+ Depth map tensor of shape (H, W), or None if not available.
112
+ clockwise (bool):
113
+ If True, rotate 90 degrees clockwise; else 90 degrees counterclockwise.
114
+
115
+ Returns:
116
+ tuple:
117
+ (rotated_image, rotated_depth_map)
118
+ """
119
+ rotated_depth_map = None
120
+ if clockwise:
121
+ rotated_image = torch.rot90(image, k=-1, dims=[-2, -1])
122
+ if depth_map is not None:
123
+ rotated_depth_map = torch.rot90(depth_map, k=-1, dims=[-2, -1])
124
+ else:
125
+ rotated_image = torch.rot90(image, k=1, dims=[-2, -1])
126
+ if depth_map is not None:
127
+ rotated_depth_map = torch.rot90(depth_map, k=1, dims=[-2, -1])
128
+ return rotated_image, rotated_depth_map
129
+
130
+
131
+ def adjust_extrinsic_matrix_rot90(extri_opencv: torch.Tensor, clockwise: bool):
132
+ """
133
+ Adjusts the extrinsic matrix (3x4) for a 90-degree rotation of the image.
134
+
135
+ The rotation is in the image plane. This modifies the camera orientation
136
+ accordingly. The function applies either a clockwise or counterclockwise
137
+ 90-degree rotation.
138
+
139
+ Args:
140
+ extri_opencv (torch.Tensor):
141
+ Extrinsic matrix (3x4) in OpenCV convention.
142
+ clockwise (bool):
143
+ If True, rotate extrinsic for a 90-degree clockwise image rotation;
144
+ otherwise, counterclockwise.
145
+
146
+ Returns:
147
+ torch.Tensor:
148
+ A new 3x4 extrinsic matrix after the rotation.
149
+ """
150
+ R = extri_opencv[:3, :3]
151
+ t = extri_opencv[:3, 3]
152
+
153
+ if clockwise:
154
+ R_rotation = torch.tensor([
155
+ [0, -1, 0],
156
+ [1, 0, 0],
157
+ [0, 0, 1]
158
+ ], dtype=extri_opencv.dtype, device=extri_opencv.device)
159
+ else:
160
+ R_rotation = torch.tensor([
161
+ [0, 1, 0],
162
+ [-1, 0, 0],
163
+ [0, 0, 1]
164
+ ], dtype=extri_opencv.dtype, device=extri_opencv.device)
165
+
166
+ new_R = torch.matmul(R_rotation, R)
167
+ new_t = torch.matmul(R_rotation, t)
168
+ new_extri_opencv = torch.cat((new_R, new_t.reshape(-1, 1)), dim=1)
169
+ new_extri_opencv = torch.cat((new_extri_opencv,
170
+ torch.tensor([[0, 0, 0, 1]],
171
+ dtype=extri_opencv.dtype, device=extri_opencv.device)), dim=0)
172
+ return new_extri_opencv
173
+
174
+
175
+ def adjust_intrinsic_matrix_rot90(intri_opencv: torch.Tensor, image_width: int, image_height: int, clockwise: bool):
176
+ """
177
+ Adjusts the intrinsic matrix (3x3) for a 90-degree rotation of the image in the image plane.
178
+
179
+ Args:
180
+ intri_opencv (torch.Tensor):
181
+ Intrinsic matrix (3x3).
182
+ image_width (int):
183
+ Original width of the image.
184
+ image_height (int):
185
+ Original height of the image.
186
+ clockwise (bool):
187
+ If True, rotate 90 degrees clockwise; else 90 degrees counterclockwise.
188
+
189
+ Returns:
190
+ torch.Tensor:
191
+ A new 3x3 intrinsic matrix after the rotation.
192
+ """
193
+ intri_opencv = copy.deepcopy(intri_opencv)
194
+ intri_opencv[0, :] *= image_width
195
+ intri_opencv[1, :] *= image_height
196
+
197
+ fx, fy, cx, cy = (
198
+ intri_opencv[0, 0],
199
+ intri_opencv[1, 1],
200
+ intri_opencv[0, 2],
201
+ intri_opencv[1, 2],
202
+ )
203
+
204
+ new_intri_opencv = torch.eye(3, dtype=intri_opencv.dtype, device=intri_opencv.device)
205
+ if clockwise:
206
+ new_intri_opencv[0, 0] = fy
207
+ new_intri_opencv[1, 1] = fx
208
+ new_intri_opencv[0, 2] = image_height - cy
209
+ new_intri_opencv[1, 2] = cx
210
+ else:
211
+ new_intri_opencv[0, 0] = fy
212
+ new_intri_opencv[1, 1] = fx
213
+ new_intri_opencv[0, 2] = cy
214
+ new_intri_opencv[1, 2] = image_width - cx
215
+
216
+ new_intri_opencv[0, :] /= image_height
217
+ new_intri_opencv[1, :] /= image_width
218
+
219
+ return new_intri_opencv
src/dataset/shims/bounds_shim.py ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from einops import einsum, reduce, repeat
3
+ from jaxtyping import Float
4
+ from torch import Tensor
5
+
6
+ from ..types import BatchedExample
7
+
8
+
9
+ def compute_depth_for_disparity(
10
+ extrinsics: Float[Tensor, "batch view 4 4"],
11
+ intrinsics: Float[Tensor, "batch view 3 3"],
12
+ image_shape: tuple[int, int],
13
+ disparity: float,
14
+ delta_min: float = 1e-6, # This prevents motionless scenes from lacking depth.
15
+ ) -> Float[Tensor, " batch"]:
16
+ """Compute the depth at which moving the maximum distance between cameras
17
+ corresponds to the specified disparity (in pixels).
18
+ """
19
+
20
+ # Use the furthest distance between cameras as the baseline.
21
+ origins = extrinsics[:, :, :3, 3]
22
+ deltas = (origins[:, None, :, :] - origins[:, :, None, :]).norm(dim=-1)
23
+ deltas = deltas.clip(min=delta_min)
24
+ baselines = reduce(deltas, "b v ov -> b", "max")
25
+
26
+ # Compute a single pixel's size at depth 1.
27
+ h, w = image_shape
28
+ pixel_size = 1 / torch.tensor((w, h), dtype=torch.float32, device=extrinsics.device)
29
+ pixel_size = einsum(
30
+ intrinsics[..., :2, :2].inverse(), pixel_size, "... i j, j -> ... i"
31
+ )
32
+
33
+ # This wouldn't make sense with non-square pixels, but then again, non-square pixels
34
+ # don't make much sense anyway.
35
+ mean_pixel_size = reduce(pixel_size, "b v xy -> b", "mean")
36
+
37
+ return baselines / (disparity * mean_pixel_size)
38
+
39
+
40
+ def apply_bounds_shim(
41
+ batch: BatchedExample,
42
+ near_disparity: float,
43
+ far_disparity: float,
44
+ ) -> BatchedExample:
45
+ """Compute reasonable near and far planes (lower and upper bounds on depth). This
46
+ assumes that all of an example's views are of roughly the same thing.
47
+ """
48
+
49
+ context = batch["context"]
50
+ _, cv, _, h, w = context["image"].shape
51
+
52
+ # Compute near and far planes using the context views.
53
+ near = compute_depth_for_disparity(
54
+ context["extrinsics"],
55
+ context["intrinsics"],
56
+ (h, w),
57
+ near_disparity,
58
+ )
59
+ far = compute_depth_for_disparity(
60
+ context["extrinsics"],
61
+ context["intrinsics"],
62
+ (h, w),
63
+ far_disparity,
64
+ )
65
+
66
+ target = batch["target"]
67
+ _, tv, _, _, _ = target["image"].shape
68
+ return {
69
+ **batch,
70
+ "context": {
71
+ **context,
72
+ "near": repeat(near, "b -> b v", v=cv),
73
+ "far": repeat(far, "b -> b v", v=cv),
74
+ },
75
+ "target": {
76
+ **target,
77
+ "near": repeat(near, "b -> b v", v=tv),
78
+ "far": repeat(far, "b -> b v", v=tv),
79
+ },
80
+ }
src/dataset/shims/crop_shim.py ADDED
@@ -0,0 +1,196 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import random
2
+ import numpy as np
3
+ import torch
4
+ from einops import rearrange
5
+ from jaxtyping import Float
6
+ from PIL import Image
7
+ from torch import Tensor
8
+ import torchvision.transforms.functional as F
9
+ import cv2
10
+
11
+ from ..types import AnyExample, AnyViews
12
+
13
+
14
+ def rescale(
15
+ image: Float[Tensor, "3 h_in w_in"],
16
+ shape: tuple[int, int],
17
+ ) -> Float[Tensor, "3 h_out w_out"]:
18
+ h, w = shape
19
+ image_new = (image * 255).clip(min=0, max=255).type(torch.uint8)
20
+ image_new = rearrange(image_new, "c h w -> h w c").detach().cpu().numpy()
21
+ image_new = Image.fromarray(image_new)
22
+ image_new = image_new.resize((w, h), Image.LANCZOS)
23
+ image_new = np.array(image_new) / 255
24
+ image_new = torch.tensor(image_new, dtype=image.dtype, device=image.device)
25
+ return rearrange(image_new, "h w c -> c h w")
26
+
27
+ def rescale_depth(
28
+ depth: Float[Tensor, "1 h w"],
29
+ shape: tuple[int, int],
30
+ ) -> Float[Tensor, "1 h_out w_out"]:
31
+ h, w = shape
32
+ depth_new = depth.detach().cpu().numpy()
33
+ depth_new = cv2.resize(depth_new, (w,h), interpolation=cv2.INTER_NEAREST)
34
+ depth_new = torch.from_numpy(depth_new).to(depth.device)
35
+ return depth_new
36
+
37
+ def center_crop(
38
+ images: Float[Tensor, "*#batch c h w"],
39
+ intrinsics: Float[Tensor, "*#batch 3 3"],
40
+ shape: tuple[int, int],
41
+ depths: Float[Tensor, "*#batch 1 h w"] | None = None,
42
+ ) -> tuple[
43
+ Float[Tensor, "*#batch c h_out w_out"], # updated images
44
+ Float[Tensor, "*#batch 3 3"], # updated intrinsics
45
+ Float[Tensor, "*#batch 1 h_out w_out"] | None, # updated depths
46
+ ]:
47
+ *_, h_in, w_in = images.shape
48
+ h_out, w_out = shape
49
+
50
+ # Note that odd input dimensions induce half-pixel misalignments.
51
+ row = (h_in - h_out) // 2
52
+ col = (w_in - w_out) // 2
53
+
54
+ # Center-crop the image.
55
+ images = images[..., :, row : row + h_out, col : col + w_out]
56
+
57
+ if depths is not None:
58
+ depths = depths[..., row : row + h_out, col : col + w_out]
59
+
60
+ # Adjust the intrinsics to account for the cropping.
61
+ intrinsics = intrinsics.clone()
62
+ intrinsics[..., 0, 0] *= w_in / w_out # fx
63
+ intrinsics[..., 1, 1] *= h_in / h_out # fy
64
+
65
+
66
+ if depths is not None:
67
+ return images, intrinsics, depths
68
+ else:
69
+ return images, intrinsics
70
+
71
+
72
+ def rescale_and_crop(
73
+ images: Float[Tensor, "*#batch c h w"],
74
+ intrinsics: Float[Tensor, "*#batch 3 3"],
75
+ shape: tuple[int, int],
76
+ intr_aug: bool = False,
77
+ scale_range: tuple[float, float] = (0.77, 1.0),
78
+ depths: Float[Tensor, "*#batch 1 h w"] | None = None,
79
+ ) -> tuple[
80
+ Float[Tensor, "*#batch c h_out w_out"], # updated images
81
+ Float[Tensor, "*#batch 3 3"], # updated intrinsics
82
+ Float[Tensor, "*#batch 1 h_out w_out"] | None, # updated depths
83
+ ]:
84
+ if type(images) == list:
85
+ images_new = []
86
+ intrinsics_new = []
87
+ for i in range(len(images)):
88
+ image = images[i]
89
+ intrinsic = intrinsics[i]
90
+
91
+ *_, h_in, w_in = image.shape
92
+ h_out, w_out = shape
93
+
94
+ scale_factor = max(h_out / h_in, w_out / w_in)
95
+ h_scaled = round(h_in * scale_factor)
96
+ w_scaled = round(w_in * scale_factor)
97
+ image = F.resize(image, (h_scaled, w_scaled))
98
+ image = F.center_crop(image, (h_out, w_out))
99
+ images_new.append(image)
100
+
101
+ intrinsic_new = intrinsic.clone()
102
+ intrinsic_new[..., 0, 0] *= w_scaled / w_in # fx
103
+ intrinsic_new[..., 1, 1] *= h_scaled / h_in # fy
104
+ intrinsics_new.append(intrinsic_new)
105
+
106
+ if depths is not None:
107
+ depths_new = []
108
+ for i in range(len(depths)):
109
+ depth = depths[i]
110
+ depth = rescale_depth(depth, (h_out, w_out))
111
+ depth = F.center_crop(depth, (h_out, w_out))
112
+ depths_new.append(depth)
113
+ return torch.stack(images_new), torch.stack(intrinsics_new), torch.stack(depths_new)
114
+ else:
115
+ return torch.stack(images_new), torch.stack(intrinsics_new)
116
+
117
+ else:
118
+ # we only support intr_aug for clean datasets
119
+ *_, h_in, w_in = images.shape
120
+ h_out, w_out = shape
121
+ # assert h_out <= h_in and w_out <= w_in # to avoid the case that the image is too small, like co3d
122
+
123
+ if intr_aug:
124
+ scale = random.uniform(*scale_range)
125
+ h_scale = round(h_out * scale)
126
+ w_scale = round(w_out * scale)
127
+ else:
128
+ h_scale = h_out
129
+ w_scale = w_out
130
+
131
+ scale_factor = max(h_scale / h_in, w_scale / w_in)
132
+ h_scaled = round(h_in * scale_factor)
133
+ w_scaled = round(w_in * scale_factor)
134
+ assert h_scaled == h_scale or w_scaled == w_scale
135
+
136
+ # Reshape the images to the correct size. Assume we don't have to worry about
137
+ # changing the intrinsics based on how the images are rounded.
138
+ *batch, c, h, w = images.shape
139
+ images = images.reshape(-1, c, h, w)
140
+ images = torch.stack([rescale(image, (h_scaled, w_scaled)) for image in images])
141
+ images = images.reshape(*batch, c, h_scaled, w_scaled)
142
+
143
+ if depths is not None:
144
+ if type(depths) == list:
145
+ depths_new = []
146
+ for i in range(len(depths)):
147
+ depth = depths[i]
148
+ depth = rescale_depth(depth, (h_scaled, w_scaled))
149
+ depths_new.append(depth)
150
+ depths = torch.stack(depths_new)
151
+ else:
152
+ depths = depths.reshape(-1, h, w)
153
+ depths = torch.stack([rescale_depth(depth, (h_scaled, w_scaled)) for depth in depths])
154
+ depths = depths.reshape(*batch, h_scaled, w_scaled)
155
+
156
+ images, intrinsics, depths = center_crop(images, intrinsics, (h_scale, w_scale), depths)
157
+
158
+ if intr_aug:
159
+ images = F.resize(images, size=(h_out, w_out), interpolation=F.InterpolationMode.BILINEAR)
160
+ depths = F.resize(depths, size=(h_out, w_out), interpolation=F.InterpolationMode.NEAREST)
161
+
162
+ return images, intrinsics, depths
163
+ else:
164
+ images, intrinsics = center_crop(images, intrinsics, (h_scale, w_scale))
165
+
166
+ if intr_aug:
167
+ images = F.resize(images, size=(h_out, w_out))
168
+
169
+ return images, intrinsics
170
+
171
+
172
+ def apply_crop_shim_to_views(views: AnyViews, shape: tuple[int, int], intr_aug: bool = False) -> AnyViews:
173
+ if "depth" in views.keys():
174
+ images, intrinsics, depths = rescale_and_crop(views["image"], views["intrinsics"], shape, depths=views["depth"], intr_aug=intr_aug)
175
+ return {
176
+ **views,
177
+ "image": images,
178
+ "intrinsics": intrinsics,
179
+ "depth": depths,
180
+ }
181
+ else:
182
+ images, intrinsics = rescale_and_crop(views["image"], views["intrinsics"], shape, intr_aug)
183
+ return {
184
+ **views,
185
+ "image": images,
186
+ "intrinsics": intrinsics,
187
+ }
188
+
189
+
190
+ def apply_crop_shim(example: AnyExample, shape: tuple[int, int], intr_aug: bool = False) -> AnyExample:
191
+ """Crop images in the example."""
192
+ return {
193
+ **example,
194
+ "context": apply_crop_shim_to_views(example["context"], shape, intr_aug),
195
+ "target": apply_crop_shim_to_views(example["target"], shape, intr_aug),
196
+ }
src/dataset/shims/geometry_shim.py ADDED
@@ -0,0 +1,383 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (C) 2024-present Naver Corporation. All rights reserved.
2
+ # Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
3
+ #
4
+ # --------------------------------------------------------
5
+ # geometry utilitary functions
6
+ # --------------------------------------------------------
7
+ import torch
8
+ import numpy as np
9
+ from scipy.spatial import cKDTree as KDTree
10
+
11
+ def invalid_to_nans(arr, valid_mask, ndim=999):
12
+ if valid_mask is not None:
13
+ arr = arr.clone()
14
+ arr[~valid_mask] = float('nan')
15
+ if arr.ndim > ndim:
16
+ arr = arr.flatten(-2 - (arr.ndim - ndim), -2)
17
+ return arr
18
+
19
+
20
+ def invalid_to_zeros(arr, valid_mask, ndim=999):
21
+ if valid_mask is not None:
22
+ arr = arr.clone()
23
+ arr[~valid_mask] = 0
24
+ nnz = valid_mask.view(len(valid_mask), -1).sum(1)
25
+ else:
26
+ nnz = arr.numel() // len(arr) if len(arr) else 0 # number of point per image
27
+ if arr.ndim > ndim:
28
+ arr = arr.flatten(-2 - (arr.ndim - ndim), -2)
29
+ return arr, nnz
30
+
31
+
32
+ def xy_grid(W, H, device=None, origin=(0, 0), unsqueeze=None, cat_dim=-1, homogeneous=False, **arange_kw):
33
+ """ Output a (H,W,2) array of int32
34
+ with output[j,i,0] = i + origin[0]
35
+ output[j,i,1] = j + origin[1]
36
+ """
37
+ if device is None:
38
+ # numpy
39
+ arange, meshgrid, stack, ones = np.arange, np.meshgrid, np.stack, np.ones
40
+ else:
41
+ # torch
42
+ arange = lambda *a, **kw: torch.arange(*a, device=device, **kw)
43
+ meshgrid, stack = torch.meshgrid, torch.stack
44
+ ones = lambda *a: torch.ones(*a, device=device)
45
+
46
+ tw, th = [arange(o, o + s, **arange_kw) for s, o in zip((W, H), origin)]
47
+ grid = meshgrid(tw, th, indexing='xy')
48
+ if homogeneous:
49
+ grid = grid + (ones((H, W)),)
50
+ if unsqueeze is not None:
51
+ grid = (grid[0].unsqueeze(unsqueeze), grid[1].unsqueeze(unsqueeze))
52
+ if cat_dim is not None:
53
+ grid = stack(grid, cat_dim)
54
+ return grid
55
+
56
+
57
+ def geotrf(Trf, pts, ncol=None, norm=False):
58
+ """ Apply a geometric transformation to a list of 3-D points.
59
+
60
+ H: 3x3 or 4x4 projection matrix (typically a Homography)
61
+ p: numpy/torch/tuple of coordinates. Shape must be (...,2) or (...,3)
62
+
63
+ ncol: int. number of columns of the result (2 or 3)
64
+ norm: float. if != 0, the resut is projected on the z=norm plane.
65
+
66
+ Returns an array of projected 2d points.
67
+ """
68
+ assert Trf.ndim >= 2
69
+ if isinstance(Trf, np.ndarray):
70
+ pts = np.asarray(pts)
71
+ elif isinstance(Trf, torch.Tensor):
72
+ pts = torch.as_tensor(pts, dtype=Trf.dtype)
73
+
74
+ # adapt shape if necessary
75
+ output_reshape = pts.shape[:-1]
76
+ ncol = ncol or pts.shape[-1]
77
+
78
+ # optimized code
79
+ if (isinstance(Trf, torch.Tensor) and isinstance(pts, torch.Tensor) and
80
+ Trf.ndim == 3 and pts.ndim == 4):
81
+ d = pts.shape[3]
82
+ if Trf.shape[-1] == d:
83
+ pts = torch.einsum("bij, bhwj -> bhwi", Trf, pts)
84
+ elif Trf.shape[-1] == d + 1:
85
+ pts = torch.einsum("bij, bhwj -> bhwi", Trf[:, :d, :d], pts) + Trf[:, None, None, :d, d]
86
+ else:
87
+ raise ValueError(f'bad shape, not ending with 3 or 4, for {pts.shape=}')
88
+ else:
89
+ if Trf.ndim >= 3:
90
+ n = Trf.ndim - 2
91
+ assert Trf.shape[:n] == pts.shape[:n], 'batch size does not match'
92
+ Trf = Trf.reshape(-1, Trf.shape[-2], Trf.shape[-1])
93
+
94
+ if pts.ndim > Trf.ndim:
95
+ # Trf == (B,d,d) & pts == (B,H,W,d) --> (B, H*W, d)
96
+ pts = pts.reshape(Trf.shape[0], -1, pts.shape[-1])
97
+ elif pts.ndim == 2:
98
+ # Trf == (B,d,d) & pts == (B,d) --> (B, 1, d)
99
+ pts = pts[:, None, :]
100
+
101
+ if pts.shape[-1] + 1 == Trf.shape[-1]:
102
+ Trf = Trf.swapaxes(-1, -2) # transpose Trf
103
+ pts = pts @ Trf[..., :-1, :] + Trf[..., -1:, :]
104
+ elif pts.shape[-1] == Trf.shape[-1]:
105
+ Trf = Trf.swapaxes(-1, -2) # transpose Trf
106
+ pts = pts @ Trf
107
+ else:
108
+ pts = Trf @ pts.T
109
+ if pts.ndim >= 2:
110
+ pts = pts.swapaxes(-1, -2)
111
+
112
+ if norm:
113
+ pts = pts / pts[..., -1:] # DONT DO /= BECAUSE OF WEIRD PYTORCH BUG
114
+ if norm != 1:
115
+ pts *= norm
116
+
117
+ res = pts[..., :ncol].reshape(*output_reshape, ncol)
118
+ return res
119
+
120
+
121
+ def inv(mat):
122
+ """ Invert a torch or numpy matrix
123
+ """
124
+ if isinstance(mat, torch.Tensor):
125
+ return torch.linalg.inv(mat)
126
+ if isinstance(mat, np.ndarray):
127
+ return np.linalg.inv(mat)
128
+ raise ValueError(f'bad matrix type = {type(mat)}')
129
+
130
+
131
+ def depthmap_to_pts3d(depth, pseudo_focal, pp=None, **_):
132
+ """
133
+ Args:
134
+ - depthmap (BxHxW array):
135
+ - pseudo_focal: [B,H,W] ; [B,2,H,W] or [B,1,H,W]
136
+ Returns:
137
+ pointmap of absolute coordinates (BxHxWx3 array)
138
+ """
139
+
140
+ if len(depth.shape) == 4:
141
+ B, H, W, n = depth.shape
142
+ else:
143
+ B, H, W = depth.shape
144
+ n = None
145
+
146
+ if len(pseudo_focal.shape) == 3: # [B,H,W]
147
+ pseudo_focalx = pseudo_focaly = pseudo_focal
148
+ elif len(pseudo_focal.shape) == 4: # [B,2,H,W] or [B,1,H,W]
149
+ pseudo_focalx = pseudo_focal[:, 0]
150
+ if pseudo_focal.shape[1] == 2:
151
+ pseudo_focaly = pseudo_focal[:, 1]
152
+ else:
153
+ pseudo_focaly = pseudo_focalx
154
+ else:
155
+ raise NotImplementedError("Error, unknown input focal shape format.")
156
+
157
+ assert pseudo_focalx.shape == depth.shape[:3]
158
+ assert pseudo_focaly.shape == depth.shape[:3]
159
+ grid_x, grid_y = xy_grid(W, H, cat_dim=0, device=depth.device)[:, None]
160
+
161
+ # set principal point
162
+ if pp is None:
163
+ grid_x = grid_x - (W - 1) / 2
164
+ grid_y = grid_y - (H - 1) / 2
165
+ else:
166
+ grid_x = grid_x.expand(B, -1, -1) - pp[:, 0, None, None]
167
+ grid_y = grid_y.expand(B, -1, -1) - pp[:, 1, None, None]
168
+
169
+ if n is None:
170
+ pts3d = torch.empty((B, H, W, 3), device=depth.device)
171
+ pts3d[..., 0] = depth * grid_x / pseudo_focalx
172
+ pts3d[..., 1] = depth * grid_y / pseudo_focaly
173
+ pts3d[..., 2] = depth
174
+ else:
175
+ pts3d = torch.empty((B, H, W, 3, n), device=depth.device)
176
+ pts3d[..., 0, :] = depth * (grid_x / pseudo_focalx)[..., None]
177
+ pts3d[..., 1, :] = depth * (grid_y / pseudo_focaly)[..., None]
178
+ pts3d[..., 2, :] = depth
179
+ return pts3d
180
+
181
+
182
+ def depthmap_to_camera_coordinates(depthmap, camera_intrinsics, pseudo_focal=None):
183
+ """
184
+ Args:
185
+ - depthmap (HxW array):
186
+ - camera_intrinsics: a 3x3 matrix
187
+ Returns:
188
+ pointmap of absolute coordinates (HxWx3 array), and a mask specifying valid pixels.
189
+ """
190
+ camera_intrinsics = np.float32(camera_intrinsics)
191
+ H, W = depthmap.shape
192
+
193
+ # Compute 3D ray associated with each pixel
194
+ # Strong assumption: there are no skew terms
195
+ assert camera_intrinsics[0, 1] == 0.0
196
+ assert camera_intrinsics[1, 0] == 0.0
197
+ if pseudo_focal is None:
198
+ fu = camera_intrinsics[0, 0]
199
+ fv = camera_intrinsics[1, 1]
200
+ else:
201
+ assert pseudo_focal.shape == (H, W)
202
+ fu = fv = pseudo_focal
203
+ cu = camera_intrinsics[0, 2]
204
+ cv = camera_intrinsics[1, 2]
205
+
206
+ u, v = np.meshgrid(np.arange(W), np.arange(H))
207
+ z_cam = depthmap
208
+ x_cam = (u - cu) * z_cam / fu
209
+ y_cam = (v - cv) * z_cam / fv
210
+ X_cam = np.stack((x_cam, y_cam, z_cam), axis=-1).astype(np.float32)
211
+
212
+ # Mask for valid coordinates
213
+ valid_mask = (depthmap > 0.0)
214
+ return X_cam, valid_mask
215
+
216
+
217
+ def depthmap_to_absolute_camera_coordinates(depthmap, camera_intrinsics, camera_pose=None, **kw):
218
+ """
219
+ Args:
220
+ - depthmap (HxW array):
221
+ - camera_intrinsics: a 3x3 matrix
222
+ - camera_pose: a 4x3 or 4x4 cam2world matrix
223
+ Returns:
224
+ pointmap of absolute coordinates (HxWx3 array), and a mask specifying valid pixels."""
225
+ X_cam, valid_mask = depthmap_to_camera_coordinates(depthmap, camera_intrinsics)
226
+
227
+ X_world = X_cam # default
228
+ if camera_pose is not None:
229
+ # R_cam2world = np.float32(camera_params["R_cam2world"])
230
+ # t_cam2world = np.float32(camera_params["t_cam2world"]).squeeze()
231
+ R_cam2world = camera_pose[:3, :3]
232
+ t_cam2world = camera_pose[:3, 3]
233
+
234
+ # Express in absolute coordinates (invalid depth values)
235
+ X_world = np.einsum("ik, vuk -> vui", R_cam2world, X_cam) + t_cam2world[None, None, :]
236
+
237
+ return X_world, valid_mask
238
+
239
+
240
+ def colmap_to_opencv_intrinsics(K):
241
+ """
242
+ Modify camera intrinsics to follow a different convention.
243
+ Coordinates of the center of the top-left pixels are by default:
244
+ - (0.5, 0.5) in Colmap
245
+ - (0,0) in OpenCV
246
+ """
247
+ K = K.copy()
248
+ K[0, 2] -= 0.5
249
+ K[1, 2] -= 0.5
250
+ return K
251
+
252
+
253
+ def opencv_to_colmap_intrinsics(K):
254
+ """
255
+ Modify camera intrinsics to follow a different convention.
256
+ Coordinates of the center of the top-left pixels are by default:
257
+ - (0.5, 0.5) in Colmap
258
+ - (0,0) in OpenCV
259
+ """
260
+ K = K.copy()
261
+ K[0, 2] += 0.5
262
+ K[1, 2] += 0.5
263
+ return K
264
+
265
+
266
+ def normalize_pointcloud(pts1, pts2, norm_mode='avg_dis', valid1=None, valid2=None, ret_factor=False):
267
+ """ renorm pointmaps pts1, pts2 with norm_mode
268
+ """
269
+ assert pts1.ndim >= 3 and pts1.shape[-1] == 3
270
+ assert pts2 is None or (pts2.ndim >= 3 and pts2.shape[-1] == 3)
271
+ norm_mode, dis_mode = norm_mode.split('_')
272
+
273
+ if norm_mode == 'avg':
274
+ # gather all points together (joint normalization)
275
+ nan_pts1, nnz1 = invalid_to_zeros(pts1, valid1, ndim=3)
276
+ nan_pts2, nnz2 = invalid_to_zeros(pts2, valid2, ndim=3) if pts2 is not None else (None, 0)
277
+ all_pts = torch.cat((nan_pts1, nan_pts2), dim=1) if pts2 is not None else nan_pts1
278
+
279
+ # compute distance to origin
280
+ all_dis = all_pts.norm(dim=-1)
281
+ if dis_mode == 'dis':
282
+ pass # do nothing
283
+ elif dis_mode == 'log1p':
284
+ all_dis = torch.log1p(all_dis)
285
+ elif dis_mode == 'warp-log1p':
286
+ # actually warp input points before normalizing them
287
+ log_dis = torch.log1p(all_dis)
288
+ warp_factor = log_dis / all_dis.clip(min=1e-8)
289
+ H1, W1 = pts1.shape[1:-1]
290
+ pts1 = pts1 * warp_factor[:, :W1 * H1].view(-1, H1, W1, 1)
291
+ if pts2 is not None:
292
+ H2, W2 = pts2.shape[1:-1]
293
+ pts2 = pts2 * warp_factor[:, W1 * H1:].view(-1, H2, W2, 1)
294
+ all_dis = log_dis # this is their true distance afterwards
295
+ else:
296
+ raise ValueError(f'bad {dis_mode=}')
297
+
298
+ norm_factor = all_dis.sum(dim=1) / (nnz1 + nnz2 + 1e-8)
299
+ else:
300
+ # gather all points together (joint normalization)
301
+ nan_pts1 = invalid_to_nans(pts1, valid1, ndim=3)
302
+ nan_pts2 = invalid_to_nans(pts2, valid2, ndim=3) if pts2 is not None else None
303
+ all_pts = torch.cat((nan_pts1, nan_pts2), dim=1) if pts2 is not None else nan_pts1
304
+
305
+ # compute distance to origin
306
+ all_dis = all_pts.norm(dim=-1)
307
+
308
+ if norm_mode == 'avg':
309
+ norm_factor = all_dis.nanmean(dim=1)
310
+ elif norm_mode == 'median':
311
+ norm_factor = all_dis.nanmedian(dim=1).values.detach()
312
+ elif norm_mode == 'sqrt':
313
+ norm_factor = all_dis.sqrt().nanmean(dim=1)**2
314
+ else:
315
+ raise ValueError(f'bad {norm_mode=}')
316
+
317
+ norm_factor = norm_factor.clip(min=1e-8)
318
+ while norm_factor.ndim < pts1.ndim:
319
+ norm_factor.unsqueeze_(-1)
320
+
321
+ res = pts1 / norm_factor
322
+ if pts2 is not None:
323
+ res = (res, pts2 / norm_factor)
324
+ if ret_factor:
325
+ res = res + (norm_factor,)
326
+ return res
327
+
328
+
329
+ @torch.no_grad()
330
+ def get_joint_pointcloud_depth(z1, z2, valid_mask1, valid_mask2=None, quantile=0.5):
331
+ # set invalid points to NaN
332
+ _z1 = invalid_to_nans(z1, valid_mask1).reshape(len(z1), -1)
333
+ _z2 = invalid_to_nans(z2, valid_mask2).reshape(len(z2), -1) if z2 is not None else None
334
+ _z = torch.cat((_z1, _z2), dim=-1) if z2 is not None else _z1
335
+
336
+ # compute median depth overall (ignoring nans)
337
+ if quantile == 0.5:
338
+ shift_z = torch.nanmedian(_z, dim=-1).values
339
+ else:
340
+ shift_z = torch.nanquantile(_z, quantile, dim=-1)
341
+ return shift_z # (B,)
342
+
343
+
344
+ @torch.no_grad()
345
+ def get_joint_pointcloud_center_scale(pts1, pts2, valid_mask1=None, valid_mask2=None, z_only=False, center=True):
346
+ # set invalid points to NaN
347
+ _pts1 = invalid_to_nans(pts1, valid_mask1).reshape(len(pts1), -1, 3)
348
+ _pts2 = invalid_to_nans(pts2, valid_mask2).reshape(len(pts2), -1, 3) if pts2 is not None else None
349
+ _pts = torch.cat((_pts1, _pts2), dim=1) if pts2 is not None else _pts1
350
+
351
+ # compute median center
352
+ _center = torch.nanmedian(_pts, dim=1, keepdim=True).values # (B,1,3)
353
+ if z_only:
354
+ _center[..., :2] = 0 # do not center X and Y
355
+
356
+ # compute median norm
357
+ _norm = ((_pts - _center) if center else _pts).norm(dim=-1)
358
+ scale = torch.nanmedian(_norm, dim=1).values
359
+ return _center[:, None, :, :], scale[:, None, None, None]
360
+
361
+
362
+ def find_reciprocal_matches(P1, P2):
363
+ """
364
+ returns 3 values:
365
+ 1 - reciprocal_in_P2: a boolean array of size P2.shape[0], a "True" value indicates a match
366
+ 2 - nn2_in_P1: a int array of size P2.shape[0], it contains the indexes of the closest points in P1
367
+ 3 - reciprocal_in_P2.sum(): the number of matches
368
+ """
369
+ tree1 = KDTree(P1)
370
+ tree2 = KDTree(P2)
371
+
372
+ _, nn1_in_P2 = tree2.query(P1, workers=8)
373
+ _, nn2_in_P1 = tree1.query(P2, workers=8)
374
+
375
+ reciprocal_in_P1 = (nn2_in_P1[nn1_in_P2] == np.arange(len(nn1_in_P2)))
376
+ reciprocal_in_P2 = (nn1_in_P2[nn2_in_P1] == np.arange(len(nn2_in_P1)))
377
+ assert reciprocal_in_P1.sum() == reciprocal_in_P2.sum()
378
+ return reciprocal_in_P2, nn2_in_P1, reciprocal_in_P2.sum()
379
+
380
+
381
+ def get_med_dist_between_poses(poses):
382
+ from scipy.spatial.distance import pdist
383
+ return np.median(pdist([p[:3, 3].detach().cpu().numpy() for p in poses]))
src/dataset/shims/load_shim.py ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import cv2
2
+
3
+ def imread_cv2(path, options=cv2.IMREAD_COLOR):
4
+ """Open an image or a depthmap with opencv-python."""
5
+ if path.endswith((".exr", "EXR")):
6
+ options = cv2.IMREAD_ANYDEPTH
7
+ img = cv2.imread(path, options)
8
+ if img is None:
9
+ raise IOError(f"Could not load image={path} with {options=}")
10
+ if img.ndim == 3:
11
+ img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
12
+ return img
src/dataset/shims/normalize_shim.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from einops import einsum, reduce, repeat
3
+ from jaxtyping import Float
4
+ from torch import Tensor
5
+
6
+ from ..types import BatchedExample
7
+
8
+
9
+ def inverse_normalize_image(tensor, mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)):
10
+ mean = torch.as_tensor(mean, dtype=tensor.dtype, device=tensor.device).view(-1, 1, 1)
11
+ std = torch.as_tensor(std, dtype=tensor.dtype, device=tensor.device).view(-1, 1, 1)
12
+ return tensor * std + mean
13
+
14
+
15
+ def normalize_image(tensor, mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)):
16
+ mean = torch.as_tensor(mean, dtype=tensor.dtype, device=tensor.device).view(-1, 1, 1)
17
+ std = torch.as_tensor(std, dtype=tensor.dtype, device=tensor.device).view(-1, 1, 1)
18
+ return (tensor - mean) / std
19
+
20
+
21
+ def apply_normalize_shim(
22
+ batch: BatchedExample,
23
+ mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
24
+ std: tuple[float, float, float] = (0.5, 0.5, 0.5),
25
+ ) -> BatchedExample:
26
+ batch["context"]["image"] = normalize_image(batch["context"]["image"], mean, std)
27
+ return batch
src/dataset/shims/patch_shim.py ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from ..types import BatchedExample, BatchedViews
2
+
3
+
4
+ def apply_patch_shim_to_views(views: BatchedViews, patch_size: int) -> BatchedViews:
5
+ _, _, _, h, w = views["image"].shape
6
+
7
+ # Image size must be even so that naive center-cropping does not cause misalignment.
8
+ assert h % 2 == 0 and w % 2 == 0
9
+
10
+ h_new = (h // patch_size) * patch_size
11
+ row = (h - h_new) // 2
12
+ w_new = (w // patch_size) * patch_size
13
+ col = (w - w_new) // 2
14
+
15
+ # Center-crop the image.
16
+ image = views["image"][:, :, :, row : row + h_new, col : col + w_new]
17
+
18
+ # Adjust the intrinsics to account for the cropping.
19
+ intrinsics = views["intrinsics"].clone()
20
+ intrinsics[:, :, 0, 0] *= w / w_new # fx
21
+ intrinsics[:, :, 1, 1] *= h / h_new # fy
22
+
23
+ return {
24
+ **views,
25
+ "image": image,
26
+ "intrinsics": intrinsics,
27
+ }
28
+
29
+
30
+ def apply_patch_shim(batch: BatchedExample, patch_size: int) -> BatchedExample:
31
+ """Crop images in the batch so that their dimensions are cleanly divisible by the
32
+ specified patch size.
33
+ """
34
+ return {
35
+ **batch,
36
+ "context": apply_patch_shim_to_views(batch["context"], patch_size),
37
+ "target": apply_patch_shim_to_views(batch["target"], patch_size),
38
+ }
src/dataset/types.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Callable, Literal, TypedDict
2
+
3
+ from jaxtyping import Float, Int64
4
+ from torch import Tensor
5
+
6
+ Stage = Literal["train", "val", "test"]
7
+
8
+
9
+ # The following types mainly exist to make type-hinted keys show up in VS Code. Some
10
+ # dimensions are annotated as "_" because either:
11
+ # 1. They're expected to change as part of a function call (e.g., resizing the dataset).
12
+ # 2. They're expected to vary within the same function call (e.g., the number of views,
13
+ # which differs between context and target BatchedViews).
14
+
15
+
16
+ class BatchedViews(TypedDict, total=False):
17
+ extrinsics: Float[Tensor, "batch _ 4 4"] # batch view 4 4
18
+ intrinsics: Float[Tensor, "batch _ 3 3"] # batch view 3 3
19
+ image: Float[Tensor, "batch _ _ _ _"] # batch view channel height width
20
+ near: Float[Tensor, "batch _"] # batch view
21
+ far: Float[Tensor, "batch _"] # batch view
22
+ index: Int64[Tensor, "batch _"] # batch view
23
+ overlap: Float[Tensor, "batch _"] # batch view
24
+
25
+
26
+ class BatchedExample(TypedDict, total=False):
27
+ target: BatchedViews
28
+ context: BatchedViews
29
+ scene: list[str]
30
+
31
+
32
+ class UnbatchedViews(TypedDict, total=False):
33
+ extrinsics: Float[Tensor, "_ 4 4"]
34
+ intrinsics: Float[Tensor, "_ 3 3"]
35
+ image: Float[Tensor, "_ 3 height width"]
36
+ near: Float[Tensor, " _"]
37
+ far: Float[Tensor, " _"]
38
+ index: Int64[Tensor, " _"]
39
+
40
+
41
+ class UnbatchedExample(TypedDict, total=False):
42
+ target: UnbatchedViews
43
+ context: UnbatchedViews
44
+ scene: str
45
+
46
+
47
+ # A data shim modifies the example after it's been returned from the data loader.
48
+ DataShim = Callable[[BatchedExample], BatchedExample]
49
+
50
+ AnyExample = BatchedExample | UnbatchedExample
51
+ AnyViews = BatchedViews | UnbatchedViews
src/dataset/validation_wrapper.py ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Iterator, Optional
2
+
3
+ import torch
4
+ from torch.utils.data import Dataset, IterableDataset
5
+
6
+
7
+ class ValidationWrapper(Dataset):
8
+ """Wraps a dataset so that PyTorch Lightning's validation step can be turned into a
9
+ visualization step.
10
+ """
11
+
12
+ dataset: Dataset
13
+ dataset_iterator: Optional[Iterator]
14
+ length: int
15
+
16
+ def __init__(self, dataset: Dataset, length: int) -> None:
17
+ super().__init__()
18
+ self.dataset = dataset
19
+ self.length = length
20
+ self.dataset_iterator = None
21
+
22
+ def __len__(self):
23
+ return self.length
24
+
25
+ def __getitem__(self, index: tuple):
26
+ if isinstance(self.dataset, IterableDataset):
27
+ if self.dataset_iterator is None:
28
+ self.dataset_iterator = iter(self.dataset)
29
+ return next(self.dataset_iterator)
30
+
31
+ random_index = torch.randint(0, len(self.dataset), tuple())
32
+ random_context_num = torch.randint(2, self.dataset.view_sampler.num_context_views + 1, tuple())
33
+ # breakpoint()
34
+ return self.dataset[random_index.item(), random_context_num.item()]
src/dataset/view_sampler/__init__.py ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any
2
+
3
+ from ...misc.step_tracker import StepTracker
4
+ from ..types import Stage
5
+ from .view_sampler import ViewSampler
6
+ from .view_sampler_all import ViewSamplerAll, ViewSamplerAllCfg
7
+ from .view_sampler_arbitrary import ViewSamplerArbitrary, ViewSamplerArbitraryCfg
8
+ from .view_sampler_bounded import ViewSamplerBounded, ViewSamplerBoundedCfg
9
+ from .view_sampler_evaluation import ViewSamplerEvaluation, ViewSamplerEvaluationCfg
10
+ from .view_sampler_rank import ViewSamplerRank, ViewSamplerRankCfg
11
+ VIEW_SAMPLERS: dict[str, ViewSampler[Any]] = {
12
+ "all": ViewSamplerAll,
13
+ "arbitrary": ViewSamplerArbitrary,
14
+ "bounded": ViewSamplerBounded,
15
+ "evaluation": ViewSamplerEvaluation,
16
+ "rank": ViewSamplerRank,
17
+ }
18
+
19
+ ViewSamplerCfg = (
20
+ ViewSamplerArbitraryCfg
21
+ | ViewSamplerBoundedCfg
22
+ | ViewSamplerEvaluationCfg
23
+ | ViewSamplerAllCfg
24
+ | ViewSamplerRankCfg
25
+ )
26
+
27
+ def get_view_sampler(
28
+ cfg: ViewSamplerCfg,
29
+ stage: Stage,
30
+ overfit: bool,
31
+ cameras_are_circular: bool,
32
+ step_tracker: StepTracker | None,
33
+ ) -> ViewSampler[Any]:
34
+ return VIEW_SAMPLERS[cfg.name](
35
+ cfg,
36
+ stage,
37
+ overfit,
38
+ cameras_are_circular,
39
+ step_tracker,
40
+ )
src/dataset/view_sampler/three_view_hack.py ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from jaxtyping import Int
3
+ from torch import Tensor
4
+
5
+
6
+ def add_third_context_index(
7
+ indices: Int[Tensor, "*batch 2"]
8
+ ) -> Int[Tensor, "*batch 3"]:
9
+ left, right = indices.unbind(dim=-1)
10
+ return torch.stack((left, (left + right) // 2, right), dim=-1)
src/dataset/view_sampler/view_sampler.py ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from abc import ABC, abstractmethod
2
+ from typing import Generic, TypeVar
3
+
4
+ import torch
5
+ from jaxtyping import Float, Int64
6
+ from torch import Tensor
7
+
8
+ from ...misc.step_tracker import StepTracker
9
+ from ..types import Stage
10
+
11
+ T = TypeVar("T")
12
+
13
+
14
+ class ViewSampler(ABC, Generic[T]):
15
+ cfg: T
16
+ stage: Stage
17
+ is_overfitting: bool
18
+ cameras_are_circular: bool
19
+ step_tracker: StepTracker | None
20
+
21
+ def __init__(
22
+ self,
23
+ cfg: T,
24
+ stage: Stage,
25
+ is_overfitting: bool,
26
+ cameras_are_circular: bool,
27
+ step_tracker: StepTracker | None,
28
+ ) -> None:
29
+ self.cfg = cfg
30
+ self.stage = stage
31
+ self.is_overfitting = is_overfitting
32
+ self.cameras_are_circular = cameras_are_circular
33
+ self.step_tracker = step_tracker
34
+
35
+ @abstractmethod
36
+ def sample(
37
+ self,
38
+ scene: str,
39
+ extrinsics: Float[Tensor, "view 4 4"],
40
+ intrinsics: Float[Tensor, "view 3 3"],
41
+ device: torch.device = torch.device("cpu"),
42
+ ) -> tuple[
43
+ Int64[Tensor, " context_view"], # indices for context views
44
+ Int64[Tensor, " target_view"], # indices for target views
45
+ Float[Tensor, " overlap"], # overlap
46
+ ]:
47
+ pass
48
+
49
+ @property
50
+ @abstractmethod
51
+ def num_target_views(self) -> int:
52
+ pass
53
+
54
+ @property
55
+ @abstractmethod
56
+ def num_context_views(self) -> int:
57
+ pass
58
+
59
+ @property
60
+ def global_step(self) -> int:
61
+ return 0 if self.step_tracker is None else self.step_tracker.get_step()
src/dataset/view_sampler/view_sampler_all.py ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+ from typing import Literal
3
+
4
+ import torch
5
+ from jaxtyping import Float, Int64
6
+ from torch import Tensor
7
+
8
+ from .view_sampler import ViewSampler
9
+
10
+
11
+ @dataclass
12
+ class ViewSamplerAllCfg:
13
+ name: Literal["all"]
14
+
15
+
16
+ class ViewSamplerAll(ViewSampler[ViewSamplerAllCfg]):
17
+ def sample(
18
+ self,
19
+ scene: str,
20
+ extrinsics: Float[Tensor, "view 4 4"],
21
+ intrinsics: Float[Tensor, "view 3 3"],
22
+ device: torch.device = torch.device("cpu"),
23
+ ) -> tuple[
24
+ Int64[Tensor, " context_view"], # indices for context views
25
+ Int64[Tensor, " target_view"], # indices for target views
26
+ ]:
27
+ v, _, _ = extrinsics.shape
28
+ all_frames = torch.arange(v, device=device)
29
+ return all_frames, all_frames
30
+
31
+ @property
32
+ def num_context_views(self) -> int:
33
+ return 0
34
+
35
+ @property
36
+ def num_target_views(self) -> int:
37
+ return 0
src/dataset/view_sampler/view_sampler_arbitrary.py ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+ from typing import Literal
3
+
4
+ import torch
5
+ from jaxtyping import Float, Int64
6
+ from torch import Tensor
7
+
8
+ from .three_view_hack import add_third_context_index
9
+ from .view_sampler import ViewSampler
10
+
11
+
12
+ @dataclass
13
+ class ViewSamplerArbitraryCfg:
14
+ name: Literal["arbitrary"]
15
+ num_context_views: int
16
+ num_target_views: int
17
+ context_views: list[int] | None
18
+ target_views: list[int] | None
19
+
20
+
21
+ class ViewSamplerArbitrary(ViewSampler[ViewSamplerArbitraryCfg]):
22
+ def sample(
23
+ self,
24
+ scene: str,
25
+ extrinsics: Float[Tensor, "view 4 4"],
26
+ intrinsics: Float[Tensor, "view 3 3"],
27
+ device: torch.device = torch.device("cpu"),
28
+ ) -> tuple[
29
+ Int64[Tensor, " context_view"], # indices for context views
30
+ Int64[Tensor, " target_view"], # indices for target views
31
+ Float[Tensor, " overlap"], # overlap
32
+ ]:
33
+ """Arbitrarily sample context and target views."""
34
+ num_views, _, _ = extrinsics.shape
35
+
36
+ index_context = torch.randint(
37
+ 0,
38
+ num_views,
39
+ size=(self.cfg.num_context_views,),
40
+ device=device,
41
+ )
42
+
43
+ # Allow the context views to be fixed.
44
+ if self.cfg.context_views is not None:
45
+ index_context = torch.tensor(
46
+ self.cfg.context_views, dtype=torch.int64, device=device
47
+ )
48
+
49
+ if self.cfg.num_context_views == 3 and len(self.cfg.context_views) == 2:
50
+ index_context = add_third_context_index(index_context)
51
+ else:
52
+ assert len(self.cfg.context_views) == self.cfg.num_context_views
53
+ index_target = torch.randint(
54
+ 0,
55
+ num_views,
56
+ size=(self.cfg.num_target_views,),
57
+ device=device,
58
+ )
59
+
60
+ # Allow the target views to be fixed.
61
+ if self.cfg.target_views is not None:
62
+ assert len(self.cfg.target_views) == self.cfg.num_target_views
63
+ index_target = torch.tensor(
64
+ self.cfg.target_views, dtype=torch.int64, device=device
65
+ )
66
+
67
+ overlap = torch.tensor([0.5], dtype=torch.float32, device=device) # dummy
68
+
69
+ return index_context, index_target, overlap
70
+
71
+ @property
72
+ def num_context_views(self) -> int:
73
+ return self.cfg.num_context_views
74
+
75
+ @property
76
+ def num_target_views(self) -> int:
77
+ return self.cfg.num_target_views
src/dataset/view_sampler/view_sampler_bounded.py ADDED
@@ -0,0 +1,151 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+ from typing import Literal
3
+
4
+ import torch
5
+ from jaxtyping import Float, Int64
6
+ from torch import Tensor
7
+
8
+ from .view_sampler import ViewSampler
9
+
10
+
11
+ @dataclass
12
+ class ViewSamplerBoundedCfg:
13
+ name: Literal["bounded"]
14
+ num_context_views: int
15
+ num_target_views: int
16
+ min_distance_between_context_views: int
17
+ max_distance_between_context_views: int
18
+ min_distance_to_context_views: int
19
+ warm_up_steps: int
20
+ initial_min_distance_between_context_views: int
21
+ initial_max_distance_between_context_views: int
22
+ max_img_per_gpu: int
23
+ min_gap_multiplier: int
24
+ max_gap_multiplier: int
25
+
26
+ class ViewSamplerBounded(ViewSampler[ViewSamplerBoundedCfg]):
27
+ def schedule(self, initial: int, final: int) -> int:
28
+ fraction = self.global_step / self.cfg.warm_up_steps
29
+ return min(initial + int((final - initial) * fraction), final)
30
+
31
+ def sample(
32
+ self,
33
+ scene: str,
34
+ num_context_views: int,
35
+ extrinsics: Float[Tensor, "view 4 4"],
36
+ intrinsics: Float[Tensor, "view 3 3"],
37
+ device: torch.device = torch.device("cpu"),
38
+ ) -> tuple[
39
+ Int64[Tensor, " context_view"], # indices for context views
40
+ Int64[Tensor, " target_view"], # indices for target views
41
+ Float[Tensor, " overlap"], # overlap
42
+ ]:
43
+ num_views, _, _ = extrinsics.shape
44
+
45
+ # Compute the context view spacing based on the current global step.
46
+ if self.stage == "test":
47
+ # When testing, always use the full gap.
48
+ max_gap = self.cfg.max_distance_between_context_views
49
+ min_gap = self.cfg.max_distance_between_context_views
50
+ # elif self.cfg.warm_up_steps > 0:
51
+ # max_gap = self.schedule(
52
+ # self.cfg.initial_max_distance_between_context_views,
53
+ # self.cfg.max_distance_between_context_views,
54
+ # )
55
+ # min_gap = self.schedule(
56
+ # self.cfg.initial_min_distance_between_context_views,
57
+ # self.cfg.min_distance_between_context_views,
58
+ # )
59
+ # else:
60
+ # max_gap = self.cfg.max_distance_between_context_views
61
+ # min_gap = self.cfg.min_distance_between_context_views
62
+
63
+ min_gap, max_gap = self.num_ctxt_gap_mapping[num_context_views]
64
+ max_gap = min(max_gap, num_views-1)
65
+ # Pick the gap between the context views.
66
+ if not self.cameras_are_circular:
67
+ max_gap = min(num_views - 1, max_gap)
68
+ min_gap = max(2 * self.cfg.min_distance_to_context_views, min_gap)
69
+ if max_gap < min_gap:
70
+ raise ValueError("Example does not have enough frames!")
71
+ context_gap = torch.randint(
72
+ min_gap,
73
+ max_gap + 1,
74
+ size=tuple(),
75
+ device=device,
76
+ ).item()
77
+
78
+ # Pick the left and right context indices.
79
+ index_context_left = torch.randint(
80
+ num_views if self.cameras_are_circular else num_views - context_gap,
81
+ size=tuple(),
82
+ device=device,
83
+ ).item()
84
+ if self.stage == "test":
85
+ index_context_left = index_context_left * 0
86
+ index_context_right = index_context_left + context_gap
87
+
88
+ if self.is_overfitting:
89
+ index_context_left *= 0
90
+ index_context_right *= 0
91
+ index_context_right += max_gap
92
+
93
+ # Pick the target view indices.
94
+ if self.stage == "test":
95
+ # When testing, pick all.
96
+ index_target = torch.arange(
97
+ index_context_left,
98
+ index_context_right + 1,
99
+ device=device,
100
+ )
101
+ else:
102
+ # When training or validating (visualizing), pick at random.
103
+ index_target = torch.randint(
104
+ index_context_left + self.cfg.min_distance_to_context_views,
105
+ index_context_right + 1 - self.cfg.min_distance_to_context_views,
106
+ size=(self.cfg.num_target_views,),
107
+ device=device,
108
+ )
109
+
110
+ # Apply modulo for circular datasets.
111
+ if self.cameras_are_circular:
112
+ index_target %= num_views
113
+ index_context_right %= num_views
114
+
115
+ # If more than two context views are desired, pick extra context views between
116
+ # the left and right ones.
117
+ if num_context_views > 2:
118
+ num_extra_views = num_context_views - 2
119
+ extra_views = []
120
+ while len(set(extra_views)) != num_extra_views:
121
+ extra_views = torch.randint(
122
+ index_context_left + 1,
123
+ index_context_right,
124
+ (num_extra_views,),
125
+ ).tolist()
126
+ else:
127
+ extra_views = []
128
+
129
+ overlap = torch.tensor([0.5], dtype=torch.float32, device=device) # dummy
130
+
131
+ return (
132
+ torch.tensor((index_context_left, *extra_views, index_context_right)),
133
+ index_target,
134
+ overlap
135
+ )
136
+
137
+ @property
138
+ def num_context_views(self) -> int:
139
+ return self.cfg.num_context_views
140
+
141
+ @property
142
+ def num_target_views(self) -> int:
143
+ return self.cfg.num_target_views
144
+
145
+ @property
146
+ def num_ctxt_gap_mapping(self) -> dict:
147
+ mapping = dict()
148
+ for num_ctxt in range(2, self.cfg.num_context_views + 1):
149
+ mapping[num_ctxt] = [min(num_ctxt * self.cfg.min_gap_multiplier, self.cfg.min_distance_between_context_views),
150
+ min(max(num_ctxt * self.cfg.max_gap_multiplier, num_ctxt ** 2), self.cfg.max_distance_between_context_views)]
151
+ return mapping
src/dataset/view_sampler/view_sampler_evaluation.py ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ from dataclasses import dataclass
3
+ from pathlib import Path
4
+ from typing import Literal
5
+
6
+ import torch
7
+ from dacite import Config, from_dict
8
+ from jaxtyping import Float, Int64
9
+ from torch import Tensor
10
+
11
+ from ...evaluation.evaluation_index_generator import IndexEntry
12
+ from ...global_cfg import get_cfg
13
+ from ...misc.step_tracker import StepTracker
14
+ from ..types import Stage
15
+ from .three_view_hack import add_third_context_index
16
+ from .view_sampler import ViewSampler
17
+
18
+
19
+ @dataclass
20
+ class ViewSamplerEvaluationCfg:
21
+ name: Literal["evaluation"]
22
+ index_path: Path
23
+ num_context_views: int
24
+
25
+
26
+ class ViewSamplerEvaluation(ViewSampler[ViewSamplerEvaluationCfg]):
27
+ index: dict[str, IndexEntry | None]
28
+
29
+ def __init__(
30
+ self,
31
+ cfg: ViewSamplerEvaluationCfg,
32
+ stage: Stage,
33
+ is_overfitting: bool,
34
+ cameras_are_circular: bool,
35
+ step_tracker: StepTracker | None,
36
+ ) -> None:
37
+ super().__init__(cfg, stage, is_overfitting, cameras_are_circular, step_tracker)
38
+
39
+ dacite_config = Config(cast=[tuple])
40
+ with cfg.index_path.open("r") as f:
41
+ self.index = {
42
+ k: None if v is None else from_dict(IndexEntry, v, dacite_config)
43
+ for k, v in json.load(f).items()
44
+ }
45
+
46
+ def sample(
47
+ self,
48
+ scene: str,
49
+ extrinsics: Float[Tensor, "view 4 4"],
50
+ intrinsics: Float[Tensor, "view 3 3"],
51
+ device: torch.device = torch.device("cpu"),
52
+ **kwargs,
53
+ ) -> tuple[
54
+ Int64[Tensor, " context_view"], # indices for context views
55
+ Int64[Tensor, " target_view"], # indices for target views
56
+ ]:
57
+ entry = self.index.get(scene)
58
+ if entry is None:
59
+ raise ValueError(f"No indices available for scene {scene}.")
60
+ context_indices = torch.tensor(entry.context, dtype=torch.int64, device=device)
61
+ target_indices = torch.tensor(entry.target, dtype=torch.int64, device=device)
62
+ return context_indices, target_indices, torch.zeros(1)
63
+
64
+ @property
65
+ def num_context_views(self) -> int:
66
+ return 0
67
+
68
+ @property
69
+ def num_target_views(self) -> int:
70
+ return 0
src/dataset/view_sampler/view_sampler_rank.py ADDED
@@ -0,0 +1,263 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ from dataclasses import dataclass
3
+ from typing import Literal
4
+
5
+ import torch
6
+ import copy
7
+ from jaxtyping import Float, Int64
8
+ from torch import Tensor
9
+ import random
10
+ from .view_sampler import ViewSampler
11
+
12
+
13
+ @dataclass
14
+ class ViewSamplerRankCfg:
15
+ name: Literal["rank"]
16
+ num_context_views: int # max number of context views
17
+ num_target_views: int
18
+ min_distance_between_context_views: int
19
+ max_distance_between_context_views: int
20
+ min_distance_to_context_views: int
21
+ warm_up_steps: int
22
+ initial_min_distance_between_context_views: int
23
+ initial_max_distance_between_context_views: int
24
+ max_img_per_gpu: int
25
+
26
+
27
+ def rotation_angle(R1, R2):
28
+ # R1 and R2 are 3x3 rotation matrices
29
+ R = R1.T @ R2
30
+ # Numerical stability: clamp values into [-1,1]
31
+ val = (torch.trace(R) - 1) / 2
32
+ val = torch.clamp(val, -1.0, 1.0)
33
+ angle_rad = torch.acos(val)
34
+ angle_deg = angle_rad * 180 / torch.pi # Convert radians to degrees
35
+ return angle_deg
36
+
37
+ def extrinsic_distance(extrinsic1, extrinsic2, lambda_t=1.0):
38
+ R1, t1 = extrinsic1[:3, :3], extrinsic1[:3, 3]
39
+ R2, t2 = extrinsic2[:3, :3], extrinsic2[:3, 3]
40
+ rot_diff = rotation_angle(R1, R2) / 180
41
+
42
+ center_diff = torch.norm(t1 - t2)
43
+ return rot_diff + lambda_t * center_diff
44
+
45
+ def rotation_angle_batch(R1, R2):
46
+ # R1, R2: shape (N, 3, 3)
47
+ # We want a matrix of rotation angles for all pairs.
48
+ # We'll get R1^T R2 for each pair.
49
+ # Expand dimensions to broadcast:
50
+ # R1^T: (N,3,3) -> (N,1,3,3)
51
+ # R2: (N,3,3) -> (1,N,3,3)
52
+ R1_t = R1.transpose(-2, -1)[:, None, :, :] # shape (N,1,3,3)
53
+ R2_b = R2[None, :, :, :] # shape (1,N,3,3)
54
+ R_mult = torch.matmul(R1_t, R2_b) # shape (N,N,3,3)
55
+ # trace(R) for each pair
56
+ trace_vals = R_mult[..., 0, 0] + R_mult[..., 1, 1] + R_mult[..., 2, 2] # (N,N)
57
+ val = (trace_vals - 1) / 2
58
+ val = torch.clamp(val, -1.0, 1.0)
59
+ angle_rad = torch.acos(val)
60
+ angle_deg = angle_rad * 180 / torch.pi
61
+ return angle_deg / 180.0 # normalized rotation difference
62
+
63
+ def extrinsic_distance_batch(extrinsics, lambda_t=1.0):
64
+ # extrinsics: (N,4,4)
65
+ # Extract rotation and translation
66
+ R = extrinsics[:, :3, :3] # (N,3,3)
67
+ t = extrinsics[:, :3, 3] # (N,3)
68
+ # Compute all pairwise rotation differences
69
+ rot_diff = rotation_angle_batch(R, R) # (N,N)
70
+ # Compute all pairwise translation differences
71
+ # For t, shape (N,3). We want all pair differences: t[i] - t[j].
72
+ # t_i: (N,1,3), t_j: (1,N,3)
73
+ t_i = t[:, None, :] # (N,1,3)
74
+ t_j = t[None, :, :] # (1,N,3)
75
+ trans_diff = torch.norm(t_i - t_j, dim=2) # (N,N)
76
+ dists = rot_diff + lambda_t * trans_diff
77
+ return dists
78
+
79
+
80
+ def compute_ranking(extrinsics, lambda_t=1.0, normalize=True, batched=True):
81
+
82
+ if normalize:
83
+ extrinsics = copy.deepcopy(extrinsics)
84
+ camera_center = copy.deepcopy(extrinsics[:, :3, 3])
85
+ camera_center_scale = torch.norm(camera_center, dim=1)
86
+ avg_scale = torch.mean(camera_center_scale)
87
+ extrinsics[:, :3, 3] = extrinsics[:, :3, 3] / avg_scale
88
+
89
+
90
+ if batched:
91
+ dists = extrinsic_distance_batch(extrinsics, lambda_t=lambda_t)
92
+ else:
93
+ N = extrinsics.shape[0]
94
+ dists = torch.zeros((N, N), device=extrinsics.device)
95
+ for i in range(N):
96
+ for j in range(N):
97
+ dists[i,j] = extrinsic_distance(extrinsics[i], extrinsics[j], lambda_t=lambda_t)
98
+ ranking = torch.argsort(dists, dim=1)
99
+ return ranking, dists
100
+
101
+ # class ViewSamplerRank(ViewSampler[ViewSamplerRankCfg]):
102
+ # def schedule(self, initial: int, final: int) -> int:
103
+ # fraction = self.global_step / self.cfg.warm_up_steps
104
+ # return min(initial + int((final - initial) * fraction), final)
105
+
106
+ # def sample(
107
+ # self,
108
+ # scene: str,
109
+ # extrinsics: Float[Tensor, "view 4 4"],
110
+ # intrinsics: Float[Tensor, "view 3 3"],
111
+ # device: torch.device = torch.device("cpu"),
112
+ # ) -> tuple[
113
+ # Int64[Tensor, " context_view"], # indices for context views
114
+ # Int64[Tensor, " target_view"], # indices for target views
115
+ # Float[Tensor, " overlap"], # overlap
116
+ # ]:
117
+ # num_views, _, _ = extrinsics.shape
118
+ # # breakpoint()
119
+ # # Compute the context view spacing based on the current global step.
120
+ # ranking, dists = compute_ranking(extrinsics, lambda_t=1.0, normalize=True, batched=True)
121
+ # reference_view = random.sample(range(num_views), 1)[0]
122
+
123
+ # refview_ranking = ranking[reference_view]
124
+ # # if self.cfg.warm_up_steps > 0:
125
+ # # max_gap = self.schedule(
126
+ # # self.cfg.initial_max_distance_between_context_views,
127
+ # # self.cfg.max_distance_between_context_views,
128
+ # # )
129
+ # # min_gap = self.schedule(
130
+ # # self.cfg.initial_min_distance_between_context_views,
131
+ # # self.cfg.min_distance_between_context_views,
132
+ # # )
133
+ # # else:
134
+ # max_gap = self.cfg.max_distance_between_context_views
135
+ # min_gap = self.cfg.min_distance_between_context_views
136
+
137
+ # index_context_left = reference_view
138
+ # rightmost_index = random.sample(range(min_gap, max_gap + 1), 1)[0] + 1
139
+ # index_context_right = refview_ranking[rightmost_index].item()
140
+
141
+ # middle_indices = refview_ranking[1: rightmost_index].tolist()
142
+ # index_target = random.sample(middle_indices, self.num_target_views)
143
+
144
+ # remaining_indices = [idx for idx in middle_indices if idx not in index_target]
145
+
146
+ # # Sample extra context views if needed
147
+ # extra_views = []
148
+ # num_extra_views = self.num_context_views - 2 # subtract left and right context views
149
+ # if num_extra_views > 0 and remaining_indices:
150
+ # extra_views = random.sample(remaining_indices, min(num_extra_views, len(remaining_indices)))
151
+ # else:
152
+ # extra_views = []
153
+
154
+ # overlap = torch.zeros(1)
155
+
156
+ # return (
157
+ # torch.tensor((index_context_left, *extra_views, index_context_right)),
158
+ # torch.tensor(index_target),
159
+ # overlap
160
+ # )
161
+
162
+
163
+ # @property
164
+ # def num_context_views(self) -> int:
165
+ # return self.cfg.num_context_views
166
+
167
+ # @property
168
+ # def num_target_views(self) -> int:
169
+ # return self.cfg.num_target_views
170
+
171
+
172
+ class ViewSamplerRank(ViewSampler[ViewSamplerRankCfg]):
173
+
174
+ def sample(
175
+ self,
176
+ scene: str,
177
+ num_context_views: int,
178
+ extrinsics: Float[Tensor, "view 4 4"],
179
+ intrinsics: Float[Tensor, "view 3 3"],
180
+ device: torch.device = torch.device("cpu"),
181
+ ) -> tuple[
182
+ Int64[Tensor, " context_view"], # indices for context views
183
+ Int64[Tensor, " target_view"], # indices for target views
184
+ Float[Tensor, " overlap"], # overlap
185
+ ]:
186
+ num_views, _, _ = extrinsics.shape
187
+ # breakpoint()
188
+ extrinsics = extrinsics.clone()
189
+ # Compute the context view spacing based on the current global step.
190
+ ranking, dists = compute_ranking(extrinsics, lambda_t=1.0, normalize=True, batched=True)
191
+ reference_view = random.sample(range(num_views), 1)[0]
192
+
193
+ refview_ranking = ranking[reference_view]
194
+ # if self.cfg.warm_up_steps > 0:
195
+ # max_gap = self.schedule(
196
+ # self.cfg.initial_max_distance_between_context_views,
197
+ # self.cfg.max_distance_between_context_views,
198
+ # )
199
+ # min_gap = self.schedule(
200
+ # self.cfg.initial_min_distance_between_context_views,
201
+ # self.cfg.min_distance_between_context_views,
202
+ # )
203
+ # else:
204
+ min_gap, max_gap = self.num_ctxt_gap_mapping[num_context_views]
205
+
206
+ # min_gap = self.cfg.min_distance_between_context_views
207
+ # max_gap = self.cfg.max_distance_between_context_views
208
+
209
+ max_gap = min(max_gap, num_views-1)
210
+ # print(f"num_context_views: {num_context_views}, min_gap: {min_gap}, max_gap: {max_gap}")
211
+ index_context_left = reference_view
212
+ rightmost_index = random.sample(range(min_gap, max_gap + 1), 1)[0]
213
+
214
+ # #! hard code for visualization
215
+ # rightmost_index = self.cfg.max_distance_between_context_views
216
+
217
+ index_context_right = refview_ranking[rightmost_index].item()
218
+
219
+ middle_indices = refview_ranking[1: rightmost_index].tolist()
220
+ index_target = random.sample(middle_indices, self.num_target_views)
221
+
222
+ remaining_indices = [idx for idx in middle_indices if idx not in index_target]
223
+
224
+ # Sample extra context views if needed
225
+ extra_views = []
226
+ num_extra_views = num_context_views - 2 # subtract left and right context views
227
+ if num_extra_views > 0 and remaining_indices:
228
+ extra_views = random.sample(remaining_indices, min(num_extra_views, len(remaining_indices)))
229
+ else:
230
+ extra_views = []
231
+
232
+ overlap = torch.zeros(1)
233
+
234
+ return (
235
+ torch.tensor((index_context_left, *extra_views, index_context_right)),
236
+ torch.tensor(index_target),
237
+ overlap
238
+ )
239
+
240
+
241
+ @property
242
+ def num_context_views(self) -> int:
243
+ return self.cfg.num_context_views
244
+
245
+ @property
246
+ def num_target_views(self) -> int:
247
+ return self.cfg.num_target_views
248
+
249
+ @property
250
+ def num_ctxt_gap_mapping_target(self) -> dict:
251
+ mapping = dict()
252
+ for num_ctxt in range(2, self.cfg.num_context_views + 1):
253
+ mapping[num_ctxt] = [max(num_ctxt * 2, self.cfg.num_target_views + num_ctxt), max(self.cfg.num_target_views + num_ctxt, min(num_ctxt ** 2, self.cfg.max_distance_between_context_views))]
254
+ return mapping
255
+
256
+ @property
257
+ def num_ctxt_gap_mapping(self) -> dict:
258
+ mapping = dict()
259
+ for num_ctxt in range(2, self.cfg.num_context_views + 1):
260
+ mapping[num_ctxt] = [min(num_ctxt * 3, self.cfg.min_distance_between_context_views), min(max(num_ctxt * 5, num_ctxt ** 2), self.cfg.max_distance_between_context_views)]
261
+ return mapping
262
+
263
+
src/eval_nvs.py ADDED
@@ -0,0 +1,115 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from pathlib import Path
3
+ import sys
4
+ import json
5
+ import gzip
6
+ import argparse
7
+ import numpy as np
8
+ from PIL import Image
9
+
10
+ import torch
11
+ import torch.nn as nn
12
+ import torchvision
13
+ from einops import rearrange
14
+
15
+ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
16
+
17
+ from src.evaluation.metrics import compute_lpips, compute_psnr, compute_ssim
18
+ from misc.image_io import save_image, save_interpolated_video
19
+ from src.utils.image import process_image
20
+
21
+ from src.model.model.anysplat import AnySplat
22
+ from src.model.encoder.vggt.utils.pose_enc import pose_encoding_to_extri_intri
23
+
24
+ def setup_args():
25
+ """Set up command-line arguments for the eval NVS script."""
26
+ parser = argparse.ArgumentParser(description='Test AnySplat on NVS evaluation')
27
+ parser.add_argument('--data_dir', type=str, required=True, help='Path to NVS dataset')
28
+ parser.add_argument('--llffhold', type=int, default=8, help='LLFF holdout')
29
+ parser.add_argument('--output_path', type=str, default="outputs/nvs", help='Path to output directory')
30
+ return parser.parse_args()
31
+
32
+ def compute_metrics(pred_image, image):
33
+ psnr = compute_psnr(pred_image, image)
34
+ ssim = compute_ssim(pred_image, image)
35
+ lpips = compute_lpips(pred_image, image)
36
+ return psnr, ssim, lpips
37
+
38
+ def evaluate(args: argparse.Namespace):
39
+ model = AnySplat.from_pretrained("lhjiang/anysplat")
40
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
41
+ model.to(device)
42
+ model.eval()
43
+ for param in model.parameters():
44
+ param.requires_grad = False
45
+
46
+ os.makedirs(args.output_path, exist_ok=True)
47
+
48
+ # load images
49
+ image_folder = args.data_dir
50
+ image_names = sorted([os.path.join(image_folder, f) for f in os.listdir(image_folder) if f.lower().endswith(('.png', '.jpg', '.jpeg'))])
51
+ images = [process_image(img_path) for img_path in image_names]
52
+ ctx_indices = [idx for idx, name in enumerate(image_names) if idx % args.llffhold != 0]
53
+ tgt_indices = [idx for idx, name in enumerate(image_names) if idx % args.llffhold == 0]
54
+
55
+ ctx_images = torch.stack([images[i] for i in ctx_indices], dim=0).unsqueeze(0).to(device)
56
+ tgt_images = torch.stack([images[i] for i in tgt_indices], dim=0).unsqueeze(0).to(device)
57
+ ctx_images = (ctx_images+1)*0.5
58
+ tgt_images = (tgt_images+1)*0.5
59
+ b, v, _, h, w = tgt_images.shape
60
+
61
+ # run inference
62
+ encoder_output = model.encoder(
63
+ ctx_images,
64
+ global_step=0,
65
+ visualization_dump={},
66
+ )
67
+ gaussians, pred_context_pose = encoder_output.gaussians, encoder_output.pred_context_pose
68
+
69
+ num_context_view = ctx_images.shape[1]
70
+ vggt_input_image = torch.cat((ctx_images, tgt_images), dim=1).to(torch.bfloat16)
71
+ with torch.no_grad(), torch.cuda.amp.autocast(enabled=False, dtype=torch.bfloat16):
72
+ aggregated_tokens_list, patch_start_idx = model.encoder.aggregator(vggt_input_image, intermediate_layer_idx=model.encoder.cfg.intermediate_layer_idx)
73
+ with torch.cuda.amp.autocast(enabled=False):
74
+ fp32_tokens = [token.float() for token in aggregated_tokens_list]
75
+ pred_all_pose_enc = model.encoder.camera_head(fp32_tokens)[-1]
76
+ pred_all_extrinsic, pred_all_intrinsic = pose_encoding_to_extri_intri(pred_all_pose_enc, vggt_input_image.shape[-2:])
77
+
78
+ extrinsic_padding = torch.tensor([0, 0, 0, 1], device=pred_all_extrinsic.device, dtype=pred_all_extrinsic.dtype).view(1, 1, 1, 4).repeat(b, vggt_input_image.shape[1], 1, 1)
79
+ pred_all_extrinsic = torch.cat([pred_all_extrinsic, extrinsic_padding], dim=2).inverse()
80
+
81
+ pred_all_intrinsic[:, :, 0] = pred_all_intrinsic[:, :, 0] / w
82
+ pred_all_intrinsic[:, :, 1] = pred_all_intrinsic[:, :, 1] / h
83
+ pred_all_context_extrinsic, pred_all_target_extrinsic = pred_all_extrinsic[:, :num_context_view], pred_all_extrinsic[:, num_context_view:]
84
+ pred_all_context_intrinsic, pred_all_target_intrinsic = pred_all_intrinsic[:, :num_context_view], pred_all_intrinsic[:, num_context_view:]
85
+
86
+ scale_factor = pred_context_pose['extrinsic'][:, :, :3, 3].mean() / pred_all_context_extrinsic[:, :, :3, 3].mean()
87
+ pred_all_target_extrinsic[..., :3, 3] = pred_all_target_extrinsic[..., :3, 3] * scale_factor
88
+ pred_all_context_extrinsic[..., :3, 3] = pred_all_context_extrinsic[..., :3, 3] * scale_factor
89
+ print("scale_factor:", scale_factor)
90
+
91
+ output = model.decoder.forward(
92
+ gaussians,
93
+ pred_all_target_extrinsic,
94
+ pred_all_target_intrinsic.float(),
95
+ torch.ones(1, v, device=device) * 0.01,
96
+ torch.ones(1, v, device=device) * 100,
97
+ (h, w)
98
+ )
99
+
100
+ save_interpolated_video(pred_all_context_extrinsic, pred_all_context_intrinsic, b, h, w, gaussians, args.output_path, model.decoder)
101
+
102
+ # Save original images
103
+ save_path = Path(args.output_path)
104
+ # os.makedirs(save_path, exist_ok=True)
105
+ for idx, (gt_image, pred_image) in enumerate(zip(tgt_images[0], output.color[0])):
106
+ save_image(gt_image, save_path / "gt" / f"{idx:0>6}.jpg")
107
+ save_image(pred_image, save_path / "pred" / f"{idx:0>6}.jpg")
108
+
109
+ # compute metrics
110
+ psnr, ssim, lpips = compute_metrics(output.color[0], tgt_images[0])
111
+ print(f"PSNR: {psnr.mean():.2f}, SSIM: {ssim.mean():.3f}, LPIPS: {lpips.mean():.3f}")
112
+
113
+ if __name__ == "__main__":
114
+ args = setup_args()
115
+ evaluate(args)
src/eval_pose.py ADDED
@@ -0,0 +1,339 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import sys
3
+ import json
4
+ import gzip
5
+ import argparse
6
+ import numpy as np
7
+ from PIL import Image
8
+
9
+ import torch
10
+ import torch.nn as nn
11
+ import torchvision
12
+ from einops import rearrange
13
+ from lpips import LPIPS
14
+
15
+ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
16
+
17
+ from src.model.model.anysplat import AnySplat
18
+ from src.model.encoder.vggt.utils.pose_enc import pose_encoding_to_extri_intri
19
+ from src.model.encoder.vggt.utils.load_fn import load_and_preprocess_images
20
+ from src.utils.pose import align_to_first_camera, calculate_auc_np, convert_pt3d_RT_to_opencv, se3_to_relative_pose_error
21
+ from src.misc.cam_utils import camera_normalization, pose_auc, rotation_6d_to_matrix, update_pose, get_pnp_pose
22
+
23
+ def setup_args():
24
+ """Set up command-line arguments for the CO3D evaluation script."""
25
+ parser = argparse.ArgumentParser(description='Test AnySplat on CO3D dataset')
26
+ parser.add_argument('--debug', action='store_true', help='Enable debug mode (only test on specific category)')
27
+ parser.add_argument('--use_ba', action='store_true', default=False, help='Enable bundle adjustment')
28
+ parser.add_argument('--fast_eval', action='store_true', default=False, help='Only evaluate 10 sequences per category')
29
+ parser.add_argument('--min_num_images', type=int, default=50, help='Minimum number of images for a sequence')
30
+ parser.add_argument('--num_frames', type=int, default=10, help='Number of frames to use for testing')
31
+ parser.add_argument('--co3d_dir', type=str, required=True, help='Path to CO3D dataset')
32
+ parser.add_argument('--co3d_anno_dir', type=str, required=True, help='Path to CO3D annotations')
33
+ parser.add_argument('--seed', type=int, default=0, help='Random seed for reproducibility')
34
+ return parser.parse_args()
35
+
36
+ lpips = LPIPS(net="vgg")
37
+
38
+ def rendering_loss(pred_image, image):
39
+ lpips_loss = lpips.forward(rearrange(pred_image, "b v c h w -> (b v) c h w"), rearrange(image, "b v c h w -> (b v) c h w"), normalize=True)
40
+ delta = pred_image - (image + 1) / 2
41
+ mse_loss = (delta**2).mean()
42
+ return mse_loss + 0.05 * lpips_loss.mean()
43
+
44
+ def process_sequence(model, seq_name, seq_data, category, co3d_dir, min_num_images, num_frames, use_ba, device, dtype):
45
+ """
46
+ Process a single sequence and compute pose errors.
47
+
48
+ Args:
49
+ model: AnySplat model
50
+ seq_name: Sequence name
51
+ seq_data: Sequence data
52
+ category: Category name
53
+ co3d_dir: CO3D dataset directory
54
+ min_num_images: Minimum number of images required
55
+ num_frames: Number of frames to sample
56
+ use_ba: Whether to use bundle adjustment
57
+ device: Device to run on
58
+ dtype: Data type for model inference
59
+
60
+ Returns:
61
+ rError: Rotation errors
62
+ tError: Translation errors
63
+ """
64
+ if len(seq_data) < min_num_images:
65
+ return None, None
66
+
67
+ metadata = []
68
+ for data in seq_data:
69
+ # Make sure translations are not ridiculous
70
+ if data["T"][0] + data["T"][1] + data["T"][2] > 1e5:
71
+ return None, None
72
+
73
+ extri_opencv = convert_pt3d_RT_to_opencv(data["R"], data["T"])
74
+ metadata.append({
75
+ "filepath": data["filepath"],
76
+ "extri": extri_opencv,
77
+ })
78
+
79
+ ids = np.random.choice(len(metadata), num_frames, replace=False)
80
+ image_names = [os.path.join(co3d_dir, metadata[i]["filepath"]) for i in ids]
81
+ gt_extri = [np.array(metadata[i]["extri"]) for i in ids]
82
+ gt_extri = np.stack(gt_extri, axis=0)
83
+
84
+ max_size = max(Image.open(image_names[0]).size)
85
+ if max_size < 448:
86
+ return None, None
87
+ images = load_and_preprocess_images(image_names)[None].to(device)
88
+
89
+ batch = {
90
+ "context": {
91
+ "image": images*2.0-1,
92
+ "image_names": image_names,
93
+ "index": ids,
94
+ },
95
+ "scene": "co3d"
96
+ }
97
+
98
+ if use_ba:
99
+ try:
100
+ encoder_output = model.encoder(
101
+ batch,
102
+ global_step=0,
103
+ visualization_dump={},
104
+ )
105
+ gaussians, pred_context_pose = encoder_output.gaussians, encoder_output.pred_context_pose
106
+ pred_extrinsic = pred_context_pose['extrinsic']
107
+ pred_intrinsic = pred_context_pose['intrinsic']
108
+ # rendering ba
109
+ b, v, _, h, w = images.shape
110
+ with torch.set_grad_enabled(True), torch.cuda.amp.autocast(enabled=False, dtype=torch.float32):
111
+ cam_rot_delta = nn.Parameter(torch.zeros([b, v, 6], requires_grad=True, device=pred_extrinsic.device, dtype=torch.float32))
112
+ cam_trans_delta = nn.Parameter(torch.zeros([b, v, 3], requires_grad=True, device=pred_extrinsic.device, dtype=torch.float32))
113
+ opt_params = []
114
+ model.register_buffer("identity", torch.tensor([1.0, 0.0, 0.0, 0.0, 1.0, 0.0], dtype=torch.float32).to(pred_extrinsic.device))
115
+ opt_params.append(
116
+ {
117
+ "params": [cam_rot_delta],
118
+ "lr": 0.005,
119
+ }
120
+ )
121
+ opt_params.append(
122
+ {
123
+ "params": [cam_trans_delta],
124
+ "lr": 0.005,
125
+ }
126
+ )
127
+ pose_optimizer = torch.optim.Adam(opt_params)
128
+ extrinsics = pred_extrinsic.clone().float()
129
+
130
+ for i in range(100):
131
+ pose_optimizer.zero_grad()
132
+ dx, drot = cam_trans_delta, cam_rot_delta
133
+ rot = rotation_6d_to_matrix(
134
+ drot + model.identity.expand(b, v, -1)
135
+ ) # (..., 3, 3)
136
+
137
+ transform = torch.eye(4, device=extrinsics.device).repeat((b, v, 1, 1))
138
+ transform[..., :3, :3] = rot
139
+ transform[..., :3, 3] = dx
140
+
141
+ new_extrinsics = torch.matmul(extrinsics, transform)
142
+ # breakpoint()
143
+ output = model.decoder.forward(
144
+ gaussians,
145
+ new_extrinsics,
146
+ pred_intrinsic.float(),
147
+ 0.1,
148
+ 100.0,
149
+ (h, w),
150
+ # cam_rot_delta=cam_rot_delta,
151
+ # cam_trans_delta=cam_trans_delta,
152
+ )
153
+ # export_ply(gaussians.means[0], gaussians.scales[0], gaussians.rotations[0], gaussians.harmonics[0], gaussians.opacities[0], Path(f"gaussians_co3d.ply"))
154
+ rendering_loss = rendering_loss(output.color, images*2.0-1)
155
+ torchvision.utils.save_image(output.color[0], f"outputs/vis/output_co3d_{i}.png")
156
+ print(f"Rendering loss: {rendering_loss.item()}")
157
+ # print(f"Rendering loss: {rendering_loss.item()}")
158
+
159
+ rendering_loss.backward()
160
+ pose_optimizer.step()
161
+ torchvision.utils.save_image(images[0], f"outputs/vis/gt_co3d.png")
162
+ pred_extrinsic = new_extrinsics.inverse()[0][:,:-1,:]
163
+
164
+ except Exception as e:
165
+ print(f"BA failed with error: {e}. Falling back to standard VGGT inference.")
166
+ with torch.no_grad(), torch.cuda.amp.autocast(dtype=dtype):
167
+ aggregated_tokens_list, patch_start_idx = model.encoder.aggregator(images, intermediate_layer_idx=model.encoder.cfg.intermediate_layer_idx)
168
+ with torch.cuda.amp.autocast(dtype=torch.float32):
169
+ fp32_tokens = [token.float() for token in aggregated_tokens_list]
170
+ pred_all_pose_enc = model.encoder.camera_head(fp32_tokens)[-1]
171
+ pred_all_extrinsic, pred_all_intrinsic = pose_encoding_to_extri_intri(pred_all_pose_enc, images.shape[-2:])
172
+ pred_extrinsic = pred_all_extrinsic[0]
173
+ else:
174
+ with torch.no_grad(), torch.cuda.amp.autocast(dtype=dtype):
175
+ aggregated_tokens_list, patch_start_idx = model.encoder.aggregator(images, intermediate_layer_idx=model.encoder.cfg.intermediate_layer_idx)
176
+ with torch.cuda.amp.autocast(dtype=torch.float32):
177
+ fp32_tokens = [token.float() for token in aggregated_tokens_list]
178
+ pred_all_pose_enc = model.encoder.camera_head(fp32_tokens)[-1]
179
+ pred_all_extrinsic, pred_all_intrinsic = pose_encoding_to_extri_intri(pred_all_pose_enc, images.shape[-2:])
180
+ pred_extrinsic = pred_all_extrinsic[0]
181
+
182
+ with torch.cuda.amp.autocast(dtype=torch.float32):
183
+ gt_extrinsic = torch.from_numpy(gt_extri).to(device)
184
+ add_row = torch.tensor([0, 0, 0, 1], device=device).expand(pred_extrinsic.size(0), 1, 4)
185
+
186
+ pred_se3 = torch.cat((pred_extrinsic, add_row), dim=1)
187
+ gt_se3 = torch.cat((gt_extrinsic, add_row), dim=1)
188
+
189
+ # Set the coordinate of the first camera as the coordinate of the world
190
+ # NOTE: DO NOT REMOVE THIS UNLESS YOU KNOW WHAT YOU ARE DOING
191
+ # pred_se3 = align_to_first_camera(pred_se3)
192
+ gt_se3 = align_to_first_camera(gt_se3)
193
+
194
+ rel_rangle_deg, rel_tangle_deg = se3_to_relative_pose_error(pred_se3, gt_se3, num_frames)
195
+ print(f"{category} sequence {seq_name} Rot Error: {rel_rangle_deg.mean().item():.4f}")
196
+ print(f"{category} sequence {seq_name} Trans Error: {rel_tangle_deg.mean().item():.4f}")
197
+
198
+ return rel_rangle_deg.cpu().numpy(), rel_tangle_deg.cpu().numpy()
199
+
200
+ def evaluate(args: argparse.Namespace):
201
+ model = AnySplat.from_pretrained("lhjiang/anysplat")
202
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
203
+ model.to(device)
204
+ model.eval()
205
+ for param in model.parameters():
206
+ param.requires_grad = False
207
+
208
+ # CO3D evaluation
209
+ SEEN_CATEGORIES = [
210
+ "apple", "backpack", "banana", "baseballbat", "baseballglove",
211
+ "bench", "bicycle", "bottle", "bowl", "broccoli",
212
+ "cake", "car", "carrot", "cellphone", "chair",
213
+ "cup", "donut", "hairdryer", "handbag", "hydrant",
214
+ "keyboard", "laptop", "microwave", "motorcycle", "mouse",
215
+ "orange", "parkingmeter", "pizza", "plant", "stopsign",
216
+ "teddybear", "toaster", "toilet", "toybus", "toyplane",
217
+ "toytrain", "toytruck", "tv", "umbrella", "vase", "wineglass",
218
+ ]
219
+
220
+ if args.debug:
221
+ SEEN_CATEGORIES = ["apple"]
222
+
223
+ per_category_results = {}
224
+
225
+ for category in SEEN_CATEGORIES:
226
+ print(f"Loading annotation for {category} test set")
227
+ annotation_file = os.path.join(args.co3d_anno_dir, f"{category}_test.jgz")
228
+
229
+ try:
230
+ with gzip.open(annotation_file, "r") as fin:
231
+ annotation = json.loads(fin.read())
232
+ except FileNotFoundError:
233
+ print(f"Annotation file not found for {category}, skipping")
234
+ continue
235
+
236
+ rError = []
237
+ tError = []
238
+
239
+ for seq_name, seq_data in annotation.items():
240
+ print("-" * 50)
241
+
242
+ print(f"Processing {seq_name} for {category} test set")
243
+ if args.debug and not os.path.exists(os.path.join(args.co3d_dir, category, seq_name)):
244
+ print(f"Skipping {seq_name} (not found)")
245
+ continue
246
+
247
+ seq_rError, seq_tError = process_sequence(
248
+ model, seq_name, seq_data, category, args.co3d_dir,
249
+ args.min_num_images, args.num_frames, args.use_ba, device, torch.bfloat16
250
+ )
251
+
252
+ print("-" * 50)
253
+
254
+ if seq_rError is not None and seq_tError is not None:
255
+ rError.extend(seq_rError)
256
+ tError.extend(seq_tError)
257
+
258
+ if not rError:
259
+ print(f"No valid sequences found for {category}, skipping")
260
+ continue
261
+
262
+ rError = np.array(rError)
263
+ tError = np.array(tError)
264
+
265
+ thresholds = [5, 10, 20, 30]
266
+ Aucs = {}
267
+
268
+ for threshold in thresholds:
269
+ Auc, _ = calculate_auc_np(rError, tError, max_threshold=threshold)
270
+ Aucs[threshold] = Auc
271
+
272
+ print("="*80)
273
+ print(f"AUC of {category} test set: {Aucs[30]:.4f}")
274
+ print("="*80)
275
+
276
+ per_category_results[category] = {
277
+ "rError": rError,
278
+ "tError": tError,
279
+ "Auc_5": Aucs[5],
280
+ "Auc_10": Aucs[10],
281
+ "Auc_20": Aucs[20],
282
+ "Auc_30": Aucs[30],
283
+ }
284
+
285
+ # Print summary results
286
+ print("\nSummary of AUC results:")
287
+ print("-"*50)
288
+ for category in sorted(per_category_results.keys()):
289
+ print(f"{category:<15} AUC_5: {per_category_results[category]['Auc_5']:.4f}")
290
+ print(f"{category:<15} AUC_30: {per_category_results[category]['Auc_30']:.4f}")
291
+ print(f"{category:<15} AUC_20: {per_category_results[category]['Auc_20']:.4f}")
292
+ print(f"{category:<15} AUC_10: {per_category_results[category]['Auc_10']:.4f}")
293
+
294
+ if per_category_results:
295
+ mean_AUC_30 = np.mean([per_category_results[category]["Auc_30"] for category in per_category_results])
296
+ mean_AUC_20 = np.mean([per_category_results[category]["Auc_20"] for category in per_category_results])
297
+ mean_AUC_10 = np.mean([per_category_results[category]["Auc_10"] for category in per_category_results])
298
+ mean_AUC_5 = np.mean([per_category_results[category]["Auc_5"] for category in per_category_results])
299
+ print("-"*50)
300
+ print(f"Mean AUC_5: {mean_AUC_5:.4f}")
301
+ print(f"Mean AUC_30: {mean_AUC_30:.4f}")
302
+ print(f"Mean AUC_20: {mean_AUC_20:.4f}")
303
+ print(f"Mean AUC_10: {mean_AUC_10:.4f}")
304
+
305
+ # Generate a random index to avoid overwriting previous results
306
+ # random_index = torch.randint(0, 10000, (1,)).item()
307
+ # Use timestamp as index instead of random number
308
+ import datetime
309
+ timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
310
+ random_index = timestamp
311
+ results_file = f"co3d_results_{random_index}.txt"
312
+
313
+ with open(results_file, "w") as f:
314
+ f.write("CO3D Evaluation Results\n")
315
+ f.write("=" * 50 + "\n\n")
316
+
317
+ f.write("Per-category results:\n")
318
+ f.write("-" * 50 + "\n")
319
+ for category in sorted(per_category_results.keys()):
320
+ f.write(f"{category:<15} AUC_30: {per_category_results[category]['Auc_30']:.4f}\n")
321
+ f.write(f"{category:<15} AUC_20: {per_category_results[category]['Auc_20']:.4f}\n")
322
+ f.write(f"{category:<15} AUC_10: {per_category_results[category]['Auc_10']:.4f}\n")
323
+ f.write(f"{category:<15} AUC_5: {per_category_results[category]['Auc_5']:.4f}\n")
324
+ f.write("\n")
325
+
326
+ if per_category_results:
327
+ f.write("-" * 50 + "\n")
328
+ f.write(f"Mean AUC_30: {mean_AUC_30:.4f}\n")
329
+ f.write(f"Mean AUC_20: {mean_AUC_20:.4f}\n")
330
+ f.write(f"Mean AUC_10: {mean_AUC_10:.4f}\n")
331
+ f.write(f"Mean AUC_5: {mean_AUC_5:.4f}\n")
332
+ f.write("\n" + "=" * 50 + "\n")
333
+
334
+ print(f"Results saved to {results_file}")
335
+
336
+
337
+ if __name__ == "__main__":
338
+ args = setup_args()
339
+ evaluate(args)
src/evaluation/evaluation_cfg.py ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass
2
+ from pathlib import Path
3
+
4
+
5
+ @dataclass
6
+ class MethodCfg:
7
+ name: str
8
+ key: str
9
+ path: Path
10
+
11
+
12
+ @dataclass
13
+ class SceneCfg:
14
+ scene: str
15
+ target_index: int
16
+
17
+
18
+ @dataclass
19
+ class EvaluationCfg:
20
+ methods: list[MethodCfg]
21
+ side_by_side_path: Path | None
22
+ output_metrics_path: Path
23
+ animate_side_by_side: bool
24
+ highlighted: list[SceneCfg]
src/evaluation/evaluation_index_generator.py ADDED
@@ -0,0 +1,160 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ from dataclasses import asdict, dataclass
3
+ from pathlib import Path
4
+ from typing import Optional
5
+
6
+ import torch
7
+ from einops import rearrange
8
+ from lightning.pytorch import LightningModule
9
+ from tqdm import tqdm
10
+
11
+ from ..geometry.epipolar_lines import project_rays
12
+ from ..geometry.projection import get_world_rays, sample_image_grid
13
+ from ..misc.image_io import save_image
14
+ from ..visualization.annotation import add_label
15
+ from ..visualization.layout import add_border, hcat
16
+
17
+
18
+ @dataclass
19
+ class EvaluationIndexGeneratorCfg:
20
+ num_target_views: int
21
+ min_distance: int
22
+ max_distance: int
23
+ min_overlap: float
24
+ max_overlap: float
25
+ output_path: Path
26
+ save_previews: bool
27
+ seed: int
28
+
29
+
30
+ @dataclass
31
+ class IndexEntry:
32
+ context: tuple[int, ...]
33
+ target: tuple[int, ...]
34
+ overlap: Optional[str | float] = None # choose from ["small", "medium", "large"] or a float number indicates the overlap ratio
35
+
36
+
37
+ class EvaluationIndexGenerator(LightningModule):
38
+ generator: torch.Generator
39
+ cfg: EvaluationIndexGeneratorCfg
40
+ index: dict[str, IndexEntry | None]
41
+
42
+ def __init__(self, cfg: EvaluationIndexGeneratorCfg) -> None:
43
+ super().__init__()
44
+ self.cfg = cfg
45
+ self.generator = torch.Generator()
46
+ self.generator.manual_seed(cfg.seed)
47
+ self.index = {}
48
+
49
+ def test_step(self, batch, batch_idx):
50
+ b, v, _, h, w = batch["target"]["image"].shape
51
+ assert b == 1
52
+ extrinsics = batch["target"]["extrinsics"][0]
53
+ intrinsics = batch["target"]["intrinsics"][0]
54
+ scene = batch["scene"][0]
55
+
56
+ context_indices = torch.randperm(v, generator=self.generator)
57
+ for context_index in tqdm(context_indices, "Finding context pair"):
58
+ xy, _ = sample_image_grid((h, w), self.device)
59
+ context_origins, context_directions = get_world_rays(
60
+ rearrange(xy, "h w xy -> (h w) xy"),
61
+ extrinsics[context_index],
62
+ intrinsics[context_index],
63
+ )
64
+
65
+ # Step away from context view until the minimum overlap threshold is met.
66
+ valid_indices = []
67
+ for step in (1, -1):
68
+ min_distance = self.cfg.min_distance
69
+ max_distance = self.cfg.max_distance
70
+ current_index = context_index + step * min_distance
71
+
72
+ while 0 <= current_index.item() < v:
73
+ # Compute overlap.
74
+ current_origins, current_directions = get_world_rays(
75
+ rearrange(xy, "h w xy -> (h w) xy"),
76
+ extrinsics[current_index],
77
+ intrinsics[current_index],
78
+ )
79
+ projection_onto_current = project_rays(
80
+ context_origins,
81
+ context_directions,
82
+ extrinsics[current_index],
83
+ intrinsics[current_index],
84
+ )
85
+ projection_onto_context = project_rays(
86
+ current_origins,
87
+ current_directions,
88
+ extrinsics[context_index],
89
+ intrinsics[context_index],
90
+ )
91
+ overlap_a = projection_onto_context["overlaps_image"].float().mean()
92
+ overlap_b = projection_onto_current["overlaps_image"].float().mean()
93
+
94
+ overlap = min(overlap_a, overlap_b)
95
+ delta = (current_index - context_index).abs()
96
+
97
+ min_overlap = self.cfg.min_overlap
98
+ max_overlap = self.cfg.max_overlap
99
+ if min_overlap <= overlap <= max_overlap:
100
+ valid_indices.append(
101
+ (current_index.item(), overlap_a, overlap_b)
102
+ )
103
+
104
+ # Stop once the camera has panned away too much.
105
+ if overlap < min_overlap or delta > max_distance:
106
+ break
107
+
108
+ current_index += step
109
+
110
+ if valid_indices:
111
+ # Pick a random valid view. Index the resulting views.
112
+ num_options = len(valid_indices)
113
+ chosen = torch.randint(
114
+ 0, num_options, size=tuple(), generator=self.generator
115
+ )
116
+ chosen, overlap_a, overlap_b = valid_indices[chosen]
117
+
118
+ context_left = min(chosen, context_index.item())
119
+ context_right = max(chosen, context_index.item())
120
+ delta = context_right - context_left
121
+
122
+ # Pick non-repeated random target views.
123
+ while True:
124
+ target_views = torch.randint(
125
+ context_left,
126
+ context_right + 1,
127
+ (self.cfg.num_target_views,),
128
+ generator=self.generator,
129
+ )
130
+ if (target_views.unique(return_counts=True)[1] == 1).all():
131
+ break
132
+
133
+ target = tuple(sorted(target_views.tolist()))
134
+ self.index[scene] = IndexEntry(
135
+ context=(context_left, context_right),
136
+ target=target,
137
+ )
138
+
139
+ # Optionally, save a preview.
140
+ if self.cfg.save_previews:
141
+ preview_path = self.cfg.output_path / "previews"
142
+ preview_path.mkdir(exist_ok=True, parents=True)
143
+ a = batch["target"]["image"][0, chosen]
144
+ a = add_label(a, f"Overlap: {overlap_a * 100:.1f}%")
145
+ b = batch["target"]["image"][0, context_index]
146
+ b = add_label(b, f"Overlap: {overlap_b * 100:.1f}%")
147
+ vis = add_border(add_border(hcat(a, b)), 1, 0)
148
+ vis = add_label(vis, f"Distance: {delta} frames")
149
+ save_image(add_border(vis), preview_path / f"{scene}.png")
150
+ break
151
+ else:
152
+ # This happens if no starting frame produces a valid evaluation example.
153
+ self.index[scene] = None
154
+
155
+ def save_index(self) -> None:
156
+ self.cfg.output_path.mkdir(exist_ok=True, parents=True)
157
+ with (self.cfg.output_path / "evaluation_index.json").open("w") as f:
158
+ json.dump(
159
+ {k: None if v is None else asdict(v) for k, v in self.index.items()}, f
160
+ )
src/evaluation/metrics.py ADDED
@@ -0,0 +1,134 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from functools import cache
2
+
3
+ import torch
4
+ from einops import reduce
5
+ from jaxtyping import Float
6
+ from lpips import LPIPS
7
+ from skimage.metrics import structural_similarity
8
+ from torch import Tensor
9
+
10
+
11
+ @torch.no_grad()
12
+ def compute_psnr(
13
+ ground_truth: Float[Tensor, "batch channel height width"],
14
+ predicted: Float[Tensor, "batch channel height width"],
15
+ ) -> Float[Tensor, " batch"]:
16
+ ground_truth = ground_truth.clip(min=0, max=1)
17
+ predicted = predicted.clip(min=0, max=1)
18
+ mse = reduce((ground_truth - predicted) ** 2, "b c h w -> b", "mean")
19
+ return -10 * mse.log10()
20
+
21
+
22
+ @cache
23
+ def get_lpips(device: torch.device) -> LPIPS:
24
+ return LPIPS(net="vgg").to(device)
25
+
26
+
27
+ @torch.no_grad()
28
+ def compute_lpips(
29
+ ground_truth: Float[Tensor, "batch channel height width"],
30
+ predicted: Float[Tensor, "batch channel height width"],
31
+ ) -> Float[Tensor, " batch"]:
32
+ value = get_lpips(predicted.device).forward(ground_truth, predicted, normalize=True)
33
+ return value[:, 0, 0, 0]
34
+
35
+
36
+ @torch.no_grad()
37
+ def compute_ssim(
38
+ ground_truth: Float[Tensor, "batch channel height width"],
39
+ predicted: Float[Tensor, "batch channel height width"],
40
+ ) -> Float[Tensor, " batch"]:
41
+ ssim = [
42
+ structural_similarity(
43
+ gt.detach().cpu().numpy(),
44
+ hat.detach().cpu().numpy(),
45
+ win_size=11,
46
+ gaussian_weights=True,
47
+ channel_axis=0,
48
+ data_range=1.0,
49
+ )
50
+ for gt, hat in zip(ground_truth, predicted)
51
+ ]
52
+ return torch.tensor(ssim, dtype=predicted.dtype, device=predicted.device)
53
+
54
+
55
+ def compute_geodesic_distance_from_two_matrices(m1, m2):
56
+ batch = m1.shape[0]
57
+ m = torch.bmm(m1, m2.transpose(1, 2)) # batch*3*3
58
+
59
+ cos = (m[:, 0, 0] + m[:, 1, 1] + m[:, 2, 2] - 1) / 2
60
+ cos = torch.min(cos, torch.autograd.Variable(torch.ones(batch).to(m1.device)))
61
+ cos = torch.max(cos, torch.autograd.Variable(torch.ones(batch).to(m1.device)) * -1)
62
+
63
+ theta = torch.acos(cos)
64
+
65
+ # theta = torch.min(theta, 2*np.pi - theta)
66
+
67
+ return theta
68
+
69
+
70
+ def angle_error_mat(R1, R2):
71
+ cos = (torch.trace(torch.mm(R1.T, R2)) - 1) / 2
72
+ cos = torch.clamp(cos, -1.0, 1.0) # numerical errors can make it out of bounds
73
+ return torch.rad2deg(torch.abs(torch.acos(cos)))
74
+
75
+
76
+ def angle_error_vec(v1, v2):
77
+ n = torch.norm(v1) * torch.norm(v2)
78
+ cos_theta = torch.dot(v1, v2) / n
79
+ cos_theta = torch.clamp(cos_theta, -1.0, 1.0) # numerical errors can make it out of bounds
80
+ return torch.rad2deg(torch.acos(cos_theta))
81
+
82
+
83
+ def compute_translation_error(t1, t2):
84
+ return torch.norm(t1 - t2)
85
+
86
+
87
+ @torch.no_grad()
88
+ def compute_pose_error(pose_gt, pose_pred):
89
+ R_gt = pose_gt[:3, :3]
90
+ t_gt = pose_gt[:3, 3]
91
+
92
+ R = pose_pred[:3, :3]
93
+ t = pose_pred[:3, 3]
94
+
95
+ error_t = angle_error_vec(t, t_gt)
96
+ error_t = torch.minimum(error_t, 180 - error_t) # ambiguity of E estimation
97
+ error_t_scale = compute_translation_error(t, t_gt)
98
+ error_R = angle_error_mat(R, R_gt)
99
+ return error_t, error_t_scale, error_R
100
+
101
+ @torch.no_grad()
102
+ def abs_relative_difference(output, target, valid_mask=None):
103
+ actual_output = output
104
+ actual_target = target
105
+ abs_relative_diff = torch.abs(actual_output - actual_target) / actual_target
106
+ if valid_mask is not None:
107
+ abs_relative_diff[~valid_mask] = 0
108
+ n = valid_mask.sum((-1, -2))
109
+ else:
110
+ n = output.shape[-1] * output.shape[-2]
111
+ abs_relative_diff = torch.sum(abs_relative_diff, (-1, -2)) / n
112
+ return abs_relative_diff.mean()
113
+
114
+ # adapt from: https://github.com/imran3180/depth-map-prediction/blob/master/main.py
115
+ @torch.no_grad()
116
+ def threshold_percentage(output, target, threshold_val, valid_mask=None):
117
+ d1 = output / target
118
+ d2 = target / output
119
+ max_d1_d2 = torch.max(d1, d2)
120
+ zero = torch.zeros_like(output)
121
+ one = torch.ones_like(output)
122
+ bit_mat = torch.where(max_d1_d2 < threshold_val, one, zero)
123
+ if valid_mask is not None:
124
+ bit_mat[~valid_mask] = 0
125
+ n = valid_mask.sum((-1, -2))
126
+ else:
127
+ n = output.shape[-1] * output.shape[-2]
128
+ count_mat = torch.sum(bit_mat, (-1, -2))
129
+ threshold_mat = count_mat / n
130
+ return threshold_mat.mean()
131
+
132
+ @torch.no_grad()
133
+ def delta1_acc(pred, gt, valid_mask):
134
+ return threshold_percentage(pred, gt, 1.25, valid_mask)
src/geometry/camera_emb.py ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from einops import rearrange
2
+
3
+ from .projection import sample_image_grid, get_local_rays
4
+ from ..misc.sht import rsh_cart_2, rsh_cart_4, rsh_cart_6, rsh_cart_8
5
+
6
+
7
+ def get_intrinsic_embedding(context, degree=0, downsample=1, merge_hw=False):
8
+ assert degree in [0, 2, 4, 8]
9
+
10
+ b, v, _, h, w = context["image"].shape
11
+ device = context["image"].device
12
+ tgt_h, tgt_w = h // downsample, w // downsample
13
+ xy_ray, _ = sample_image_grid((tgt_h, tgt_w), device)
14
+ xy_ray = xy_ray[None, None, ...].expand(b, v, -1, -1, -1) # [b, v, h, w, 2]
15
+ directions = get_local_rays(xy_ray, rearrange(context["intrinsics"], "b v i j -> b v () () i j"),)
16
+
17
+ if degree == 2:
18
+ directions = rsh_cart_2(directions)
19
+ elif degree == 4:
20
+ directions = rsh_cart_4(directions)
21
+ elif degree == 8:
22
+ directions = rsh_cart_8(directions)
23
+
24
+ if merge_hw:
25
+ directions = rearrange(directions, "b v h w d -> b v (h w) d")
26
+ else:
27
+ directions = rearrange(directions, "b v h w d -> b v d h w")
28
+
29
+ return directions
src/geometry/epipolar_lines.py ADDED
@@ -0,0 +1,292 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import itertools
2
+ from typing import Iterable, Literal, Optional, TypedDict
3
+
4
+ import torch
5
+ from einops import einsum, repeat
6
+ from jaxtyping import Bool, Float
7
+ from torch import Tensor
8
+ from torch.utils.data.dataloader import default_collate
9
+
10
+ from .projection import (
11
+ get_world_rays,
12
+ homogenize_points,
13
+ homogenize_vectors,
14
+ intersect_rays,
15
+ project_camera_space,
16
+ )
17
+
18
+
19
+ def _is_in_bounds(
20
+ xy: Float[Tensor, "*batch 2"],
21
+ epsilon: float = 1e-6,
22
+ ) -> Bool[Tensor, " *batch"]:
23
+ """Check whether the specified XY coordinates are within the normalized image plane,
24
+ which has a range from 0 to 1 in each direction.
25
+ """
26
+ return (xy >= -epsilon).all(dim=-1) & (xy <= 1 + epsilon).all(dim=-1)
27
+
28
+
29
+ def _is_in_front_of_camera(
30
+ xyz: Float[Tensor, "*batch 3"],
31
+ epsilon: float = 1e-6,
32
+ ) -> Bool[Tensor, " *batch"]:
33
+ """Check whether the specified points in camera space are in front of the camera."""
34
+ return xyz[..., -1] > -epsilon
35
+
36
+
37
+ def _is_positive_t(
38
+ t: Float[Tensor, " *batch"],
39
+ epsilon: float = 1e-6,
40
+ ) -> Bool[Tensor, " *batch"]:
41
+ """Check whether the specified t value is positive."""
42
+ return t > -epsilon
43
+
44
+
45
+ class PointProjection(TypedDict):
46
+ t: Float[Tensor, " *batch"] # ray parameter, as in xyz = origin + t * direction
47
+ xy: Float[Tensor, "*batch 2"] # image-space xy (normalized to 0 to 1)
48
+
49
+ # A "valid" projection satisfies two conditions:
50
+ # 1. It is in front of the camera (i.e., its 3D Z coordinate is positive).
51
+ # 2. It is within the image frame (i.e., its 2D coordinates are between 0 and 1).
52
+ valid: Bool[Tensor, " *batch"]
53
+
54
+
55
+ def _intersect_image_coordinate(
56
+ intrinsics: Float[Tensor, "*#batch 3 3"],
57
+ origins: Float[Tensor, "*#batch 3"],
58
+ directions: Float[Tensor, "*#batch 3"],
59
+ dimension: Literal["x", "y"],
60
+ coordinate_value: float,
61
+ ) -> PointProjection:
62
+ """Compute the intersection of the projection of a camera-space ray with a line
63
+ that's parallel to the image frame, either horizontally or vertically.
64
+ """
65
+
66
+ # Define shorthands.
67
+ dim = "xy".index(dimension)
68
+ other_dim = 1 - dim
69
+ fs = intrinsics[..., dim, dim] # focal length, same coordinate
70
+ fo = intrinsics[..., other_dim, other_dim] # focal length, other coordinate
71
+ cs = intrinsics[..., dim, 2] # principal point, same coordinate
72
+ co = intrinsics[..., other_dim, 2] # principal point, other coordinate
73
+ os = origins[..., dim] # ray origin, same coordinate
74
+ oo = origins[..., other_dim] # ray origin, other coordinate
75
+ ds = directions[..., dim] # ray direction, same coordinate
76
+ do = directions[..., other_dim] # ray direction, other coordinate
77
+ oz = origins[..., 2] # ray origin, z coordinate
78
+ dz = directions[..., 2] # ray direction, z coordinate
79
+ c = (coordinate_value - cs) / fs # coefficient (computed once and factored out)
80
+
81
+ # Compute the value of t at the intersection.
82
+ # Note: Infinite values of t are fine. No need to handle division by zero.
83
+ t_numerator = c * oz - os
84
+ t_denominator = ds - c * dz
85
+ t = t_numerator / t_denominator
86
+
87
+ # Compute the value of the other coordinate at the intersection.
88
+ # Note: Infinite coordinate values are fine. No need to handle division by zero.
89
+ coordinate_numerator = fo * (oo * (c * dz - ds) + do * (os - c * oz))
90
+ coordinate_denominator = dz * os - ds * oz
91
+ coordinate_other = co + coordinate_numerator / coordinate_denominator
92
+ coordinate_same = torch.ones_like(coordinate_other) * coordinate_value
93
+ xy = [coordinate_same]
94
+ xy.insert(other_dim, coordinate_other)
95
+ xy = torch.stack(xy, dim=-1)
96
+ xyz = origins + t[..., None] * directions
97
+
98
+ # These will all have exactly the same batch shape (no broadcasting necessary). In
99
+ # terms of jaxtyping annotations, they all match *batch, not just *#batch.
100
+ return {
101
+ "t": t,
102
+ "xy": xy,
103
+ "valid": _is_in_bounds(xy) & _is_in_front_of_camera(xyz) & _is_positive_t(t),
104
+ }
105
+
106
+
107
+ def _compare_projections(
108
+ intersections: Iterable[PointProjection],
109
+ reduction: Literal["min", "max"],
110
+ ) -> PointProjection:
111
+ intersections = {k: v.clone() for k, v in default_collate(intersections).items()}
112
+ t = intersections["t"]
113
+ xy = intersections["xy"]
114
+ valid = intersections["valid"]
115
+
116
+ # Make sure out-of-bounds values are not chosen.
117
+ lowest_priority = {
118
+ "min": torch.inf,
119
+ "max": -torch.inf,
120
+ }[reduction]
121
+ t[~valid] = lowest_priority
122
+
123
+ # Run the reduction (either t.min() or t.max()).
124
+ reduced, selector = getattr(t, reduction)(dim=0)
125
+
126
+ # Index the results.
127
+ return {
128
+ "t": reduced,
129
+ "xy": xy.gather(0, repeat(selector, "... -> () ... xy", xy=2))[0],
130
+ "valid": valid.gather(0, selector[None])[0],
131
+ }
132
+
133
+
134
+ def _compute_point_projection(
135
+ xyz: Float[Tensor, "*#batch 3"],
136
+ t: Float[Tensor, "*#batch"],
137
+ intrinsics: Float[Tensor, "*#batch 3 3"],
138
+ ) -> PointProjection:
139
+ xy = project_camera_space(xyz, intrinsics)
140
+ return {
141
+ "t": t,
142
+ "xy": xy,
143
+ "valid": _is_in_bounds(xy) & _is_in_front_of_camera(xyz) & _is_positive_t(t),
144
+ }
145
+
146
+
147
+ class RaySegmentProjection(TypedDict):
148
+ t_min: Float[Tensor, " *batch"] # ray parameter
149
+ t_max: Float[Tensor, " *batch"] # ray parameter
150
+ xy_min: Float[Tensor, "*batch 2"] # image-space xy (normalized to 0 to 1)
151
+ xy_max: Float[Tensor, "*batch 2"] # image-space xy (normalized to 0 to 1)
152
+
153
+ # Whether the segment overlaps the image. If not, the above values are meaningless.
154
+ overlaps_image: Bool[Tensor, " *batch"]
155
+
156
+
157
+ def project_rays(
158
+ origins: Float[Tensor, "*#batch 3"],
159
+ directions: Float[Tensor, "*#batch 3"],
160
+ extrinsics: Float[Tensor, "*#batch 4 4"],
161
+ intrinsics: Float[Tensor, "*#batch 3 3"],
162
+ near: Optional[Float[Tensor, "*#batch"]] = None,
163
+ far: Optional[Float[Tensor, "*#batch"]] = None,
164
+ epsilon: float = 1e-6,
165
+ ) -> RaySegmentProjection:
166
+ # Transform the rays into camera space.
167
+ world_to_cam = torch.linalg.inv(extrinsics)
168
+ origins = homogenize_points(origins)
169
+ origins = einsum(world_to_cam, origins, "... i j, ... j -> ... i")
170
+ directions = homogenize_vectors(directions)
171
+ directions = einsum(world_to_cam, directions, "... i j, ... j -> ... i")
172
+ origins = origins[..., :3]
173
+ directions = directions[..., :3]
174
+
175
+ # Compute intersections with the image's frame.
176
+ frame_intersections = (
177
+ _intersect_image_coordinate(intrinsics, origins, directions, "x", 0.0),
178
+ _intersect_image_coordinate(intrinsics, origins, directions, "x", 1.0),
179
+ _intersect_image_coordinate(intrinsics, origins, directions, "y", 0.0),
180
+ _intersect_image_coordinate(intrinsics, origins, directions, "y", 1.0),
181
+ )
182
+ frame_intersection_min = _compare_projections(frame_intersections, "min")
183
+ frame_intersection_max = _compare_projections(frame_intersections, "max")
184
+
185
+ if near is None:
186
+ # Compute the ray's projection at zero depth. If an origin's depth (z value) is
187
+ # within epsilon of zero, this can mean one of two things:
188
+ # 1. The origin is at the camera's position. In this case, use the direction
189
+ # instead (the ray is probably coming from the camera).
190
+ # 2. The origin isn't at the camera's position, and randomly happens to be on
191
+ # the plane at zero depth. In this case, its projection is outside the image
192
+ # plane, and is thus marked as invalid.
193
+ origins_for_projection = origins.clone()
194
+ mask_depth_zero = origins_for_projection[..., -1] < epsilon
195
+ mask_at_camera = origins_for_projection.norm(dim=-1) < epsilon
196
+ origins_for_projection[mask_at_camera] = directions[mask_at_camera]
197
+ projection_at_zero = _compute_point_projection(
198
+ origins_for_projection,
199
+ torch.zeros_like(frame_intersection_min["t"]),
200
+ intrinsics,
201
+ )
202
+ projection_at_zero["valid"][mask_depth_zero & ~mask_at_camera] = False
203
+ else:
204
+ # If a near plane is specified, use it instead.
205
+ t_near = near.broadcast_to(frame_intersection_min["t"].shape)
206
+ projection_at_zero = _compute_point_projection(
207
+ origins + near[..., None] * directions,
208
+ t_near,
209
+ intrinsics,
210
+ )
211
+
212
+ if far is None:
213
+ # Compute the ray's projection at infinite depth. Using the projection function
214
+ # with directions (vectors) instead of points may seem wonky, but is equivalent
215
+ # to projecting the point at (origins + infinity * directions).
216
+ projection_at_infinity = _compute_point_projection(
217
+ directions,
218
+ torch.ones_like(frame_intersection_min["t"]) * torch.inf,
219
+ intrinsics,
220
+ )
221
+ else:
222
+ # If a far plane is specified, use it instead.
223
+ t_far = far.broadcast_to(frame_intersection_min["t"].shape)
224
+ projection_at_infinity = _compute_point_projection(
225
+ origins + far[..., None] * directions,
226
+ t_far,
227
+ intrinsics,
228
+ )
229
+
230
+ # Build the result by handling cases for ray intersection.
231
+ result = {
232
+ "t_min": torch.empty_like(projection_at_zero["t"]),
233
+ "t_max": torch.empty_like(projection_at_infinity["t"]),
234
+ "xy_min": torch.empty_like(projection_at_zero["xy"]),
235
+ "xy_max": torch.empty_like(projection_at_infinity["xy"]),
236
+ "overlaps_image": torch.empty_like(projection_at_zero["valid"]),
237
+ }
238
+
239
+ for min_valid, max_valid in itertools.product([True, False], [True, False]):
240
+ min_mask = projection_at_zero["valid"] ^ (not min_valid)
241
+ max_mask = projection_at_infinity["valid"] ^ (not max_valid)
242
+ mask = min_mask & max_mask
243
+ min_value = projection_at_zero if min_valid else frame_intersection_min
244
+ max_value = projection_at_infinity if max_valid else frame_intersection_max
245
+ result["t_min"][mask] = min_value["t"][mask]
246
+ result["t_max"][mask] = max_value["t"][mask]
247
+ result["xy_min"][mask] = min_value["xy"][mask]
248
+ result["xy_max"][mask] = max_value["xy"][mask]
249
+ result["overlaps_image"][mask] = (min_value["valid"] & max_value["valid"])[mask]
250
+
251
+ return result
252
+
253
+
254
+ class RaySegmentProjection(TypedDict):
255
+ t_min: Float[Tensor, " *batch"] # ray parameter
256
+ t_max: Float[Tensor, " *batch"] # ray parameter
257
+ xy_min: Float[Tensor, "*batch 2"] # image-space xy (normalized to 0 to 1)
258
+ xy_max: Float[Tensor, "*batch 2"] # image-space xy (normalized to 0 to 1)
259
+
260
+ # Whether the segment overlaps the image. If not, the above values are meaningless.
261
+ overlaps_image: Bool[Tensor, " *batch"]
262
+
263
+
264
+ def lift_to_3d(
265
+ origins: Float[Tensor, "*#batch 3"],
266
+ directions: Float[Tensor, "*#batch 3"],
267
+ xy: Float[Tensor, "*#batch 2"],
268
+ extrinsics: Float[Tensor, "*#batch 4 4"],
269
+ intrinsics: Float[Tensor, "*#batch 3 3"],
270
+ ) -> Float[Tensor, "*batch 3"]:
271
+ """Calculate the 3D positions that correspond to the specified 2D points on the
272
+ epipolar lines defined by the origins and directions. The extrinsics and intrinsics
273
+ are for the images the 2D points lie on.
274
+ """
275
+
276
+ xy_origins, xy_directions = get_world_rays(xy, extrinsics, intrinsics)
277
+ return intersect_rays(origins, directions, xy_origins, xy_directions)
278
+
279
+
280
+ def get_depth(
281
+ origins: Float[Tensor, "*#batch 3"],
282
+ directions: Float[Tensor, "*#batch 3"],
283
+ xy: Float[Tensor, "*#batch 2"],
284
+ extrinsics: Float[Tensor, "*#batch 4 4"],
285
+ intrinsics: Float[Tensor, "*#batch 3 3"],
286
+ ) -> Float[Tensor, " *batch"]:
287
+ """Calculate the depths that correspond to the specified 2D points on the epipolar
288
+ lines defined by the origins and directions. The extrinsics and intrinsics are for
289
+ the images the 2D points lie on.
290
+ """
291
+ xyz = lift_to_3d(origins, directions, xy, extrinsics, intrinsics)
292
+ return (xyz - origins).norm(dim=-1)