luca115's picture
Deploy from 3dvalley spaces/make-it-animatable (part 2)
6f20840 verified
Raw History Blame Contribute Delete
45.4 kB
import argparse import gc import os import signal import sys import warnings from contextlib import suppress from datetime import datetime from functools import partial, wraps from time import perf_counter from typing import Callable, ParamSpec, TypeVar import numpy as np import torch import torch.distributed as dist import torch.nn as nn import torch.nn.functional as F import trimesh from pytorch3d.transforms import Transform3d Tensor_or_Array = TypeVar("Tensor_or_Array", torch.Tensor, np.ndarray) class HiddenPrints: def __init__(self, enable=True, suppress_err=False): self.enabled = enable self.suppress_err = suppress_err self._original_stdout = sys.stdout self._original_stderr = sys.stderr def __enter__(self): if not self.enabled: return sys.stdout = open(os.devnull, "w") if self.suppress_err: sys.stderr = open(os.devnull, "w") def __exit__(self, exc_type, exc_val, exc_tb): if not self.enabled: return sys.stdout.close() sys.stdout = self._original_stdout if self.suppress_err: sys.stderr.close() sys.stderr = self._original_stderr class TimePrints: def __init__(self, enable=True): self.enabled = enable self._original_stdout = sys.stdout def write(self, text): if self.enabled: timestamp = datetime.now().strftime("%m/%d %H:%M:%S") self._original_stdout.write(text.replace("\n", " [{}]\n".format(str(timestamp)))) else: self._original_stdout.write(text) def flush(self): self._original_stdout.flush() def enable(self): self.enabled = True sys.stdout = self def disable(self): self.enabled = False sys.stdout = self._original_stdout def __enter__(self): self.enable() def __exit__(self, exc_type, exc_val, exc_tb): self.disable() class Timing: def __init__(self, enable=True, msg="Time cost:", repeat_times=1, print_fn=print, print_even_on_error=False): self.enable = enable self.msg = str(msg) self.repeat_times = repeat_times if not isinstance(print_fn, (list, tuple)): print_fn = [print_fn] self.print_fn = tuple(print_fn) self.print_even_on_error = print_even_on_error def __enter__(self): if self.enable: self.tic = perf_counter() return self def __exit__(self, exc_type, exc_val, exc_tb): if self.enable and (exc_type is None or self.print_even_on_error): self.toc = perf_counter() info = f"{self.msg} {(self.toc - self.tic) / self.repeat_times:.2f}s" for fn in self.print_fn: fn(info) T = TypeVar("T") def __call__(self, func: Callable[..., T]) -> Callable[..., T]: import inspect @wraps(func) def wrapped_function(*args, **kwargs): with self: output = func(*args, **kwargs) if inspect.isgeneratorfunction(func): yield from output else: return output return wrapped_function P = ParamSpec("P") R = TypeVar("R") def robust_dataloader(func: Callable[P, R]) -> Callable[P, R]: """ A generic decorator that seamlessly intercepts DataLoader and injects breakpoint resume capability, defending against OS-level crashes (e.g., bpy Segmentation fault) during multiprocessing. """ @wraps(func) def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: class _CachedSampler: """A dummy sampler that yields from a pre-cached sequence.""" def __init__(self, sequence): self.sequence = sequence def __iter__(self): return iter(self.sequence) def __len__(self): return len(self.sequence) class RobustWrapper: def __init__(self, original_dl: torch.utils.data.DataLoader): self.original_dl = original_dl self.dataset = original_dl.dataset self.target_length = len(original_dl) self.epoch_sequence = None def __len__(self): return self.target_length def _create_loader(self, skip_batches: int): dl_kwargs = {"dataset": self.dataset} if self.original_dl.batch_sampler is not None: if self.epoch_sequence is None: self.epoch_sequence = list(self.original_dl.batch_sampler) remaining = self.epoch_sequence[skip_batches:] if not remaining: return None dl_kwargs["batch_sampler"] = _CachedSampler(remaining) else: if self.epoch_sequence is None: self.epoch_sequence = list(self.original_dl.sampler) remaining = self.epoch_sequence[skip_batches:] if not remaining: return None dl_kwargs["sampler"] = _CachedSampler(remaining) dl_kwargs["batch_size"] = None # Inherit valid properties from the original dataloader valid_keys = [ "num_workers", "collate_fn", "pin_memory", "timeout", "worker_init_fn", "multiprocessing_context", "generator", "persistent_workers", "pin_memory_device", ] for key in valid_keys: if hasattr(self.original_dl, key): dl_kwargs[key] = getattr(self.original_dl, key) if dl_kwargs.get("num_workers", 0) > 0 and hasattr(self.original_dl, "prefetch_factor"): dl_kwargs["prefetch_factor"] = self.original_dl.prefetch_factor return torch.utils.data.DataLoader(**dl_kwargs) def __iter__(self): yielded_batches = 0 current_dl = self._create_loader(skip_batches=0) if current_dl is None: return current_iter = iter(current_dl) # Disable PyTorch's asynchronous signal handling to prevent mid-execution interrupts if hasattr(signal, "SIGCHLD"): signal.signal(signal.SIGCHLD, signal.SIG_DFL) while yielded_batches < self.target_length: try: batch = next(current_iter) yield batch yielded_batches += 1 except StopIteration: break except RuntimeError as e: error_msg = str(e).lower() crash_keywords = [ "segmentation fault", "killed by signal", "exited unexpectedly", ] if any(kw in error_msg for kw in crash_keywords): warnings.warn( f"Intercepted OS-level worker crash at batch {yielded_batches}/{self.target_length}. " f"Cleaning up memory and seamlessly restarting dataloader...", ) # Safely terminate the background pin_memory_thread to prevent memory/FD leaks if hasattr(current_iter, "_pin_memory_thread_done_event"): current_iter._pin_memory_thread_done_event.set() if hasattr(current_iter, "_worker_result_queue"): with suppress(Exception): current_iter._worker_result_queue.put((None, None)) current_iter._worker_result_queue.cancel_join_thread() # Prevent deadlock in pin_memory_thread.join() during cleanup if hasattr(current_iter, "_pin_memory_thread"): del current_iter._pin_memory_thread # if hasattr(current_iter, "_shutdown"): # current_iter._shutdown = True del current_iter del current_dl gc.collect() current_dl = self._create_loader(skip_batches=yielded_batches) if current_dl is None: break current_iter = iter(current_dl) if hasattr(signal, "SIGCHLD"): signal.signal(signal.SIGCHLD, signal.SIG_DFL) else: raise e # Intercept and wrap DataLoader arguments new_args = list(args) for i, arg in enumerate(new_args): if isinstance(arg, torch.utils.data.DataLoader): new_args[i] = RobustWrapper(arg) for k, v in kwargs.items(): if isinstance(v, torch.utils.data.DataLoader): kwargs[k] = RobustWrapper(v) return func(*new_args, **kwargs) return wrapper def fix_random(seed=0): import random import torch.backends.cudnn import torch.cuda random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) # torch.cuda.manual_seed(seed) torch.cuda.manual_seed_all(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False # torch.use_deterministic_algorithms(True) # os.environ["PYTHONHASHSEED"] = "0" # os.environ["CUDA_LAUNCH_BLOCKING"] = "1" # os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" def str2bool(v: str) -> bool: if isinstance(v, bool): return v elif isinstance(v, str): v = v.strip().lower() if v in ("yes", "true", "t", "y", "1"): return True elif v in ("no", "false", "f", "n", "0"): return False raise argparse.ArgumentTypeError(f"Unsupported value encountered: {v}") def _str2list(v: str, element_type=None) -> list: if not isinstance(v, (list, tuple, set)): v = v.lstrip("[").rstrip("]") v = v.split(",") v = list(map(str.strip, v)) if element_type is not None: v = list(map(element_type, v)) return v def str2list(type=None): return partial(_str2list, element_type=type) def dir_path(path: str): if os.path.isdir(path): return path else: raise NotADirectoryError(os.path.abspath(path)) def file_path(path: str): if os.path.isfile(path): return path else: raise FileNotFoundError(os.path.abspath(path)) def find_ckpt(ckpt_dir: str, epoch=-1, prefix="checkpoint-", suffix=".pth"): if not os.path.exists(ckpt_dir): raise FileNotFoundError(os.path.abspath(ckpt_dir)) if os.path.isfile(ckpt_dir): return ckpt_dir elif not os.path.isdir(ckpt_dir): raise NotADirectoryError(os.path.abspath(ckpt_dir)) if epoch >= 0: filepath = os.path.join(ckpt_dir, f"{prefix}{epoch}{suffix}") if os.path.isfile(filepath): return filepath else: raise FileNotFoundError(os.path.abspath(filepath)) file_list = [p for p in os.listdir(ckpt_dir) if p.startswith(prefix) and p.endswith(suffix)] if not file_list: raise FileNotFoundError(os.path.abspath(ckpt_dir)) file_list.sort( key=lambda x: ( int(x.split(prefix)[-1].split(suffix)[0]) if x.split(prefix)[-1].split(suffix)[0].isdigit() else -99 ), reverse=True, ) return os.path.join(ckpt_dir, file_list[0]) def synchronize(): """ Helper function to synchronize (barrier) among all processes when using distributed training """ if not dist.is_available(): return if not dist.is_initialized(): return world_size = dist.get_world_size() if world_size == 1: return dist.barrier() def init_dist(): from datetime import timedelta assert torch.cuda.is_available(), "cuda is not available" assert ( "RANK" in os.environ and "WORLD_SIZE" in os.environ ), "To use distributed mode, use `python -m torch.distributed.launch` or `torchrun` to launch the program" local_rank = int(os.environ["LOCAL_RANK"]) device_count = torch.cuda.device_count() if int(os.environ["LOCAL_WORLD_SIZE"]) <= device_count: backend = "nccl" device_id = local_rank os.environ["NCCL_ASYNC_ERROR_HANDLING"] = "1" else: if local_rank == 0: print("Using gloo as backend, because NCCL does not support using the same device for multiple ranks") backend = "gloo" device_id = local_rank % device_count # torch.cuda.set_device(device_id) devices_list = os.environ.get("CUDA_VISIBLE_DEVICES", list(range(device_count))) devices_list = [int(d.strip()) for d in devices_list.split(",")] if isinstance(devices_list, str) else devices_list os.environ["CUDA_VISIBLE_DEVICES"] = str(devices_list[device_id]) dist.init_process_group(backend=backend, init_method="env://", timeout=timedelta(hours=10)) synchronize() return local_rank, dist.get_world_size() def get_local_index(total_num: int) -> tuple[int, int, int]: import math if "RANK" in os.environ and "WORLD_SIZE" in os.environ: local_rank, world_size = init_dist() num_per_node = math.ceil(total_num / world_size) idx_begin = local_rank * num_per_node if local_rank != world_size - 1: if idx_begin == 0: idx_begin = -1 idx_end = (local_rank + 1) * num_per_node else: idx_end = total_num else: local_rank, world_size = 0, 1 idx_begin, idx_end = -1, total_num if idx_begin == -1: idx_begin = 0 return local_rank, idx_begin, idx_end class DummySummaryWriter: def __init__(self, *args, **kwargs): pass def __call__(self, *args, **kwargs): return self def __getattr__(self, *args, **kwargs): return self def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): pass def normalize_pts(points: np.ndarray, keep_ratio=True) -> np.ndarray: """Normalize (N, 3) points to [-1, 1] according to its bounding box""" min_vals = np.min(points, axis=0) max_vals = np.max(points, axis=0) scale = 2.0 / np.max(max_vals - min_vals) if keep_ratio else 2.0 / (max_vals - min_vals) center = (max_vals + min_vals) / 2.0 normalized_points = (points - center) * scale return normalized_points def normalize_pts_torch(points: torch.Tensor, keep_ratio=True) -> torch.Tensor: """Normalize (B, N, 3) points to [-1, 1] according to its bounding box""" min_vals = torch.min(points, dim=1, keepdim=True)[0] max_vals = torch.max(points, dim=1, keepdim=True)[0] scale = 2.0 / torch.max(max_vals - min_vals, dim=2, keepdim=True)[0] if keep_ratio else 2.0 / (max_vals - min_vals) center = (max_vals + min_vals) / 2.0 normalized_points = (points - center) * scale return normalized_points class NormalizePoints(nn.Module): def __init__(self, keep_ratio=True): self.keep_ratio = keep_ratio self.transl = None self.scaling = None def clear(self): self.transl = None self.scaling = None return self def transform_points(self, points: torch.Tensor): """points: (B, N, 3)""" # return normalize_pts_torch(points, self.keep_ratio) if None in (self.transl, self.scaling): min_vals = torch.min(points, dim=1, keepdim=True)[0] max_vals = torch.max(points, dim=1, keepdim=True)[0] if self.keep_ratio: self.scaling = 2.0 / torch.max(max_vals - min_vals, dim=2, keepdim=True)[0] else: self.scaling = 2.0 / (max_vals - min_vals) center = (max_vals + min_vals) / 2.0 self.transl = -center return (points + self.transl) * self.scaling def forward(self, points: torch.Tensor): return self.transform_points(points) def get_normalize_transform(points: torch.Tensor, keep_ratio=True, recenter=True): """PyTorch3D version points: (B, N, 3) """ from pytorch3d.transforms import Scale, Translate min_vals = torch.min(points, dim=1, keepdim=True)[0] max_vals = torch.max(points, dim=1, keepdim=True)[0] if not recenter: max_vals = torch.maximum(max_vals.abs(), min_vals.abs()) min_vals = -max_vals if keep_ratio: scaling = 2.0 / torch.max(max_vals - min_vals, dim=2, keepdim=True)[0] scaling = scaling.tile(3) else: scaling = 2.0 / (max_vals - min_vals) if recenter: center = (max_vals + min_vals) / 2.0 transl = Translate(-center.squeeze(1)) scaling = Scale(scaling.squeeze(1)) transform = transl.compose(scaling) if recenter else scaling return transform def get_homogeneous(xyz: Tensor_or_Array) -> Tensor_or_Array: """ Args: xyz: (..., 3) Returns: (..., 4) """ module = torch if isinstance(xyz, torch.Tensor) else np ones = module.ones_like(xyz[..., 0:1]) xyz_homo = module.concatenate((xyz, ones), -1) return xyz_homo def apply_transform(xyz: Tensor_or_Array, transform: Tensor_or_Array) -> Tensor_or_Array: """ Args: xyz: (..., 3) transform: (..., 4, 4) left-multiplied transformation matrix Returns: (..., 3) """ module = torch if isinstance(xyz, torch.Tensor) else np xyz_homo = get_homogeneous(xyz) xyz_homo_transformed = module.einsum("...ij,...j->...i", transform, xyz_homo) xyz_homo_transformed = xyz_homo_transformed[..., :3] return xyz_homo_transformed def matrix_to_quat(matrix: Tensor_or_Array) -> Tensor_or_Array: """ Args: matrix (..., 3, 3) rotation matrix Returns: (..., 4) quaternions """ module = torch if isinstance(matrix, torch.Tensor) else np if module is torch: from pytorch3d.transforms import matrix_to_quaternion # ! no need for `transpose` since the PyTorch3D's quaternion also expects a right-multiplied version (opposite vector part) quat = matrix_to_quaternion(matrix) else: from scipy.spatial.transform import Rotation as R quat = R.from_matrix(matrix).as_quat(canonical=True, scalar_first=True) return quat def quat_to_matrix(quat: Tensor_or_Array) -> Tensor_or_Array: """ Args: quat: (..., 4) quaternions Returns: (..., 3, 3) rotation matrix """ module = torch if isinstance(quat, torch.Tensor) else np # Prevent NaN zero_norm_mask = ~module.isclose((quat * quat).sum(-1), module.ones_like(quat[..., 0])) zero_norm_mask = zero_norm_mask[..., None] if zero_norm_mask.any(): r_zero = module.zeros_like(quat) r_zero[..., 0] = 1.0 quat = module.where(module.broadcast_to(zero_norm_mask, quat.shape), r_zero, quat) if module is torch: from pytorch3d.transforms import quaternion_to_matrix matrix = quaternion_to_matrix(quat) else: from scipy.spatial.transform import Rotation as R matrix = R.from_quat(quat, scalar_first=True).as_matrix() return matrix def decompose_transform(transform: Tensor_or_Array, return_quat=True, return_concat=True) -> Tensor_or_Array: """Decompose homogeneous transformation matrix into translation + rotation + scaling (in concatenation) The correct re-compose order: rotation, scaling, translation Args: transform: (..., 4, 4) left-multiplied transformation matrix """ module = torch if isinstance(transform, torch.Tensor) else np norm_fn = (lambda x: torch.norm(x, dim=-2)) if module is torch else (lambda x: np.linalg.norm(x, axis=-2)) transl = transform[..., :3, 3] scaling = norm_fn(transform[..., :3, :3]) rotation = transform[..., :3, :3] / scaling[..., None, :] if return_quat: rotation = matrix_to_quat(rotation) elif return_concat: rotation = rotation.reshape(*rotation.shape[:-2], 3 * 3) if return_concat: return module.concatenate((transl, rotation, scaling), -1) return transl, rotation, scaling def compose_transform(transform: Tensor_or_Array) -> Tensor_or_Array: """ Args: transform: (..., 3+r+3) decomposed translation + rotation + scaling (in concatenation) Returns: (..., 4, 4) left-multiplied transformation matrix (rotation + scaling + translation) """ if isinstance(transform, (tuple, list)): # return_concat=False if len(transform) == 3: transl, rotation, scaling = transform elif len(transform) == 2: transl, rotation = transform scaling = None else: raise ValueError(f"Invalid transform: {transform}") else: print(f"Assuming translation and scaling both have ndim=3 in {transform.shape[-1]=}") transl, rotation, scaling = transform[..., :3], transform[..., 3:-3], transform[..., -3:] module = torch if isinstance(rotation, torch.Tensor) else np if rotation.shape[-2:] != (3, 3): # return_quat=False if rotation.shape[-1] == 9: rotation = rotation.reshape(*rotation.shape[:-1], 3, 3) elif rotation.shape[-1] == 4: rotation = quat_to_matrix(rotation) assert not module.isnan(rotation).any() else: raise ValueError(f"Invalid rotation shape: {rotation.shape}") matrix = module.zeros((*rotation.shape[:-2], 4, 4), dtype=rotation.dtype) if module is torch: matrix = matrix.to(rotation) matrix[..., 3, 3] = 1.0 matrix[..., :3, :3] = rotation if scaling is not None: matrix[..., :3, :3] *= scaling[..., None, :] matrix[..., :3, 3] = transl return matrix def quat_transl_to_dualquat(quat: torch.Tensor, transl: torch.Tensor, transl_first=False): """ Args: quat: (..., 4) transl: (..., 3) Returns: (..., 8) rotation + translation as quaternions """ from pytorch3d.transforms import quaternion_raw_multiply, standardize_quaternion q_r = standardize_quaternion(F.normalize(quat, dim=-1)) q_d = 0.5 * quaternion_raw_multiply(torch.cat([torch.zeros_like(transl[..., :1]), transl], dim=-1), q_r) # translation quaternion should not normalized/standardized return torch.cat([q_d, q_r], dim=-1) if transl_first else torch.cat([q_r, q_d], dim=-1) def dualquat_to_quat_transl(dq: torch.Tensor, concat=False, transl_first=False): """ Args ---------- dq: (..., 8) rotation + translation as quaternions Returns ---------- quat: (..., 4) transl: (..., 3) """ from pytorch3d.transforms import quaternion_invert, quaternion_raw_multiply q_r, q_d = dq[..., :4], dq[..., 4:] if transl_first: q_r, q_d = q_d, q_r transl = 2.0 * quaternion_raw_multiply(q_d, quaternion_invert(q_r)) transl = transl[..., 1:] quat_trans = (transl, q_r) if transl_first else (q_r, transl) if concat: quat_trans = torch.cat(quat_trans, dim=-1) return quat_trans def matrix_to_ortho6d(matrix: torch.Tensor): """ Args: rotation matrix: (..., 3, 3) Returns: (..., 6) ortho6d """ sh = matrix.shape return matrix[..., :-1].transpose(-1, -2).reshape(*sh[:-2], 6) def ortho6d_to_matrix(ortho6d: torch.Tensor): """On the Continuity of Rotation Representations in Neural Networks https://github.com/papagina/RotationContinuity/blob/master/sanity_test/code/tools.py Args: ortho6d: (..., 6) Returns: (..., 3, 3) rotation matrix """ x_raw = ortho6d[..., 0:3] y_raw = ortho6d[..., 3:6] x = F.normalize(x_raw, dim=-1) z = torch.cross(x, y_raw, dim=-1) z = F.normalize(z, dim=-1) y = torch.cross(z, x, dim=-1) return torch.stack((x, y, z), dim=-1) def get_rotation_center(matrix: Tensor_or_Array) -> Tensor_or_Array: """https://math.stackexchange.com/questions/4397763/3d-rotation-matrix-around-a-point-not-origin/4397766 Args: matrix: (..., 4, 4) homogeneous transformation Returns: (..., 3) """ module = torch if isinstance(matrix, torch.Tensor) else np R = matrix[..., :3, :3] t = matrix[..., :3, 3] # From the formula: t = v - Rv # Therefore: v - Rv = t # (I - R)v = t # v = (I - R)^(-1) t I = module.eye(3) I = module.broadcast_to(I, R.shape) return module.linalg.solve(I - R, t) def get_rotation_about_point(rotation: Tensor_or_Array, center: Tensor_or_Array) -> Tensor_or_Array: """ Args: rotation: (..., 3, 3) center: (..., 3) Returns: (..., 4, 4) homogeneous transformation matrix """ module = torch if isinstance(rotation, torch.Tensor) else np translation = center - module.matmul(rotation, center[..., None]).squeeze(-1) batch_shape = rotation.shape[:-2] matrix = module.zeros(batch_shape + (4, 4), dtype=rotation.dtype) if module is torch: matrix = matrix.to(rotation) matrix[..., :3, :3] = rotation matrix[..., :3, 3] = translation matrix[..., 3, 3] = 1.0 return matrix def compose_transform_trt(transform: Tensor_or_Array) -> Tensor_or_Array: """ Args: transform: (..., 3+r+3) source + rotation + target (in concatenation) Returns: (..., 4, 4) transformation matrix (translation (from source to origin) + rotation + translation (from origin to target)) """ if isinstance(transform, (tuple, list)): assert len(transform) == 3, f"Invalid transform: {transform}" source, rotation, target = transform else: source, rotation, target = transform[..., :3], transform[..., 3:-3], transform[..., -3:] module = torch if isinstance(rotation, torch.Tensor) else np if rotation.shape[-2:] != (3, 3): if rotation.shape[-1] == 9: rotation = rotation.reshape(*rotation.shape[:-1], 3, 3) elif rotation.shape[-1] == 4: rotation = quat_to_matrix(rotation) assert not module.isnan(rotation).any() else: raise ValueError(f"Invalid rotation shape: {rotation.shape}") matrix = get_rotation_about_point(rotation, source) matrix[..., :3, 3] += target - source return matrix def to_pose_local(pose_repr: torch.Tensor, input_mode: str, return_quat=True) -> torch.Tensor: """ Args: pose_repr: (..., r) Returns: (..., 4) if `return_quat` else (..., 3, 3) """ if input_mode == "local_quat": return pose_repr if return_quat else quat_to_matrix(pose_repr) elif input_mode == "local_ortho6d": matrix = ortho6d_to_matrix(pose_repr) return matrix_to_quat(matrix) if return_quat else matrix else: raise NotImplementedError(f"{input_mode=}") def to_pose_matrix(pose_repr: torch.Tensor, input_mode: str, source: torch.Tensor = None) -> torch.Tensor: """ Args: pose_repr: (..., t+r) Returns: (..., 4, 4) """ if input_mode == "transl_quat": return compose_transform(torch.split(pose_repr, [3, 4], dim=-1)) elif input_mode == "dual_quat": transl_quat = dualquat_to_quat_transl(pose_repr, concat=False, transl_first=True) return compose_transform(transl_quat) elif input_mode == "transl_ortho6d": transl, rotation = torch.split(pose_repr, [3, 6], dim=-1) rotation = ortho6d_to_matrix(rotation) rotation = rotation.reshape(*rotation.shape[:-2], 3 * 3) return compose_transform([transl, rotation]) elif input_mode == "transl_matrix": return compose_transform(torch.split(pose_repr, [3, 9], dim=-1)) elif input_mode == "target_quat": target, rotation = torch.split(pose_repr, [3, 4], dim=-1) assert source is not None and source.shape == target.shape return compose_transform_trt([source, rotation, target]) elif input_mode == "target_ortho6d": target, rotation = torch.split(pose_repr, [3, 6], dim=-1) assert source is not None and source.shape == target.shape rotation = ortho6d_to_matrix(rotation) return compose_transform_trt([source, rotation, target]) elif input_mode == "target_matrix": target, rotation = torch.split(pose_repr, [3, 9], dim=-1) assert source is not None and source.shape == target.shape return compose_transform_trt([source, rotation, target]) else: raise NotImplementedError(f"{input_mode=}") def pose_local_to_global( pose_local: torch.Tensor, joints: torch.Tensor, parents: torch.Tensor, global_transl: torch.Tensor = None, relative_to_source=False, ) -> torch.Tensor: """Reference:https://github.com/vchoutas/smplx/blob/main/smplx/lbs.py Args: pose_local: (..., K, 3, 3) rotation axis: global coordinates; rotation origin: source joint with target-posed (or source-posed if `relative_to_source`) parents joints: (..., K, 3) parents: (K,) parent indices from the kinematic tree (root joint has no effect; make sure children appear after parents) Returns: pose_global: (..., K, 4, 4) The rigid transformations for all the joints with respect to the origin posed_joints: (..., K, 3) The locations of the joints after applying the pose rotations """ if pose_local.shape[-1] == 4: pose_local = quat_to_matrix(pose_local) elif pose_local.shape[-1] == 6: pose_local = ortho6d_to_matrix(pose_local) assert pose_local.shape[-2:] == (3, 3) assert pose_local.shape[-3] == joints.shape[-2] == parents.shape[0] K = pose_local.shape[-3] root_matrix = get_rotation_about_point(pose_local[..., 0, :, :], joints[..., 0, :]) root_posed = joints[..., 0, :] if global_transl is not None: root_matrix[..., :3, 3] += global_transl root_posed = root_posed + global_transl pose_global = [root_matrix] posed_joints = [root_posed] for i in range(1, K): parent_matrix = pose_global[parents[i].item()] posed = apply_transform(joints[..., i, :], parent_matrix) # No need for rotation here, since the local rotation is about itself posed_joints.append(posed) if relative_to_source: matrix = get_rotation_about_point(pose_local[..., i, :, :], joints[..., i, :]) pose = torch.matmul(parent_matrix, matrix) else: matrix = get_rotation_about_point(pose_local[..., i, :, :], posed) pose = torch.matmul(matrix, parent_matrix) pose_global.append(pose) pose_global = torch.stack(pose_global, dim=-3) posed_joints = torch.stack(posed_joints, dim=-2) return pose_global, posed_joints def pose_rot_to_global( pose_rot: torch.Tensor, joints: torch.Tensor, parents: torch.Tensor, global_transl: torch.Tensor = None ) -> torch.Tensor: """ Args: pose_rot: (..., K, 3, 3) global rotations joints: (..., K, 3) parents: (K,) parent indices from the kinematic tree (root joint has no effect; make sure children appear after parents) Returns: pose_global: (..., K, 4, 4) The rigid transformations for all the joints with respect to the origin posed_joints: (..., K, 3) The locations of the joints after applying the pose rotations """ if pose_rot.shape[-1] == 4: pose_rot = quat_to_matrix(pose_rot) elif pose_rot.shape[-1] == 6: pose_rot = ortho6d_to_matrix(pose_rot) assert pose_rot.shape[-2:] == (3, 3) assert pose_rot.shape[-3] == joints.shape[-2] == parents.shape[0] K = pose_rot.shape[-3] root_matrix = get_rotation_about_point(pose_rot[..., 0, :, :], joints[..., 0, :]) root_posed = joints[..., 0, :] if global_transl is not None: root_matrix[..., :3, 3] += global_transl root_posed = root_posed + global_transl pose_global = [root_matrix] posed_joints = [root_posed] for i in range(1, K): parent_matrix = pose_global[parents[i].item()] posed = apply_transform(joints[..., i, :], parent_matrix) posed_joints.append(posed) matrix = get_rotation_about_point(pose_rot[..., i, :, :], joints[..., i, :]) matrix[..., :3, 3] += posed - joints[..., i, :] pose_global.append(matrix) pose_global = torch.stack(pose_global, dim=-3) posed_joints = torch.stack(posed_joints, dim=-2) return pose_global, posed_joints def filter_nan(x: torch.Tensor, target: torch.Tensor): x = x.reshape(-1, x.shape[-1]) target = target.reshape(-1, target.shape[-1]) mask_row = (~torch.isnan(target)).all(dim=-1) x, target = x[mask_row, :], target[mask_row, :] mask_col = (~torch.isnan(target)).all(dim=0) x, target = x[:, mask_col], target[:, mask_col] return x, target def _sample_mesh(mesh: trimesh.Trimesh, num_pts: int, return_normal=False): if isinstance(mesh, trimesh.PointCloud): assert not return_normal, "Point clouds do not have normals" verts = mesh.vertices samples = verts[np.random.choice(verts.shape[0], size=num_pts, replace=num_pts > verts.shape[0])] else: samples, face_index = trimesh.sample.sample_surface(mesh, num_pts) if return_normal: normals = mesh.face_normals[face_index] samples = np.concatenate([samples, normals], axis=-1) return np.array(samples) def sample_near_positions( mesh: trimesh.Trimesh, positions: np.ndarray, num_pts: np.ndarray, radius: float, get_normals=False ): samples = [] for i, center in enumerate(positions): if num_pts[i] <= 0: continue box = trimesh.creation.box(extents=(radius, radius, radius)) box.vertices += center # trimesh.util.concatenate([mesh, box, trimesh.creation.axis()]).export("sample.ply") attn_mesh = ( trimesh.PointCloud(mesh.vertices[mesh.kdtree.query_ball_point(center, radius)]) if isinstance(mesh, trimesh.PointCloud) else mesh.slice_plane(box.facets_origin, -box.facets_normal) ) if attn_mesh.vertices.shape[0] == 0: continue samples.append(_sample_mesh(attn_mesh, num_pts[i], return_normal=get_normals)) return np.concatenate(samples, axis=0) if samples else np.empty((0, 6 if get_normals else 3)) def get_geometry_guided_sampling_centers(mesh: trimesh.Trimesh, num_pts: int, radius: float, num_fps=10, num_centers=5): import potpourri3d as pp3d from scipy.spatial import distance_matrix from torch_cluster import fps if isinstance(mesh, trimesh.PointCloud): raise NotImplementedError("Point cloud is not supported") # Preprocess mesh to prevent error in geodesic distance calculation mesh = trimesh.Trimesh(vertices=mesh.vertices, faces=mesh.faces, process=True) verts = np.array(mesh.vertices) idx = fps(torch.from_numpy(verts), ratio=num_fps / verts.shape[0]).numpy() # mesh_ = mesh.copy() # # trimesh.smoothing.filter_humphrey(mesh_, iterations=10, beta=1.0) # # diff = np.linalg.norm(mesh_.vertices - mesh.vertices, axis=-1) # diff = mesh_.simplify_quadric_decimation(face_count=mesh_.faces.shape[0] / 10).nearest.vertex(mesh_.vertices)[0] # import matplotlib; cmap = matplotlib.colormaps.get_cmap("plasma") # fmt: skip # mesh_ = trimesh.Trimesh(vertices=mesh_.vertices, faces=mesh_.faces, vertex_colors=cmap(diff / diff.max())[:, :3]) # mesh_.export("sample.obj") # area_ratio = [] # for i_v in idx: # box = trimesh.creation.box(extents=(radius, radius, radius)) # box.vertices += verts[i_v] # mesh_sliced = mesh.slice_plane(box.facets_origin, -box.facets_normal) # trimesh.repair.fill_holes(mesh_sliced) # mesh_area = mesh_sliced.facets_area.sum() # # oriented_box = mesh_sliced.bounding_cylinder # oriented_box = mesh_sliced.bounding_box_oriented # box_area = oriented_box.facets_area.sum() # area_ratio.append(mesh_area / box_area) # if area_ratio[-1] > 0.1: # mesh_ = trimesh.util.concatenate([mesh_, oriented_box]) # mesh_.export("sample.obj") dist = distance_matrix(verts[idx], verts) solver = pp3d.MeshHeatMethodDistanceSolver(verts, mesh.faces) dist_geo = np.empty((idx.shape[0], verts.shape[0])) for i, i_v in enumerate(idx): dist_geo[i] = solver.compute_distance(i_v) dist_ratio = dist / (dist_geo + 1e-8) dist_ratio = (dist_ratio < 0.25).sum(-1) / verts.shape[0] idx_top = idx[np.argsort(dist_ratio)[::-1][:num_centers]] geo_centers = verts[idx_top] # Refine these centers within local neighborhood for i, geo_center in enumerate(geo_centers): neighbors = verts[mesh.kdtree.query_ball_point(geo_center, radius)] geo_centers[i] = np.mean(neighbors, axis=0) geo_num_pts = [num_pts // num_centers] * (num_centers - 1) geo_num_pts.append(num_pts - sum(geo_num_pts)) return geo_centers, geo_num_pts def sample_mesh( mesh: trimesh.Trimesh, num_points: int, get_normals=False, attn_ratio=0.0, attn_centers: np.ndarray = None, attn_rel_radius=0.15, attn_geo_ratio=0.0, ): if attn_ratio <= 0 and attn_geo_ratio <= 0: pts = _sample_mesh(mesh, num_points, return_normal=get_normals) else: num_pts_attn = int(num_points * attn_ratio) num_pts_attn_geo = int(num_points * attn_geo_ratio) num_points_main = num_points - num_pts_attn - num_pts_attn_geo pts = _sample_mesh(mesh, num_points_main, return_normal=get_normals) if attn_centers is None or num_pts_attn == 0: attn_centers = np.empty((0, 3)) num_pts_attn_list = [] else: num_pts_attn_list = [num_pts_attn // len(attn_centers)] * (len(attn_centers) - 1) num_pts_attn_list.append(num_pts_attn - sum(num_pts_attn_list)) attn_radius = attn_rel_radius * (np.max(mesh.vertices) - np.min(mesh.vertices)) if num_pts_attn_geo > 0: geo_centers, geo_num_pts = get_geometry_guided_sampling_centers( mesh, num_pts=num_pts_attn_geo, radius=attn_radius ) attn_centers = np.concatenate([attn_centers, geo_centers], axis=0) num_pts_attn_list.extend(geo_num_pts) pts_attn = sample_near_positions(mesh, attn_centers, num_pts_attn_list, attn_radius, get_normals=get_normals) pts = np.concatenate([pts, pts_attn], axis=0) if pts.shape[0] < num_points: # some of the specified attention regions are empty, resample over the whole mesh pts_extra = _sample_mesh(mesh, num_points - pts.shape[0], return_normal=get_normals) pts = np.concatenate([pts, pts_extra], axis=0) # assert pts.shape[0] == num_points # colors = np.ones_like(pts) # colors[:num_points_main, :] = 0 # trimesh.Trimesh(pts, vertex_colors=colors).export("sample.ply") return pts def load_gs(path: str, compatible=True): """https://github.com/3DTopia/LGM/blob/main/core/gs.py""" from plyfile import PlyData plydata = PlyData.read(path) xyz = np.stack( ( np.asarray(plydata.elements[0]["x"]), np.asarray(plydata.elements[0]["y"]), np.asarray(plydata.elements[0]["z"]), ), axis=1, ) # print("Number of points at loading : ", xyz.shape[0]) opacities = np.asarray(plydata.elements[0]["opacity"])[..., np.newaxis] shs = np.zeros((xyz.shape[0], 3)) shs[:, 0] = np.asarray(plydata.elements[0]["f_dc_0"]) shs[:, 1] = np.asarray(plydata.elements[0]["f_dc_1"]) shs[:, 2] = np.asarray(plydata.elements[0]["f_dc_2"]) scale_names = [p.name for p in plydata.elements[0].properties if p.name.startswith("scale_")] scales = np.zeros((xyz.shape[0], len(scale_names))) for idx, attr_name in enumerate(scale_names): scales[:, idx] = np.asarray(plydata.elements[0][attr_name]) rot_names = [p.name for p in plydata.elements[0].properties if p.name.startswith("rot_")] rots = np.zeros((xyz.shape[0], len(rot_names))) for idx, attr_name in enumerate(rot_names): rots[:, idx] = np.asarray(plydata.elements[0][attr_name]) gaussians = np.concatenate([xyz, opacities, scales, rots, shs], axis=1) gaussians = torch.from_numpy(gaussians).float() # cpu if compatible: gaussians[..., 3:4] = torch.sigmoid(gaussians[..., 3:4]) gaussians[..., 4:7] = torch.exp(gaussians[..., 4:7]) gaussians[..., 11:] = 0.28209479177387814 * gaussians[..., 11:] + 0.5 return gaussians def save_gs(gaussians: torch.Tensor, path: str, compatible=True, prune=False): """https://github.com/3DTopia/LGM/blob/main/core/gs.py""" # gaussians: [N, 14] from plyfile import PlyData, PlyElement means3D = gaussians[..., 0:3].contiguous().float() opacity = gaussians[..., 3:4].contiguous().float() scales = gaussians[..., 4:7].contiguous().float() rotations = gaussians[..., 7:11].contiguous().float() shs = gaussians[..., 11:].unsqueeze(1).contiguous().float() # [N, 1, 3] if prune: # prune by opacity mask = opacity.squeeze(-1) >= 0.005 means3D = means3D[mask] opacity = opacity[mask] scales = scales[mask] rotations = rotations[mask] shs = shs[mask] # invert activation to make it compatible with the original ply format if compatible: # opacity = kiui.op.inverse_sigmoid(opacity) inverse_sigmoid = lambda x: torch.log(x) - torch.log(1 - x) opacity = inverse_sigmoid(opacity) scales = torch.log(scales + 1e-8) shs = (shs - 0.5) / 0.28209479177387814 xyzs = means3D.detach().cpu().numpy() f_dc = shs.detach().transpose(1, 2).flatten(start_dim=1).contiguous().cpu().numpy() opacities = opacity.detach().cpu().numpy() scales = scales.detach().cpu().numpy() rotations = rotations.detach().cpu().numpy() l = ["x", "y", "z"] # All channels except the 3 DC for i in range(f_dc.shape[1]): l.append(f"f_dc_{i}") l.append("opacity") for i in range(scales.shape[1]): l.append(f"scale_{i}") for i in range(rotations.shape[1]): l.append(f"rot_{i}") dtype_full = [(attribute, "f4") for attribute in l] elements = np.empty(xyzs.shape[0], dtype=dtype_full) attributes = np.concatenate((xyzs, f_dc, opacities, scales, rotations), axis=1) elements[:] = list(map(tuple, attributes)) el = PlyElement.describe(elements, "vertex") PlyData([el]).write(path) def transform_gs(gs: torch.Tensor, transform: Transform3d): from pytorch3d.transforms import quaternion_raw_multiply gs = gs.clone() assert isinstance(gs, torch.Tensor) and gs.ndim == 2 and gs.shape[-1] == 14 if not isinstance(transform, Transform3d): transform = Transform3d(matrix=torch.tensor(transform).transpose(-1, -2)) transform = transform.to(gs.device) transl, rotation, scaling = decompose_transform( transform.get_matrix().transpose(-1, -2), return_quat=True, return_concat=False ) xyz = gs[..., :3] if len(transform) > 1: xyz = xyz.unsqueeze(-2) xyz = transform.transform_points(xyz) if len(transform) > 1: xyz = xyz.squeeze(-2) gs[..., :3] = xyz gs[..., 4:7] *= scaling gs[..., 7:11] = quaternion_raw_multiply(rotation, gs[..., 7:11]) return gs def make_archive(input_path: str, output_path: str): import shutil import zipfile if not os.path.exists(input_path): raise FileNotFoundError(input_path) os.makedirs(os.path.dirname(os.path.abspath(output_path)), exist_ok=True) if os.path.isfile(input_path): with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as zipf: zipf.write(input_path, os.path.basename(input_path)) elif os.path.isdir(input_path): shutil.make_archive(output_path, "zip", input_path) else: raise NotImplementedError def convert_3d_format(input_path: str, output_path: str): import aspose.threed as a3d scene = a3d.Scene.from_file(input_path) os.makedirs(os.path.dirname(output_path), exist_ok=True) scene.save(output_path) if __name__ == "__main__": fix_random() pts = torch.randn(4, 10, 3) keep_ratio = True norm_pt = NormalizePoints(keep_ratio=keep_ratio) pts1 = norm_pt.transform_points(pts) norm_pt3d = get_normalize_transform(pts, keep_ratio=keep_ratio) pts2 = norm_pt3d.transform_points(pts) print(torch.allclose(pts1, pts2))
import argparse
import gc
import os
import signal
import sys
import warnings
from contextlib import suppress
from datetime import datetime
from functools import partial, wraps
from time import perf_counter
from typing import Callable, ParamSpec, TypeVar
import numpy as np
import torch
import torch.distributed as dist
import torch.nn as nn
import torch.nn.functional as F
import trimesh
from pytorch3d.transforms import Transform3d
Tensor_or_Array = TypeVar("Tensor_or_Array", torch.Tensor, np.ndarray)
class HiddenPrints:
def __init__(self, enable=True, suppress_err=False):
self.enabled = enable
self.suppress_err = suppress_err
self._original_stdout = sys.stdout
self._original_stderr = sys.stderr
def __enter__(self):
if not self.enabled:
return
sys.stdout = open(os.devnull, "w")
if self.suppress_err:
sys.stderr = open(os.devnull, "w")
def __exit__(self, exc_type, exc_val, exc_tb):
if not self.enabled:
return
sys.stdout.close()
sys.stdout = self._original_stdout
if self.suppress_err:
sys.stderr.close()
sys.stderr = self._original_stderr
class TimePrints:
def __init__(self, enable=True):
self.enabled = enable
self._original_stdout = sys.stdout
def write(self, text):
if self.enabled:
timestamp = datetime.now().strftime("%m/%d %H:%M:%S")
self._original_stdout.write(text.replace("\n", " [{}]\n".format(str(timestamp))))
else:
self._original_stdout.write(text)
def flush(self):
self._original_stdout.flush()
def enable(self):
self.enabled = True
sys.stdout = self
def disable(self):
self.enabled = False
sys.stdout = self._original_stdout
def __enter__(self):
self.enable()
def __exit__(self, exc_type, exc_val, exc_tb):
self.disable()
class Timing:
def __init__(self, enable=True, msg="Time cost:", repeat_times=1, print_fn=print, print_even_on_error=False):
self.enable = enable
self.msg = str(msg)
self.repeat_times = repeat_times
if not isinstance(print_fn, (list, tuple)):
print_fn = [print_fn]
self.print_fn = tuple(print_fn)
self.print_even_on_error = print_even_on_error
def __enter__(self):
if self.enable:
self.tic = perf_counter()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if self.enable and (exc_type is None or self.print_even_on_error):
self.toc = perf_counter()
info = f"{self.msg} {(self.toc - self.tic) / self.repeat_times:.2f}s"
for fn in self.print_fn:
fn(info)
T = TypeVar("T")