diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..a4e72aa7d623b7b5f587ecbe3d669775e8b768e8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +*.jpg filter=lfs diff=lfs merge=lfs -text +*.jpeg filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..94e493b36742dc8129e550ba5e05a3da2e0e58bd --- /dev/null +++ b/app.py @@ -0,0 +1,1499 @@ +# Standard library imports +import os +import sys +import shutil +import glob +import gc +import time +import base64 +import argparse +import tempfile +from datetime import datetime +from pathlib import Path + +# Third-party library imports +import cv2 +import torch +import trimesh +import numpy as np +import gradio as gr +import matplotlib +import matplotlib.pyplot as plt +from scipy.spatial.transform import Rotation + +# Custom module imports +from argus.models.argus import Argus +from argus.utils.pose_enc import pose_encoding_to_extri360 +from argus.utils.geometry import unproject_depth_to_world_points + + +# -------------------------- Argument Parsing -------------------------- +def parse_args(): + parser = argparse.ArgumentParser(description="Argus Gradio Demo") + parser.add_argument( + "--model_path", + type=str, + default=None, + help="Path to pre-trained model weights (.pt file). " + "If not specified, auto-downloads from HuggingFace.", + ) + parser.add_argument( + "--img_size", + type=int, + default=560, + help="Input panoramic image target width (height = width // 2)", + ) + parser.add_argument( + "--crop_ratio", + type=float, + default=0.15, + help="Vertical crop ratio for panoramic image preprocessing (0-0.5)", + ) + parser.add_argument( + "--port", + type=int, + default=7860, + help="Port number for Gradio server", + ) + parser.add_argument( + "--share", + action="store_true", + default=False, + help="Enable Gradio public sharing link", + ) + parser.add_argument( + "--server_name", + type=str, + default="0.0.0.0", + help="Server host address (0.0.0.0 for all interfaces)", + ) + parser.add_argument( + "--device", + type=str, + default=None, + help="Device to use (cuda/cpu). Default: auto-detect", + ) + parser.add_argument( + "--examples_dir", + type=str, + default="examples", + help="Directory containing example scenes", + ) + parser.add_argument( + "--save_tmp", + type=str, + default=None, + help="Directory to persist intermediate files (images, predictions, GLB). " + "If not set, uses system temp dir and cleans up automatically.", + ) + return parser.parse_args() + + +args = parse_args() + +# -------------------------- Global Configuration -------------------------- +# Device configuration: use specified device or auto-detect +DEVICE = args.device if args.device else ("cuda" if torch.cuda.is_available() else "cpu") +# Input panoramic image target size (ERP: W=img_size, H=img_size//2) +IMG_SIZE = args.img_size +# Vertical crop ratio for panoramic image preprocessing +CROP_RATIO = args.crop_ratio + + +def resolve_model_path(model_path: str) -> str: + """ + Resolve model path: if a local file is specified and exists, use it directly; + otherwise download from HuggingFace Hub. + Requires `huggingface-cli login` for gated repos. + """ + if model_path is not None and os.path.isfile(model_path): + return model_path + + if model_path is not None: + print(f"Specified model path '{model_path}' not found.") + + print("Downloading model from HuggingFace (RealseeTechnology/argus-realsee3d)...") + try: + from huggingface_hub import hf_hub_download + downloaded_path = hf_hub_download( + repo_id="RealseeTechnology/argus-realsee3d", + filename="argus_realsee3d.pt", + ) + print(f"Model downloaded to: {downloaded_path}") + return downloaded_path + except Exception as e: + error_msg = str(e) + if "GatedRepoError" in type(e).__name__ or "401" in error_msg: + raise RuntimeError( + "Cannot access gated model repo. Please authenticate first:\n" + " 1. Run: hf auth login\n" + " 2. Accept the model license at: https://huggingface.co/RealseeTechnology/argus-realsee3d\n" + " 3. Re-run this script.\n" + "Or download manually and specify --model_path." + ) from e + raise + + +# Pre-trained model path (auto-download if not found locally) +MODEL_PATH = resolve_model_path(args.model_path) + +# -------------------------- Model Initialization -------------------------- +print("Initializing and loading Argus model...") +# Initialize Argus model with metric scale and learning ref reorder +model = Argus(reorder_by_learning_ref=True, restore_metric_scale=True) +# Load model weights (non-strict to ignore unused parameters) +model.load_state_dict(torch.load(MODEL_PATH, map_location=DEVICE)["model"], strict=False) +# Set model to evaluation mode and move to target device +model.eval() +model = model.to(DEVICE) + + +# -------------------------- Image Preprocessing -------------------------- +def load_and_preprocess_images(image_path_list, target_size=IMG_SIZE): + """ + Load and preprocess panoramic images for model inference + Args: + image_path_list (list): List of input image file paths + target_size (int): Target width of panoramic image (height = target_size//2) + Returns: + torch.Tensor: Preprocessed tensor with shape (S, C, H, W) + S: sequence length, C: 3(RGB), H/W: image size + """ + images = [] + pano_W, pano_H = target_size, target_size // 2 + + # Load and resize each image + for image_path in image_path_list: + img = cv2.imread(image_path) # Load as BGR (H, W, C) + h, w = img.shape[:2] + if w != pano_W or h != pano_H: + img = cv2.resize(img, (pano_W, pano_H), interpolation=cv2.INTER_AREA) + images.append(img) + + # Stack and preprocess: crop vertical → BGR2RGB → normalize → reshape + images = np.stack(images) # (S, H, W, C) + # Crop top/bottom 15% of height and convert BGR to RGB + images = np.ascontiguousarray( + images[:, int(pano_H * CROP_RATIO) : int(pano_H * (1 - CROP_RATIO)), :, ::-1] + ) + # Convert to tensor and normalize to [0,1] + images = torch.from_numpy(images).float() / 255.0 + # Reshape to (S, C, H, W) for PyTorch model input + images = images.permute(0, 3, 1, 2) + + return images + + +# -------------------------- Point Cloud Utils -------------------------- +def save_point_cloud_to_ply(points: np.ndarray, save_path: str): + """ + Save 3D point cloud (N,3) to PLY format (ASCII) for universal compatibility + Args: + points (np.ndarray): 3D point cloud with shape [N, 3] (x, y, z for each point) + save_path (str): Output PLY file path + Raises: + ValueError: If input points shape is not [N, 3] + """ + # Validate input point cloud shape + if points.ndim != 2 or points.shape[1] != 3: + raise ValueError(f"Point cloud must be [N,3], got {points.shape}") + + num_points = points.shape[0] + # PLY format header (follow official specification) + ply_header = f"""ply +format ascii 1.0 +element vertex {num_points} +property float x +property float y +property float z +end_header +""" + # Write header and point data to file + with open(save_path, "w", encoding="utf-8") as f: + f.write(ply_header) + np.savetxt(f, points, fmt="%.6f %.6f %.6f") + + +# -------------------------- Core Model Inference -------------------------- +def run_model(target_dir, model) -> dict: + """ + Run Argus model inference on images in target_dir/images + Args: + target_dir (str): Root directory containing 'images' subfolder + model (Argus): Pre-initialized Argus model + Returns: + dict: Model predictions with tensor converted to numpy array + Raises: + ValueError: If CUDA unavailable or no images found in target_dir + """ + print(f"Processing images from {target_dir}") + + # Enforce CUDA for inference + if not torch.cuda.is_available(): + raise ValueError("CUDA is not available. Inference requires GPU acceleration.") + + model = model.to(DEVICE) + model.eval() + + # Load and sort input images + image_names = sorted(glob.glob(os.path.join(target_dir, "images", "*"))) + print(f"Found {len(image_names)} input images") + if len(image_names) == 0: + raise ValueError("No images found in target_dir/images. Check your upload.") + + # Preprocess images and move to device + images = load_and_preprocess_images(image_names, target_size=IMG_SIZE).to(DEVICE) + print(f"Preprocessed images shape: {images.shape}") + + # Mixed precision inference for speed and memory efficiency + print("Running model inference...") + dtype = ( + torch.bfloat16 if torch.cuda.get_device_capability()[0] >= 8 else torch.float16 + ) + + torch.cuda.synchronize() + t0 = time.perf_counter() + + with torch.no_grad(), torch.amp.autocast("cuda", dtype=dtype): + predictions = model(images) + + torch.cuda.synchronize() + t1 = time.perf_counter() + inference_time = t1 - t0 + print(f"Inference time: {inference_time:.3f} s") + + # Convert pose encoding to extrinsic/intrinsic matrices + print("Converting pose encoding to extrinsic matrices...") + extrinsic, conf = pose_encoding_to_extri360(pose_encoding=predictions["pose_enc"]) + predictions["extrinsic"] = extrinsic[:, :, :3, :] + + # Unproject depth map to 3D world coordinates + print("Computing 3D world points from depth map...") + world_points = unproject_depth_to_world_points( + predictions["depth"], predictions["extrinsic"], size=IMG_SIZE + ) + predictions["world_points_from_depth"] = world_points + + # Convert all torch tensors to numpy arrays and remove batch dimension + print("Converting model outputs to numpy arrays...") + for key in predictions.keys(): + if isinstance(predictions[key], torch.Tensor): + predictions[key] = predictions[key].cpu().float().numpy().squeeze(0) + elif isinstance(predictions[key], list): + for i in range(len(predictions[key])): + if isinstance(predictions[key][i], torch.Tensor): + predictions[key][i] = ( + predictions[key][i].cpu().float().numpy().squeeze(0) + ) + + print(f"Model prediction keys: {predictions.keys()}") + # Clear CUDA cache to save memory + torch.cuda.empty_cache() + return predictions, inference_time + + +# -------------------------- Upload File Handling -------------------------- +def handle_uploads(input_images): + """ + Create directory for uploaded images and copy files to target path. + Uses system temp dir by default; uses --save_tmp dir if specified. + Args: + input_images: Gradio uploaded file data + Returns: + tuple: (target_dir, sorted_image_paths) + """ + start_time = time.time() + gc.collect() + torch.cuda.empty_cache() + + # Create target directory: persistent if --save_tmp is set, otherwise temp + if args.save_tmp: + os.makedirs(args.save_tmp, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f") + target_dir = os.path.join(args.save_tmp, f"input_images_{timestamp}") + else: + target_dir = tempfile.mkdtemp(prefix="argus_") + target_img_dir = os.path.join(target_dir, "images") + + # Clean up if directory exists (edge case) + if os.path.exists(target_dir) and args.save_tmp: + shutil.rmtree(target_dir) + os.makedirs(target_dir, exist_ok=True) + os.makedirs(target_img_dir, exist_ok=True) + + # Copy uploaded images to target directory + image_paths = [] + if input_images is not None: + for file_data in input_images: + # Get file path from Gradio file data + file_path = file_data["name"] if isinstance(file_data, dict) else file_data + dst_path = os.path.join(target_img_dir, os.path.basename(file_path)) + shutil.copy(file_path, dst_path) + image_paths.append(dst_path) + + # Sort images for consistent processing + image_paths = sorted(image_paths) + print( + f"Files copied to {target_img_dir} | Time cost: {time.time() - start_time:.3f}s" + ) + return target_dir, image_paths + + +def update_gallery_on_upload(input_images): + """ + Update image gallery immediately after file upload + Args: + input_images: Gradio uploaded file data + Returns: + tuple: Gradio component update values + """ + if not input_images: + return None, None, None, None + target_dir, image_paths = handle_uploads(input_images) + return ( + None, + target_dir, + image_paths, + "Upload complete. Click 'Reconstruct' to begin 3D processing.", + ) + + +# -------------------------- 3D Reconstruction Pipeline -------------------------- +def gradio_demo( + target_dir, + conf_thres=5.0, + frame_filter="All", + show_cam=True, + show_index=True, + ceiling_remove=25, +): + """ + Main 3D reconstruction pipeline for Gradio interface + Args: + target_dir (str): Directory with input images + conf_thres (float): Confidence threshold for point cloud filtering + frame_filter (str): Filter frames to show in 3D model + show_cam (bool): Whether to show camera poses in 3D model + show_index (bool): Whether to show frame indices in 3D model + ceiling_remove (float): Percentage of top Y-coordinate points to remove as ceiling (0-100, 0=disabled) + Returns: + tuple: Gradio component update values (3D model, logs, dropdown, etc.) + """ + # Validate target directory + if not os.path.isdir(target_dir) or target_dir == "None": + return ( + None, + "No valid target directory. Please upload images first.", + None, + None, + None, + "", + None, + ) + + start_time = time.time() + gc.collect() + torch.cuda.empty_cache() + + # Prepare frame filter dropdown options + target_img_dir = os.path.join(target_dir, "images") + all_files = ( + sorted(os.listdir(target_img_dir)) if os.path.isdir(target_img_dir) else [] + ) + all_files = [f"{i}: {filename}" for i, filename in enumerate(all_files)] + frame_filter_choices = ["All"] + all_files + + # Run model inference + with torch.no_grad(): + predictions, inference_time = run_model(target_dir, model) + + # Save predictions to NPZ for later visualization update + pred_save_path = os.path.join(target_dir, "predictions.npz") + np.savez(pred_save_path, **predictions) + + # Default frame filter to All if None + frame_filter = frame_filter if frame_filter is not None else "All" + + # Generate unique GLB filename with parameters + glb_filename = f"glbscene_{conf_thres}_{frame_filter.replace('.', '_').replace(':', '').replace(' ', '_')}_cam{show_cam}_index{show_index}_ceiling{ceiling_remove}.glb" + glbfile = os.path.join(target_dir, glb_filename) + + # Convert model predictions to GLB 3D model + glbscene = predictions_to_glb( + predictions, + conf_thres=conf_thres, + filter_by_frames=frame_filter, + show_cam=show_cam, + show_index=show_index, + ceiling_remove=ceiling_remove, + target_dir=target_dir, + ) + glbscene.export(file_obj=glbfile) + + # Prepare measure view + measure_img, _ = update_measure_view(predictions, 0) + # Create view selector based on number of input images + num_views = ( + predictions["images"].shape[0] if predictions["images"].shape[0] > 0 else 1 + ) + view_choices = [f"View {i + 1}" for i in range(num_views)] + measure_selector = gr.Dropdown(choices=view_choices, value=view_choices[0]) + + # Clean up memory + gc.collect() + torch.cuda.empty_cache() + + total_time = time.time() - start_time + log_msg = f"Reconstruction Success ({len(all_files)} frames). Inference: {inference_time:.2f}s | Total: {total_time:.2f}s" + print(f"Reconstruction complete | Inference: {inference_time:.2f}s | Total: {total_time:.2f}s") + + return ( + glbfile, + log_msg, + gr.Dropdown(choices=frame_filter_choices, value=frame_filter, interactive=True), + predictions, + measure_img, + "", + measure_selector, + ) + + +# -------------------------- UI Utility Functions -------------------------- +def clear_fields(): + """Clear 3D model viewer for Gradio interface""" + return None + + +def update_log(): + """Update log message during model processing""" + return "Loading and Reconstructing..." + + +def update_visualization( + target_dir, + conf_thres, + frame_filter, + show_cam, + show_index, + ceiling_remove, + is_example, +): + """ + Update 3D visualization when parameters change (without re-running model) + Args: + is_example (str): Whether it's example data (skip if "True") + Returns: + tuple: (GLB file path, log message) + """ + # Skip if loading example data + if is_example == "True": + return ( + None, + "No reconstruction available. Please click the Reconstruct button first.", + ) + # Validate target directory and prediction file + if not target_dir or target_dir == "None" or not os.path.isdir(target_dir): + return None, "No valid reconstruction. Please upload and reconstruct first." + + pred_path = os.path.join(target_dir, "predictions.npz") + if not os.path.exists(pred_path): + return None, f"No prediction file found at {pred_path}. Run Reconstruct first." + + # Load saved predictions + key_list = [ + "pose_enc", + "depth", + "depth_conf", + "images", + "extrinsic", + "world_points_from_depth", + ] + loaded = np.load(pred_path) + predictions = {key: np.array(loaded[key]) for key in key_list if key in loaded} + + # Generate GLB file (create if not exists) + glb_filename = f"glbscene_{conf_thres}_{frame_filter.replace('.', '_').replace(':', '').replace(' ', '_')}_cam{show_cam}_index{show_index}_ceiling{ceiling_remove}.glb" + glbfile = os.path.join(target_dir, glb_filename) + + if not os.path.exists(glbfile): + glbscene = predictions_to_glb( + predictions, + conf_thres=conf_thres, + filter_by_frames=frame_filter, + show_cam=show_cam, + show_index=show_index, + ceiling_remove=ceiling_remove, + target_dir=target_dir, + ) + glbscene.export(file_obj=glbfile) + + return glbfile, "Visualization updated successfully" + + +# -------------------------- Metric Measurement -------------------------- +def update_measure_view(predictions, view_index): + """ + Update measure view with depth confidence mask overlay + Args: + predictions (dict): Model predictions with images and depth confidence + view_index (int): Index of the view to show + Returns: + tuple: (processed_image, empty_list) + """ + # Get image and depth confidence + image = predictions["images"][view_index].transpose(1, 2, 0).copy() + depth_conf = predictions["depth_conf"][view_index].copy() + + # Convert image to uint8 format + if image.dtype != np.uint8: + image = ( + (image * 255).astype(np.uint8) + if image.max() <= 1.0 + else image.astype(np.uint8) + ) + + # Create depth confidence mask (filter low confidence areas) + depth_conf_norm = (depth_conf - depth_conf.min()) / ( + depth_conf.max() - depth_conf.min() + ) + mask = depth_conf_norm > 0.05 + invalid_mask = ~mask + + # Apply red overlay to invalid areas (low confidence) + if invalid_mask.any(): + overlay_color = np.array([255, 220, 220], dtype=np.uint8) + alpha = 0.5 # Transparency + for c in range(3): + image[:, :, c] = np.where( + invalid_mask, + (1 - alpha) * image[:, :, c] + alpha * overlay_color[c], + image[:, :, c], + ).astype(np.uint8) + + return image, [] + + +def navigate_measure_view(processed_data, current_selector_value, direction): + """ + Navigate between different measure views (previous/next) + Args: + direction (int): -1 for previous, +1 for next + Returns: + tuple: (new_selector_value, measure_image, empty_points) + """ + if processed_data["images"].shape[0] == 0: + return "View 1", None, [] + + # Parse current view index from selector + try: + current_view = int(current_selector_value.split()[1]) - 1 + except: + current_view = 0 + + # Calculate new view index (circular navigation) + num_views = processed_data["images"].shape[0] + new_view = (current_view + direction) % num_views + + # Update selector and image + new_selector = f"View {new_view + 1}" + measure_image, _ = update_measure_view(processed_data, new_view) + return new_selector, measure_image, [] + + +def measure( + processed_data, measure_points, current_view_selector, event: gr.SelectData +): + """ + Core metric measurement function: click to select points and calculate 3D distance + Args: + event (gr.SelectData): Gradio click event data (image coordinates) + Returns: + tuple: (annotated_image, measure_points, measurement_text) + """ + try: + # Get current view index + try: + current_view = int(current_view_selector.split()[1]) - 1 + except: + current_view = 0 + # Validate view index + current_view = ( + 0 + if current_view < 0 or current_view >= processed_data["images"].shape[0] + else current_view + ) + + # Get clicked 2D point + point2d = event.index[0], event.index[1] + measure_points.append(point2d) + print(f"Measuring: clicked point {point2d} (view {current_view + 1})") + + # Get base image and 3D points + image, _ = update_measure_view(processed_data, current_view) + image = image.copy() + points3d = processed_data["world_points_from_depth"][current_view] + + # Draw blue circles for clicked points + for p in measure_points: + if 0 <= p[0] < image.shape[1] and 0 <= p[1] < image.shape[0]: + image = cv2.circle(image, p, radius=5, color=(255, 0, 0), thickness=2) + + # Calculate depth for single point + depth_text = "" + depth = processed_data["depth"][current_view].squeeze(axis=-1) + for i, p in enumerate(measure_points): + try: + if 0 <= p[1] < depth.shape[0] and 0 <= p[0] < depth.shape[1]: + d = depth[p[1], p[0]] + depth_text += f"- **P{i + 1} depth: {d:.2f}m.**\n" + else: + d = np.linalg.norm(points3d[p[1], p[0]], ord=2) + depth_text += f"- **P{i + 1} dist: {d:.2f}m.**\n" + except: + depth_text += f"- **P{i + 1}: Depth unavailable**\n" + + # Calculate 3D distance for two points + if len(measure_points) == 2: + p1, p2 = measure_points + # Draw blue line between two points + if all( + 0 <= p[0] < image.shape[1] and 0 <= p[1] < image.shape[0] + for p in [p1, p2] + ): + image = cv2.line(image, p1, p2, color=(255, 0, 0), thickness=2) + # Calculate 3D Euclidean distance + try: + p1_3d = points3d[p1[1], p1[0]] + p2_3d = points3d[p2[1], p2[0]] + distance = np.linalg.norm(p1_3d - p2_3d) + distance_text = f"- **Distance: {distance:.2f}m**" + except: + distance_text = "- **Distance: Unable to compute**" + # Reset points after measurement + measure_points = [] + return [image, measure_points, depth_text + distance_text] + + return [image, measure_points, depth_text] + except Exception as e: + print(f"Measurement error: {str(e)}") + return None, [], f"Measure error: {str(e)}" + + +# -------------------------- Example Data Loader -------------------------- +def get_scene_info(examples_dir): + """ + Load example scene information from examples directory + Args: + examples_dir (str): Directory containing example scenes + Returns: + list: List of scene dicts with name, path, thumbnail, image files + """ + scenes = [] + if not os.path.exists(examples_dir): + return scenes + + # Iterate over example scene folders + for scene_folder in sorted(os.listdir(examples_dir)): + scene_path = os.path.join(examples_dir, scene_folder) + if not os.path.isdir(scene_path): + continue + # Load all image files + img_exts = ["*.jpg", "*.jpeg", "*.png", "*.bmp", "*.tiff", "*.tif"] + image_files = [] + for ext in img_exts: + image_files.extend(glob.glob(os.path.join(scene_path, ext))) + image_files.extend(glob.glob(os.path.join(scene_path, ext.upper()))) + # Skip empty folders + if not image_files: + continue + # Sort images and get thumbnail + image_files = sorted(image_files) + scenes.append( + { + "name": scene_folder, + "path": scene_path, + "thumbnail": image_files[0], + "num_images": len(image_files), + "image_files": image_files, + } + ) + return scenes + + +def example_pipeline( + scene, + conf_thres=5.0, + show_cam=True, + show_index=True, + ceiling_remove=25, +): + """ + Pipeline for loading example scenes and running reconstruction + Args: + scene (dict): Example scene info from get_scene_info + Returns: + tuple: Gradio component update values + """ + input_image_paths = scene["image_files"] + target_dir, image_paths = handle_uploads(input_image_paths) + frame_filter = "All" # Default to all frames for examples + # Run reconstruction + ( + glbfile, + log_msg, + dropdown, + predictions, + measure_img, + measure_text, + measure_selector, + ) = gradio_demo( + target_dir, conf_thres, frame_filter, show_cam, show_index, ceiling_remove + ) + return ( + glbfile, + log_msg, + target_dir, + dropdown, + image_paths, + predictions, + measure_img, + measure_text, + measure_selector, + ) + + +# -------------------------- 3D Visualization Utilities -------------------------- +class SevenSegmentDigit: + """7-segment display definition for digital watch style 3D point cloud generation""" + # 7 segments definition: A(top), B(upper right), C(lower right), D(bottom), E(lower left), F(upper left), G(middle) + SEGMENTS = { + 'A': np.array([(x, 0.5, 0) for x in np.linspace(-0.4, 0.4, 80) for y in np.linspace(0.45, 0.55, 10)]), + 'B': np.array([(x, y, 0) for x in np.linspace(0.4, 0.5, 10) for y in np.linspace(0, 0.5, 80)]), + 'C': np.array([(x, y, 0) for x in np.linspace(0.4, 0.5, 10) for y in np.linspace(-0.5, 0, 80)]), + 'D': np.array([(x, y, 0) for x in np.linspace(-0.4, 0.4, 80) for y in np.linspace(-0.55, -0.45, 10)]), + 'E': np.array([(x, y, 0) for x in np.linspace(-0.5, -0.4, 10) for y in np.linspace(-0.5, 0, 80)]), + 'F': np.array([(x, y, 0) for x in np.linspace(-0.5, -0.4, 10) for y in np.linspace(0, 0.5, 80)]), + 'G': np.array([(x, y, 0) for x in np.linspace(-0.4, 0.4, 80) for y in np.linspace(-0.05, 0.05, 10)]) + } + + # Segment mapping for standard 0-9 digits (specify lit segments for each digit) + DIGIT_SEGMENTS = { + 0: ['A', 'B', 'C', 'D', 'E', 'F'], + 1: ['B', 'C'], + 2: ['A', 'B', 'G', 'E', 'D'], + 3: ['A', 'B', 'G', 'C', 'D'], + 4: ['F', 'G', 'B', 'C'], + 5: ['A', 'F', 'G', 'C', 'D'], + 6: ['A', 'F', 'G', 'C', 'D', 'E'], + 7: ['A', 'B', 'C'], + 8: ['A', 'B', 'C', 'D', 'E', 'F', 'G'], + 9: ['A', 'B', 'C', 'D', 'F', 'G'] + } + + @classmethod + def get_digit_points(cls, digit, scale=0.05): + """ + Generate 3D point cloud for a single digital watch style digit (0-9) + Args: + digit (int): Target digit (0-9 only) + scale (float): Scale factor for point cloud size + Returns: + np.ndarray: N×3 array of 3D points for the digit + Raises: + ValueError: If digit is not in 0-9 range + """ + if not 0 <= digit <= 9: + raise ValueError(f"Digit must be 0-9, got {digit}") + + # Combine lit segments for the target digit + segments = cls.DIGIT_SEGMENTS[digit] + points = np.vstack([cls.SEGMENTS[seg] for seg in segments]) + + # Scale point cloud and center to origin + points = points * scale + points -= points.mean(axis=0) + + # Remove duplicate points and supplement sparse points (ensure dense distribution) + points = np.unique(points.round(6), axis=0) + if len(points) < 200: + points = trimesh.sample.sample_surface(trimesh.Trimesh(points), 500)[0] + + return points + + +def create_number_point_cloud(number, scale=0.05): + """ + Generate 3D point cloud for multi-digit number (digital watch style), facing +Y axis + Args: + number (int): Non-negative target integer (any digit length) + scale (float): Scale factor for single digit point cloud size + Returns: + trimesh.PointCloud: Colored (red) 3D point cloud of the number + Raises: + ValueError: If number is negative or non-integer + """ + if not isinstance(number, int) or number < 0: + raise ValueError(f"Number must be non-negative integer, got {number}") + + # Split number into individual digits and handle 0 specially + digits = [int(d) for d in str(number)] if number != 0 else [0] + all_points, spacing = [], scale * 1.2 + total_width = (len(digits)-1) * spacing + + # Arrange digits horizontally and center the whole number + for idx, d in enumerate(digits): + digit_points = SevenSegmentDigit.get_digit_points(d, scale) + digit_points[:, 0] += -total_width/2 + idx * spacing + all_points.append(digit_points) + + # Merge all digit points and apply rotation to face +Y axis + all_points = np.vstack(all_points) + rotation = np.array([[1, 0, 0], + [0, 0, -1], + [0, 1, 0]]) + all_points = np.dot(all_points, rotation.T) + + # Create red point cloud (classic digital watch color) + colors = np.full((len(all_points), 3), [255, 0, 0], dtype=np.uint8) + + return trimesh.PointCloud(all_points, colors) + + +def predictions_to_glb( + predictions, + conf_thres=50.0, + filter_by_frames="all", + show_cam=True, + show_index=True, + ceiling_remove=25, + target_dir=None, + prediction_mode="Predicted Pointmap", +) -> trimesh.Scene: + """ + Convert VGGT model predictions to a 3D trimesh Scene (exportable to GLB) + Integrates colored point cloud, camera meshes and digital camera indexes + Args: + predictions (dict): Model prediction dict with keys: + - world_points: 3D point coordinates (S, H, W, 3) + - world_points_conf: Confidence scores (S, H, W) + - images: Input images (S, H, W, 3) + - extrinsic: Camera extrinsic matrices (S, 3, 4) + conf_thres (float): Low-confidence point filter (percentile, 0-100) + filter_by_frames (str): Frame filter ("all" or specific frame index like "0:") + show_cam (bool): Whether to add camera mesh visualization to scene + show_index (bool): Whether to add digital index point cloud above cameras + ceiling_remove (float): Percentage of top Y-coordinate points to remove as ceiling (0-100, 0=disabled) + target_dir (str): Directory for intermediate files (images) + prediction_mode (str): Prediction branch ("Predicted Pointmap" / others for depth-based) + Returns: + trimesh.Scene: 3D scene with point cloud, cameras and indexes (if enabled) + Raises: + ValueError: If predictions is not a dictionary + """ + if not isinstance(predictions, dict): + raise ValueError("predictions must be a dictionary") + + conf_thres = 10.0 if conf_thres is None else conf_thres + print("Building GLB scene") + selected_frame_idx = None + + # Parse selected frame index from filter string (e.g., "0:" -> 0) + if filter_by_frames not in ["all", "All"]: + try: + selected_frame_idx = int(filter_by_frames.split(":")[0]) + except (ValueError, IndexError): + pass + + # Select prediction branch (Pointmap direct / Depthmap derived) + if "Pointmap" in prediction_mode: + print("Using Pointmap Branch") + if "world_points" in predictions: + pred_world_points = predictions["world_points"] + pred_world_points_conf = predictions.get("world_points_conf", np.ones_like(pred_world_points[..., 0])) + else: + print("Warning: world_points not found, falling back to depth-based world points") + pred_world_points = predictions["world_points_from_depth"] + pred_world_points_conf = predictions.get("depth_conf", np.ones_like(pred_world_points[..., 0])) + else: + print("Using Depthmap and Camera Branch") + pred_world_points = predictions["world_points_from_depth"] + pred_world_points_conf = predictions.get("depth_conf", np.ones_like(pred_world_points[..., 0])) + + # Extract core prediction data: images and camera extrinsic matrices + images = predictions["images"] + camera_matrices = predictions["extrinsic"] + + # Filter prediction data to selected single frame if specified + if selected_frame_idx is not None: + pred_world_points = pred_world_points[selected_frame_idx][None] + pred_world_points_conf = pred_world_points_conf[selected_frame_idx][None] + images = images[selected_frame_idx][None] + camera_matrices = camera_matrices[selected_frame_idx][None] + + # Reshape 3D points and convert image colors to 8-bit RGB (match point cloud) + vertices_3d = pred_world_points.reshape(-1, 3) + if images.ndim == 4 and images.shape[1] == 3: # Convert NCHW to NHWC format + colors_rgb = np.transpose(images, (0, 2, 3, 1)) + else: # Direct use if already NHWC format + colors_rgb = images + colors_rgb = (colors_rgb.reshape(-1, 3) * 255).astype(np.uint8) + + # Filter points by confidence threshold (remove low-confidence points) + conf = pred_world_points_conf.reshape(-1) + conf_threshold = 0.0 if conf_thres == 0.0 else np.percentile(conf, conf_thres) + conf_mask = (conf >= conf_threshold) & (conf > 1e-5) + + vertices_3d = vertices_3d[conf_mask] + colors_rgb = colors_rgb[conf_mask] + + # Create dummy point if no valid points left (avoid scene empty error) + if vertices_3d is None or np.asarray(vertices_3d).size == 0: + vertices_3d = np.array([[1, 0, 0]]) + colors_rgb = np.array([[255, 255, 255]]) + scene_scale = 1 + else: + # Calculate scene scale by 5th/95th percentile bounding box diagonal + lower_percentile = np.percentile(vertices_3d, 5, axis=0) + upper_percentile = np.percentile(vertices_3d, 95, axis=0) + scene_scale = np.linalg.norm(upper_percentile - lower_percentile) + + # Initialize 3D scene and colormap for camera unique colors + colormap = matplotlib.colormaps.get_cmap("gist_rainbow") + scene_3d = trimesh.Scene() + + # Filter out ceiling points (remove top N% of Y-coordinates by percentile) + if ceiling_remove > 0 and vertices_3d.size > 1: + y_coords = vertices_3d[:, 1] + y_percentile = np.percentile(y_coords, ceiling_remove) + mask = y_coords > y_percentile + vertices_3d = vertices_3d[mask] + colors_rgb = colors_rgb[mask] + + # Add colored 3D point cloud to the scene + point_cloud_data = trimesh.PointCloud(vertices=vertices_3d, colors=colors_rgb) + scene_3d.add_geometry(point_cloud_data) + + # Convert 3x4 camera extrinsics to 4x4 homogeneous matrices + num_cameras = len(camera_matrices) + extrinsics_matrices = np.zeros((num_cameras, 4, 4)) + extrinsics_matrices[:, :3, :4] = camera_matrices + extrinsics_matrices[:, 3, 3] = 1 + + # Add camera meshes and digital index point clouds to the scene + for i in range(num_cameras): + camera_to_world = extrinsics_matrices[i] + rgba_color = colormap(i / num_cameras) # Unique color for each camera + current_color = tuple(int(255 * x) for x in rgba_color[:3]) + + # Add camera mesh to scene + if show_cam: + integrate_camera_into_scene(scene_3d, camera_to_world, current_color, scene_scale) + + # Add digital index point cloud above each camera (red, digital watch style) + if show_index: + camera_center = camera_to_world[:3, 3] + y_offset = 0.5 # Y-axis offset for index position (above camera) + number_position = camera_center + np.array([0, y_offset, 0]) + + # Generate index point cloud and translate to target position + number_scale = 0.3 + number_pc = create_number_point_cloud(number=i, scale=number_scale) + number_pc.apply_translation(number_position) + scene_3d.add_geometry(number_pc) + + # Align the whole scene to the first camera's viewing perspective + scene_3d = apply_scene_alignment(scene_3d, extrinsics_matrices) + + print("GLB Scene built successfully") + return scene_3d + + +def integrate_camera_into_scene( + scene: trimesh.Scene, transform: np.ndarray, face_colors: tuple, scene_scale: float +): + """ + Add a 3D cone-shaped camera mesh to the 3D scene with specified transform and color + Args: + scene (trimesh.Scene): Target 3D scene to add camera mesh + transform (np.ndarray): 4x4 camera-to-world transformation matrix + face_colors (tuple): RGB color tuple (0-255) for camera mesh faces + scene_scale (float): Overall scale of the 3D scene (for camera size adaptation) + """ + # Set camera mesh size based on scene scale + cam_width = scene_scale * 0.02 + cam_height = scene_scale * 0.02 + + # 45° Z-axis rotation for camera cone shape and backward translation + rot_45_degree = np.eye(4) + rot_45_degree[:3, :3] = Rotation.from_euler("z", 45, degrees=True).as_matrix() + rot_45_degree[2, 3] = -cam_height + + # Combine OpenGL conversion, rotation and camera transform matrices + opengl_transform = get_opengl_conversion_matrix() + complete_transform = transform @ opengl_transform @ rot_45_degree + camera_cone_shape = trimesh.creation.cone(cam_width, cam_height, sections=4) + + # Slight Z-axis rotation for camera mesh detail enhancement + slight_rotation = np.eye(4) + slight_rotation[:3, :3] = Rotation.from_euler("z", 2, degrees=True).as_matrix() + + # Combine original, scaled and rotated cone vertices for dense camera mesh + vertices_combined = np.concatenate( + [ + camera_cone_shape.vertices, + 0.95 * camera_cone_shape.vertices, + transform_points(slight_rotation, camera_cone_shape.vertices), + ] + ) + vertices_transformed = transform_points(complete_transform, vertices_combined) + + # Compute camera mesh faces from cone shape + mesh_faces = compute_camera_faces(camera_cone_shape) + + # Create camera mesh with specified color and add to scene + camera_mesh = trimesh.Trimesh(vertices=vertices_transformed, faces=mesh_faces) + camera_mesh.visual.face_colors[:, :3] = face_colors + scene.add_geometry(camera_mesh) + + +def apply_scene_alignment( + scene_3d: trimesh.Scene, extrinsics_matrices: np.ndarray +) -> trimesh.Scene: + """ + Align the 3D scene to the first camera's viewing perspective with OpenGL conversion + Args: + scene_3d (trimesh.Scene): Unaligned 3D scene + extrinsics_matrices (np.ndarray): N×4×4 camera extrinsic matrices + Returns: + trimesh.Scene: Aligned 3D scene + """ + # Get OpenGL coordinate conversion matrix and 180° Y-axis rotation for alignment + opengl_conversion_matrix = get_opengl_conversion_matrix() + align_rotation = np.eye(4) + align_rotation[:3, :3] = Rotation.from_euler("y", 180, degrees=True).as_matrix() + + # Combine transformation matrices and apply to the whole scene + initial_transformation = np.linalg.inv(extrinsics_matrices[0]) @ opengl_conversion_matrix @ align_rotation + scene_3d.apply_transform(initial_transformation) + return scene_3d + + +def get_opengl_conversion_matrix() -> np.ndarray: + """ + Create 4x4 OpenGL coordinate system conversion matrix (flip Y and Z axes) + Returns: + np.ndarray: 4x4 identity-based conversion matrix + """ + matrix = np.identity(4) + matrix[1, 1] = -1 # Flip Y axis + matrix[2, 2] = -1 # Flip Z axis + return matrix + + +def transform_points( + transformation: np.ndarray, points: np.ndarray, dim: int = None +) -> np.ndarray: + """ + Apply 4x4 homogeneous transformation matrix to a set of 3D points + Args: + transformation (np.ndarray): 4x4 transformation matrix + points (np.ndarray): N×3 array of 3D points to transform + dim (int, optional): Target dimension of output points (default: 3) + Returns: + np.ndarray: N×dim array of transformed points (same shape as input except last dim) + """ + points = np.asarray(points) + initial_shape = points.shape[:-1] + dim = dim or points.shape[-1] + + # Transpose matrix and apply affine transformation to points + transformation = transformation.swapaxes(-1, -2) + points = points @ transformation[..., :-1, :] + transformation[..., -1:, :] + + # Reshape transformed points to original shape (excluding last dimension) + result = points[..., :dim].reshape(*initial_shape, dim) + return result + + +def compute_camera_faces(cone_shape: trimesh.Trimesh) -> np.ndarray: + """ + Compute face indices for camera mesh from original cone shape faces (enhance detail) + Args: + cone_shape (trimesh.Trimesh): Original cone mesh for camera base shape + Returns: + np.ndarray: M×3 array of face indices for the camera mesh + """ + faces_list = [] + num_vertices_cone = len(cone_shape.vertices) + + # Generate enhanced faces from cone faces (skip origin vertex 0) + for face in cone_shape.faces: + if 0 in face: + continue + v1, v2, v3 = face + v1_offset, v2_offset, v3_offset = face + num_vertices_cone + v1_offset_2, v2_offset_2, v3_offset_2 = face + 2 * num_vertices_cone + + # Add multiple face variations for dense camera mesh + faces_list.extend( + [ + (v1, v2, v2_offset), + (v1, v1_offset, v3), + (v3_offset, v2, v3), + (v1, v2, v2_offset_2), + (v1, v1_offset_2, v3), + (v3_offset_2, v2, v3), + ] + ) + + # Add reversed faces for double-sided rendering + faces_list += [(v3, v2, v1) for v1, v2, v3 in faces_list] + return np.array(faces_list) + + +# -------------------------- Gradio UI Construction -------------------------- +if __name__ == "__main__": + # Gradio theme configuration + theme = gr.themes.Ocean() + theme.set( + checkbox_label_background_fill_selected="*button_primary_background_fill", + checkbox_label_text_color_selected="*button_primary_text_color", + ) + + with gr.Blocks( + theme=theme, + title="Argus - 3D Reconstruction", + css=""" + .custom-log * { + font-style: italic; + font-size: 20px !important; + background-image: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + -webkit-background-clip: text; + background-clip: text; + font-weight: 600 !important; + color: transparent !important; + text-align: center !important; + } + .example-log * { + font-size: 15px !important; + background-image: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + -webkit-background-clip: text; + background-clip: text; + color: transparent !important; + font-weight: 500 !important; + } + .header-banner { + background: linear-gradient(135deg, #f8fafc 0%, #e2e8f0 100%); + border-radius: 16px; + padding: 32px 24px 24px; + margin-bottom: 16px; + border: 1px solid #e2e8f0; + text-align: center; + } + .header-banner h1 { + font-size: 28px; + font-weight: 700; + color: #1e293b; + margin: 12px 0 8px; + } + .header-banner .links { + margin-top: 12px; + font-size: 15px; + } + .header-banner .links a { + margin: 0 10px; + color: #4f46e5; + text-decoration: none; + font-weight: 500; + } + .header-banner .links a:hover { + text-decoration: underline; + } + .instructions { + font-size: 14px; + color: #475569; + line-height: 1.7; + padding: 12px 20px; + background: #f8fafc; + border-radius: 10px; + border: 1px solid #e2e8f0; + } + .instructions ol { + padding-left: 20px; + margin: 8px 0; + } + .instructions li { + margin-bottom: 4px; + } + .param-group { + padding: 8px 0; + } + footer {visibility: hidden;} + """, + ) as demo: + # Hidden state components for data passing + is_example = gr.Textbox(label="is_example", visible=False, value="None") + processed_data_state = gr.State(value=None) + measure_points_state = gr.State(value=[]) + target_dir_output = gr.Textbox(label="Target Dir", visible=False, value="None") + + # Load and display logo (base64 encoded) + root_dir = Path(__file__).parent + logo_path = root_dir / "assets" / "argus_logo.png" + if logo_path.exists(): + with open(logo_path, "rb") as f: + logo_base64 = base64.b64encode(f.read()).decode() + logo_src = f"data:image/png;base64,{logo_base64}" + else: + logo_src = "" # Fallback if logo not found + + # UI Header and Instructions + gr.HTML( + f""" +
+
+ Argus Logo +
+

Argus: Metric Panoramic 3D Reconstruction for Indoor Scenes

+ +
+
+
    +
  1. Upload a set of ERP panoramic images on the left.
  2. +
  3. Click "Reconstruct" to run the 3D reconstruction pipeline.
  4. +
  5. Explore the 3D model — rotate, pan, zoom, and download the GLB.
  6. +
  7. Measure — switch to the Metric tab and click two points to measure real-world distance.
  8. +
+
+ """ + ) + + # Main UI Layout (2 columns: upload/gallery | 3D model/measurement) + with gr.Row(equal_height=False): + with gr.Column(scale=2, min_width=280): + input_images = gr.File( + file_count="multiple", label="📁 Upload Panoramic Images", interactive=True + ) + image_gallery = gr.Gallery( + label="Preview", + columns=3, + height="280px", + object_fit="contain", + preview=True, + ) + + with gr.Column(scale=5): + # Log output + log_output = gr.Markdown( + "Upload panoramic images (ERP), then click Reconstruct.", + elem_classes=["custom-log"], + ) + # Tabbed interface: 3D Model + Metric Measure + with gr.Tabs(): + with gr.Tab("🏠 3D Model"): + reconstruction_output = gr.Model3D( + height=540, zoom_speed=0.5, pan_speed=0.5 + ) + with gr.Tab("📏 Metric Measure"): + gr.Markdown( + "Click two points on the panorama to measure the real-world distance between them." + ) + with gr.Row(): + prev_measure_btn = gr.Button( + "◀ Prev", size="sm", scale=1 + ) + measure_view_selector = gr.Dropdown( + choices=["View 1"], + value="View 1", + label="Select View", + scale=3, + interactive=True, + allow_custom_value=True, + ) + next_measure_btn = gr.Button("Next ▶", size="sm", scale=1) + measure_image = gr.Image( + type="numpy", + show_label=False, + format="webp", + interactive=False, + sources=[], + ) + measure_text = gr.Markdown("") + + # Action buttons + with gr.Row(): + submit_btn = gr.Button("🔨 Reconstruct", scale=2, variant="primary") + clear_btn = gr.ClearButton( + [ + input_images, + reconstruction_output, + log_output, + target_dir_output, + image_gallery, + ], + value="🗑️ Clear", + scale=1, + ) + + # Reconstruction parameters + gr.Markdown("**Visualization Settings**") + with gr.Row(): + conf_thres = gr.Slider( + 0, 100, 5, 1, label="Confidence Threshold (%)" + ) + ceiling_remove = gr.Slider( + 0, 100, 25, 1, label="Ceiling Remove (%)" + ) + with gr.Row(): + frame_filter = gr.Dropdown( + ["All"], "All", label="Show Points from Frame", scale=2 + ) + show_cam = gr.Checkbox(True, label="Show Camera") + show_index = gr.Checkbox(True, label="Show Index") + + # Example Scenes Section + gr.Markdown("---") + gr.Markdown("### 🖼️ Example Scenes") + gr.Markdown("Click any thumbnail to load and reconstruct.", elem_classes=["example-log"]) + example_scenes = get_scene_info(args.examples_dir) + # Create 4-column example thumbnail grid + if example_scenes: + for i in range(0, len(example_scenes), 4): + with gr.Row(): + for j in range(4): + idx = i + j + if idx < len(example_scenes): + scene = example_scenes[idx] + with gr.Column(scale=1): + scene_state = gr.State(value=scene) + scene_img = gr.Image( + value=scene["thumbnail"], + height=150, + interactive=False, + show_label=False, + sources=[], + ) + gr.Markdown( + f"**{scene['name']}** \n {scene['num_images']} images" + ) + # Bind thumbnail click to example pipeline + scene_img.select( + example_pipeline, + [scene_state], + [ + reconstruction_output, + log_output, + target_dir_output, + frame_filter, + image_gallery, + processed_data_state, + measure_image, + measure_text, + measure_view_selector, + ], + ) + else: + with gr.Column(scale=1): + pass # Empty column for grid alignment + + # -------------------------- Gradio Event Bindings -------------------------- + # Reconstruct button logic + submit_btn.click(clear_fields, [], [reconstruction_output]).then( + update_log, [], [log_output] + ).then( + gradio_demo, + [ + target_dir_output, + conf_thres, + frame_filter, + show_cam, + show_index, + ceiling_remove, + ], + [ + reconstruction_output, + log_output, + frame_filter, + processed_data_state, + measure_image, + measure_text, + measure_view_selector, + ], + ).then( + lambda: "False", [], [is_example] + ) + + # Real-time parameter update for 3D visualization + for param in [conf_thres, frame_filter, show_cam, show_index, ceiling_remove]: + param.change( + update_visualization, + [ + target_dir_output, + conf_thres, + frame_filter, + show_cam, + show_index, + ceiling_remove, + is_example, + ], + [reconstruction_output, log_output], + ) + + # Auto-update gallery on file upload + input_images.change( + update_gallery_on_upload, + [input_images], + [reconstruction_output, target_dir_output, image_gallery, log_output], + ) + + # Metric measure event bindings + measure_image.select( + measure, + [processed_data_state, measure_points_state, measure_view_selector], + [measure_image, measure_points_state, measure_text], + ) + # Measure view navigation + prev_measure_btn.click( + lambda d, s: navigate_measure_view(d, s, -1), + [processed_data_state, measure_view_selector], + [measure_view_selector, measure_image, measure_points_state], + ) + next_measure_btn.click( + lambda d, s: navigate_measure_view(d, s, 1), + [processed_data_state, measure_view_selector], + [measure_view_selector, measure_image, measure_points_state], + ) + # Update measure view when selector changes + measure_view_selector.change( + lambda d, s: ( + update_measure_view(d, int(s.split()[1]) - 1) if s else (None, []) + ), + [processed_data_state, measure_view_selector], + [measure_image, measure_points_state], + ) + + # Footer acknowledgement + gr.HTML( + """ +
+
+

Acknowledgements

+

Built upon + VGGT & + Map-Anything +

+
+ """ + ) + + # Launch Gradio demo + demo.queue(max_size=20).launch( + show_error=True, + share=args.share, + server_name=args.server_name, + server_port=args.port, + ) diff --git a/argus/__init__.py b/argus/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7b3cdba8f0417592fec0c4b198244c248ea0130f --- /dev/null +++ b/argus/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2026 Realsee. All rights reserved. +# Licensed under the Apache License, Version 2.0. diff --git a/argus/heads/__init__.py b/argus/heads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7b3cdba8f0417592fec0c4b198244c248ea0130f --- /dev/null +++ b/argus/heads/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2026 Realsee. All rights reserved. +# Licensed under the Apache License, Version 2.0. diff --git a/argus/heads/camera_head.py b/argus/heads/camera_head.py new file mode 100644 index 0000000000000000000000000000000000000000..bf139d78fdaab8ae724b0db76f55f37f78b4a7b6 --- /dev/null +++ b/argus/heads/camera_head.py @@ -0,0 +1,142 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from argus.layers import Mlp +from argus.layers.block import Block +from argus.heads.head_act import activate_pose + + +class CameraHead(nn.Module): + """ + CameraHead predicts camera parameters from token representations using iterative refinement. + + It applies a series of transformer blocks (the "trunk") to dedicated camera tokens. + """ + + def __init__( + self, + dim_in: int = 2048, + trunk_depth: int = 4, + num_heads: int = 16, + mlp_ratio: int = 4, + init_values: float = 0.01, + trans_act: str = "linear", + quat_act: str = "linear", + ): + super().__init__() + + self.target_dim = 9 + self.trans_act = trans_act + self.quat_act = quat_act + self.trunk_depth = trunk_depth + + # Build the trunk using a sequence of transformer blocks. + self.trunk = nn.Sequential( + *[ + Block(dim=dim_in, num_heads=num_heads, mlp_ratio=mlp_ratio, init_values=init_values) + for _ in range(trunk_depth) + ] + ) + + # Normalizations for camera token and trunk output. + self.token_norm = nn.LayerNorm(dim_in) + self.trunk_norm = nn.LayerNorm(dim_in) + + # Learnable empty camera pose token. + self.empty_pose_tokens = nn.Parameter(torch.zeros(1, 1, self.target_dim)) + self.embed_pose = nn.Linear(self.target_dim, dim_in) + + # Module for producing modulation parameters: shift, scale, and a gate. + self.poseLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(dim_in, 3 * dim_in, bias=True)) + + # Adaptive layer normalization without affine parameters. + self.adaln_norm = nn.LayerNorm(dim_in, elementwise_affine=False, eps=1e-6) + self.pose_branch = Mlp(in_features=dim_in, hidden_features=dim_in // 2, out_features=self.target_dim, drop=0) + + # conf branch for T and R + self.conf_branch = Mlp(in_features=dim_in, hidden_features=dim_in // 2, out_features=2, drop=0) + + def forward(self, aggregated_tokens_list: list, num_iterations: int = 4) -> list: + """ + Forward pass to predict camera parameters. + + Args: + aggregated_tokens_list (list): List of token tensors from the network; + the last tensor is used for prediction. + num_iterations (int, optional): Number of iterative refinement steps. Defaults to 4. + + Returns: + list: A list of predicted camera encodings (post-activation) from each iteration. + """ + # Use tokens from the last block for camera prediction. + tokens = aggregated_tokens_list[-1] + + # Extract the camera tokens + pose_tokens = tokens[:, :, 0] + pose_tokens = self.token_norm(pose_tokens) + + pred_pose_enc_list = self.trunk_fn(pose_tokens, num_iterations) + return pred_pose_enc_list + + def trunk_fn(self, pose_tokens: torch.Tensor, num_iterations: int) -> list: + """ + Iteratively refine camera pose predictions. + + Args: + pose_tokens (torch.Tensor): Normalized camera tokens with shape [B, S, C]. + num_iterations (int): Number of refinement iterations. + + Returns: + list: List of activated camera encodings from each iteration. + """ + B, S, C = pose_tokens.shape + pred_pose_enc = None + pred_pose_enc_conf = None + pred_pose_enc_list = [] + + for _ in range(num_iterations): + # Use a learned empty pose for the first iteration. + if pred_pose_enc is None: + module_input = self.embed_pose(self.empty_pose_tokens.expand(B, S, -1)) + else: + # Detach the previous prediction to avoid backprop through time. + pred_pose_enc = pred_pose_enc.detach() + module_input = self.embed_pose(pred_pose_enc) + + # Generate modulation parameters and split them into shift, scale, and gate components. + shift_msa, scale_msa, gate_msa = self.poseLN_modulation(module_input).chunk(3, dim=-1) + + # Adaptive layer normalization and modulation. + pose_tokens_modulated = gate_msa * modulate(self.adaln_norm(pose_tokens), shift_msa, scale_msa) + pose_tokens_modulated = pose_tokens_modulated + pose_tokens + + pose_tokens_modulated = self.trunk(pose_tokens_modulated) + # Compute the delta update for the pose encoding. + pred_pose_enc_delta = self.pose_branch(self.trunk_norm(pose_tokens_modulated)) + pred_pose_enc_conf_delta = self.conf_branch(self.trunk_norm(pose_tokens_modulated)) + + if pred_pose_enc is None: + pred_pose_enc = pred_pose_enc_delta + pred_pose_enc_conf = pred_pose_enc_conf_delta + else: + pred_pose_enc = pred_pose_enc + pred_pose_enc_delta + pred_pose_enc_conf = pred_pose_enc_conf + pred_pose_enc_conf_delta + + # Apply final activation functions for translation, quaternion + activated_pose = activate_pose( + pred_pose_enc, trans_act=self.trans_act, quat_act=self.quat_act + ) + activated_conf = 1 + pred_pose_enc_conf.exp() + activated_pose = torch.cat([activated_pose, activated_conf], dim=-1) + pred_pose_enc_list.append(activated_pose) + + return pred_pose_enc_list + + +def modulate(x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor) -> torch.Tensor: + """ + Modulate the input tensor using scaling and shifting parameters. + """ + # modified from https://github.com/facebookresearch/DiT/blob/796c29e532f47bba17c5b9c5eb39b9354b8b7c64/models.py#L19 + return x * (1 + scale) + shift diff --git a/argus/heads/dpt_head.py b/argus/heads/dpt_head.py new file mode 100644 index 0000000000000000000000000000000000000000..02c2d43481ec86656bd2587a3954af4457f6f53c --- /dev/null +++ b/argus/heads/dpt_head.py @@ -0,0 +1,474 @@ +import os +from typing import List, Dict, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from .head_act import activate_head +from .utils import create_uv_grid, position_grid_to_embed + + +class DPTHead(nn.Module): + """ + DPT Head for dense prediction tasks. + + This implementation follows the architecture described in "Vision Transformers for Dense Prediction" + (https://arxiv.org/abs/2103.13413). The DPT head processes features from a vision transformer + backbone and produces dense predictions by fusing multi-scale features. + + Args: + dim_in (int): Input dimension (channels). + patch_size (int, optional): Patch size. Default is 14. + output_dim (int, optional): Number of output channels. Default is 4. + activation (str, optional): Activation type. Default is "inv_log". + conf_activation (str, optional): Confidence activation type. Default is "expp1". + features (int, optional): Feature channels for intermediate representations. Default is 256. + out_channels (List[int], optional): Output channels for each intermediate layer. + intermediate_layer_idx (List[int], optional): Indices of layers from aggregated tokens used for DPT. + pos_embed (bool, optional): Whether to use positional embedding. Default is True. + feature_only (bool, optional): If True, return features only without the last several layers and activation head. Default is False. + down_ratio (int, optional): Downscaling factor for the output resolution. Default is 1. + """ + + def __init__( + self, + dim_in: int, + patch_size: int = 14, + output_dim: int = 4, + activation: str = "inv_log", + conf_activation: str = "expp1", + features: int = 256, + out_channels: List[int] = [256, 512, 1024, 1024], + intermediate_layer_idx: List[int] = [4, 11, 17, 23], + pos_embed: bool = True, + feature_only: bool = False, + down_ratio: int = 1, + ) -> None: + super(DPTHead, self).__init__() + self.patch_size = patch_size + self.activation = activation + self.conf_activation = conf_activation + self.pos_embed = pos_embed + self.feature_only = feature_only + self.down_ratio = down_ratio + self.intermediate_layer_idx = intermediate_layer_idx + + self.norm = nn.LayerNorm(dim_in) + + # Projection layers for each output channel from tokens. + self.projects = nn.ModuleList( + [nn.Conv2d(in_channels=dim_in, out_channels=oc, kernel_size=1, stride=1, padding=0) for oc in out_channels] + ) + + # Resize layers for upsampling feature maps. + self.resize_layers = nn.ModuleList( + [ + nn.ConvTranspose2d( + in_channels=out_channels[0], out_channels=out_channels[0], kernel_size=4, stride=4, padding=0 + ), + nn.ConvTranspose2d( + in_channels=out_channels[1], out_channels=out_channels[1], kernel_size=2, stride=2, padding=0 + ), + nn.Identity(), + nn.Conv2d( + in_channels=out_channels[3], out_channels=out_channels[3], kernel_size=3, stride=2, padding=1 + ), + ] + ) + + self.scratch = _make_scratch(out_channels, features, expand=False) + + # Attach additional modules to scratch. + self.scratch.stem_transpose = None + self.scratch.refinenet1 = _make_fusion_block(features) + self.scratch.refinenet2 = _make_fusion_block(features) + self.scratch.refinenet3 = _make_fusion_block(features) + self.scratch.refinenet4 = _make_fusion_block(features, has_residual=False) + + head_features_1 = features + head_features_2 = 32 + + if feature_only: + self.scratch.output_conv1 = nn.Conv2d(head_features_1, head_features_1, kernel_size=3, stride=1, padding=1) + else: + self.scratch.output_conv1 = nn.Conv2d( + head_features_1, head_features_1 // 2, kernel_size=3, stride=1, padding=1 + ) + conv2_in_channels = head_features_1 // 2 + + self.scratch.output_conv2 = nn.Sequential( + nn.Conv2d(conv2_in_channels, head_features_2, kernel_size=3, stride=1, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(head_features_2, output_dim, kernel_size=1, stride=1, padding=0), + ) + + def forward( + self, + aggregated_tokens_list: List[torch.Tensor], + images: torch.Tensor, + patch_start_idx: int, + frames_chunk_size: int = 8, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + """ + Forward pass through the DPT head, supports processing by chunking frames. + Args: + aggregated_tokens_list (List[Tensor]): List of token tensors from different transformer layers. + images (Tensor): Input images with shape [B, S, 3, H, W], in range [0, 1]. + patch_start_idx (int): Starting index for patch tokens in the token sequence. + Used to separate patch tokens from other tokens (e.g., camera or register tokens). + frames_chunk_size (int, optional): Number of frames to process in each chunk. + If None or larger than S, all frames are processed at once. Default: 8. + + Returns: + Tensor or Tuple[Tensor, Tensor]: + - If feature_only=True: Feature maps with shape [B, S, C, H, W] + - Otherwise: Tuple of (predictions, confidence) both with shape [B, S, 1, H, W] + """ + B, S, _, H, W = images.shape + + # If frames_chunk_size is not specified or greater than S, process all frames at once + if frames_chunk_size is None or frames_chunk_size >= S: + return self._forward_impl(aggregated_tokens_list, images, patch_start_idx) + + # Otherwise, process frames in chunks to manage memory usage + assert frames_chunk_size > 0 + + # Process frames in batches + all_preds = [] + all_conf = [] + + for frames_start_idx in range(0, S, frames_chunk_size): + frames_end_idx = min(frames_start_idx + frames_chunk_size, S) + + # Process batch of frames + if self.feature_only: + chunk_output = self._forward_impl( + aggregated_tokens_list, images, patch_start_idx, frames_start_idx, frames_end_idx + ) + all_preds.append(chunk_output) + else: + chunk_preds, chunk_conf = self._forward_impl( + aggregated_tokens_list, images, patch_start_idx, frames_start_idx, frames_end_idx + ) + all_preds.append(chunk_preds) + all_conf.append(chunk_conf) + + # Concatenate results along the sequence dimension + if self.feature_only: + return torch.cat(all_preds, dim=1) + else: + return torch.cat(all_preds, dim=1), torch.cat(all_conf, dim=1) + + def _forward_impl( + self, + aggregated_tokens_list: List[torch.Tensor], + images: torch.Tensor, + patch_start_idx: int, + frames_start_idx: int = None, + frames_end_idx: int = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + """ + Implementation of the forward pass through the DPT head. + + This method processes a specific chunk of frames from the sequence. + + Args: + aggregated_tokens_list (List[Tensor]): List of token tensors from different transformer layers. + images (Tensor): Input images with shape [B, S, 3, H, W]. + patch_start_idx (int): Starting index for patch tokens. + frames_start_idx (int, optional): Starting index for frames to process. + frames_end_idx (int, optional): Ending index for frames to process. + + Returns: + Tensor or Tuple[Tensor, Tensor]: Feature maps or (predictions, confidence). + """ + if frames_start_idx is not None and frames_end_idx is not None: + images = images[:, frames_start_idx:frames_end_idx].contiguous() + + B, S, _, H, W = images.shape + + patch_h, patch_w = H // self.patch_size, W // self.patch_size + + out = [] + dpt_idx = 0 + + for layer_idx in self.intermediate_layer_idx: + x = aggregated_tokens_list[layer_idx][:, :, patch_start_idx:] + + # Select frames if processing a chunk + if frames_start_idx is not None and frames_end_idx is not None: + x = x[:, frames_start_idx:frames_end_idx] + + x = x.reshape(B * S, -1, x.shape[-1]) + + x = self.norm(x) + + x = x.permute(0, 2, 1).reshape((x.shape[0], x.shape[-1], patch_h, patch_w)) + + x = self.projects[dpt_idx](x) + if self.pos_embed: + x = self._apply_pos_embed(x, W, H) + x = self.resize_layers[dpt_idx](x) + + out.append(x) + dpt_idx += 1 + + # Fuse features from multiple layers. + out = self.scratch_forward(out) + # Interpolate fused output to match target image resolution. + out = custom_interpolate( + out, + (int(patch_h * self.patch_size / self.down_ratio), int(patch_w * self.patch_size / self.down_ratio)), + mode="bilinear", + align_corners=True, + ) + + if self.pos_embed: + out = self._apply_pos_embed(out, W, H) + + if self.feature_only: + return out.view(B, S, *out.shape[1:]) + + out = self.scratch.output_conv2(out) + preds, conf = activate_head(out, activation=self.activation, conf_activation=self.conf_activation) + + preds = preds.view(B, S, *preds.shape[1:]) + conf = conf.view(B, S, *conf.shape[1:]) + return preds, conf + + def _apply_pos_embed(self, x: torch.Tensor, W: int, H: int, ratio: float = 0.1) -> torch.Tensor: + """ + Apply positional embedding to tensor x. + """ + patch_w = x.shape[-1] + patch_h = x.shape[-2] + pos_embed = create_uv_grid(patch_w, patch_h, aspect_ratio=W / H, dtype=x.dtype, device=x.device) + pos_embed = position_grid_to_embed(pos_embed, x.shape[1]) + pos_embed = pos_embed * ratio + pos_embed = pos_embed.permute(2, 0, 1)[None].expand(x.shape[0], -1, -1, -1) + return x + pos_embed + + def scratch_forward(self, features: List[torch.Tensor]) -> torch.Tensor: + """ + Forward pass through the fusion blocks. + + Args: + features (List[Tensor]): List of feature maps from different layers. + + Returns: + Tensor: Fused feature map. + """ + layer_1, layer_2, layer_3, layer_4 = features + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + out = self.scratch.refinenet4(layer_4_rn, size=layer_3_rn.shape[2:]) + del layer_4_rn, layer_4 + + out = self.scratch.refinenet3(out, layer_3_rn, size=layer_2_rn.shape[2:]) + del layer_3_rn, layer_3 + + out = self.scratch.refinenet2(out, layer_2_rn, size=layer_1_rn.shape[2:]) + del layer_2_rn, layer_2 + + out = self.scratch.refinenet1(out, layer_1_rn) + del layer_1_rn, layer_1 + + out = self.scratch.output_conv1(out) + return out + + +################################################################################ +# Modules +################################################################################ + + +def _make_fusion_block(features: int, size: int = None, has_residual: bool = True, groups: int = 1) -> nn.Module: + return FeatureFusionBlock( + features, + nn.ReLU(inplace=True), + deconv=False, + bn=False, + expand=False, + align_corners=True, + size=size, + has_residual=has_residual, + groups=groups, + ) + + +def _make_scratch(in_shape: List[int], out_shape: int, groups: int = 1, expand: bool = False) -> nn.Module: + scratch = nn.Module() + out_shape1 = out_shape + out_shape2 = out_shape + out_shape3 = out_shape + if len(in_shape) >= 4: + out_shape4 = out_shape + + if expand: + out_shape1 = out_shape + out_shape2 = out_shape * 2 + out_shape3 = out_shape * 4 + if len(in_shape) >= 4: + out_shape4 = out_shape * 8 + + scratch.layer1_rn = nn.Conv2d( + in_shape[0], out_shape1, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + scratch.layer2_rn = nn.Conv2d( + in_shape[1], out_shape2, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + scratch.layer3_rn = nn.Conv2d( + in_shape[2], out_shape3, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + if len(in_shape) >= 4: + scratch.layer4_rn = nn.Conv2d( + in_shape[3], out_shape4, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + return scratch + + +class ResidualConvUnit(nn.Module): + """Residual convolution module.""" + + def __init__(self, features, activation, bn, groups=1): + """Init. + + Args: + features (int): number of features + """ + super().__init__() + + self.bn = bn + self.groups = groups + self.conv1 = nn.Conv2d(features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups) + self.conv2 = nn.Conv2d(features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups) + + self.norm1 = None + self.norm2 = None + + self.activation = activation + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: output + """ + + out = self.activation(x) + out = self.conv1(out) + if self.norm1 is not None: + out = self.norm1(out) + + out = self.activation(out) + out = self.conv2(out) + if self.norm2 is not None: + out = self.norm2(out) + + return self.skip_add.add(out, x) + + +class FeatureFusionBlock(nn.Module): + """Feature fusion block.""" + + def __init__( + self, + features, + activation, + deconv=False, + bn=False, + expand=False, + align_corners=True, + size=None, + has_residual=True, + groups=1, + ): + """Init. + + Args: + features (int): number of features + """ + super(FeatureFusionBlock, self).__init__() + + self.deconv = deconv + self.align_corners = align_corners + self.groups = groups + self.expand = expand + out_features = features + if self.expand == True: + out_features = features // 2 + + self.out_conv = nn.Conv2d( + features, out_features, kernel_size=1, stride=1, padding=0, bias=True, groups=self.groups + ) + + if has_residual: + self.resConfUnit1 = ResidualConvUnit(features, activation, bn, groups=self.groups) + + self.has_residual = has_residual + self.resConfUnit2 = ResidualConvUnit(features, activation, bn, groups=self.groups) + + self.skip_add = nn.quantized.FloatFunctional() + self.size = size + + def forward(self, *xs, size=None): + """Forward pass. + + Returns: + tensor: output + """ + output = xs[0] + + if self.has_residual: + res = self.resConfUnit1(xs[1]) + output = self.skip_add.add(output, res) + + output = self.resConfUnit2(output) + + if (size is None) and (self.size is None): + modifier = {"scale_factor": 2} + elif size is None: + modifier = {"size": self.size} + else: + modifier = {"size": size} + + output = custom_interpolate(output, **modifier, mode="bilinear", align_corners=self.align_corners) + output = self.out_conv(output) + + return output + + +def custom_interpolate( + x: torch.Tensor, + size: Tuple[int, int] = None, + scale_factor: float = None, + mode: str = "bilinear", + align_corners: bool = True, +) -> torch.Tensor: + """ + Custom interpolate to avoid INT_MAX issues in nn.functional.interpolate. + """ + if size is None: + size = (int(x.shape[-2] * scale_factor), int(x.shape[-1] * scale_factor)) + + INT_MAX = 1610612736 + + input_elements = size[0] * size[1] * x.shape[0] * x.shape[1] + + if input_elements > INT_MAX: + chunks = torch.chunk(x, chunks=(input_elements // INT_MAX) + 1, dim=0) + interpolated_chunks = [ + nn.functional.interpolate(chunk, size=size, mode=mode, align_corners=align_corners) for chunk in chunks + ] + x = torch.cat(interpolated_chunks, dim=0) + return x.contiguous() + else: + return nn.functional.interpolate(x, size=size, mode=mode, align_corners=align_corners) diff --git a/argus/heads/head_act.py b/argus/heads/head_act.py new file mode 100644 index 0000000000000000000000000000000000000000..17281e30e32a77e7814ef31d4c050037d02a783f --- /dev/null +++ b/argus/heads/head_act.py @@ -0,0 +1,122 @@ +import torch +import torch.nn.functional as F + + +def activate_pose(pred_pose_enc, trans_act="linear", quat_act="linear"): + """ + Activate pose parameters with specified activation functions. + + Args: + pred_pose_enc: Tensor containing encoded pose parameters [translation, quaternion, xx] + trans_act: Activation type for translation component + quat_act: Activation type for quaternion component + + Returns: + Activated pose parameters tensor + """ + T = pred_pose_enc[..., :3] + quat = pred_pose_enc[..., 3:7] + + T = base_pose_act(T, trans_act) + quat = base_pose_act(quat, quat_act) + + # Discard the remaining parameters + pred_pose_enc = torch.cat([T, quat], dim=-1) + + return pred_pose_enc + + +def base_pose_act(pose_enc, act_type="linear"): + """ + Apply basic activation function to pose parameters. + + Args: + pose_enc: Tensor containing encoded pose parameters + act_type: Activation type ("linear", "inv_log", "exp", "relu") + + Returns: + Activated pose parameters + """ + if act_type == "linear": + return pose_enc + elif act_type == "inv_log": + return inverse_log_transform(pose_enc) + elif act_type == "exp": + return torch.exp(pose_enc) + elif act_type == "relu": + return F.relu(pose_enc) + elif act_type == "expp1": + return 1 + pose_enc.exp() + elif act_type == "expp0": + return pose_enc.exp() + elif act_type == "sigmoid": + return torch.sigmoid(pose_enc) + else: + raise ValueError(f"Unknown act_type: {act_type}") + + +def activate_head(out, activation="norm_exp", conf_activation="expp1"): + """ + Process network output to extract 3D points and confidence values. + + Args: + out: Network output tensor (B, C, H, W) + activation: Activation type for 3D points + conf_activation: Activation type for confidence values + + Returns: + Tuple of (3D points tensor, confidence tensor) + """ + # Move channels from last dim to the 4th dimension => (B, H, W, C) + fmap = out.permute(0, 2, 3, 1) # B,H,W,C expected + + # Split into xyz (first C-1 channels) and confidence (last channel) + xyz = fmap[:, :, :, :-1] + conf = fmap[:, :, :, -1] + + if activation == "norm_exp": + d = xyz.norm(dim=-1, keepdim=True).clamp(min=1e-8) + xyz_normed = xyz / d + pts3d = xyz_normed * torch.expm1(d) + elif activation == "norm": + pts3d = xyz / xyz.norm(dim=-1, keepdim=True) + elif activation == "exp": + pts3d = torch.exp(xyz) + elif activation == "relu": + pts3d = F.relu(xyz) + elif activation == "inv_log": + pts3d = inverse_log_transform(xyz) + elif activation == "xy_inv_log": + xy, z = xyz.split([2, 1], dim=-1) + z = inverse_log_transform(z) + pts3d = torch.cat([xy * z, z], dim=-1) + elif activation == "sigmoid": + pts3d = torch.sigmoid(xyz) + elif activation == "linear": + pts3d = xyz + else: + raise ValueError(f"Unknown activation: {activation}") + + if conf_activation == "expp1": + conf_out = 1 + conf.exp() + elif conf_activation == "expp0": + conf_out = conf.exp() + elif conf_activation == "sigmoid": + conf_out = torch.sigmoid(conf) + else: + raise ValueError(f"Unknown conf_activation: {conf_activation}") + + return pts3d, conf_out + + +def inverse_log_transform(y): + """ + Apply inverse log transform: sign(y) * (exp(|y|) - 1) + + Args: + y: Input tensor + + Returns: + Transformed tensor + """ + return torch.sign(y) * (torch.expm1(torch.abs(y))) diff --git a/argus/heads/utils.py b/argus/heads/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2e3aa67016b33bb1821668c89512b3ab367782d4 --- /dev/null +++ b/argus/heads/utils.py @@ -0,0 +1,142 @@ +import torch + + +def position_grid_to_embed(pos_grid: torch.Tensor, embed_dim: int, omega_0: float = 100) -> torch.Tensor: + """ + Convert 2D position grid (HxWx2) to sinusoidal embeddings (HxWxC) + + Args: + pos_grid: Tensor of shape (H, W, 2) containing 2D coordinates + embed_dim: Output channel dimension for embeddings + + Returns: + Tensor of shape (H, W, embed_dim) with positional embeddings + """ + H, W, grid_dim = pos_grid.shape + assert grid_dim == 2 + pos_flat = pos_grid.reshape(-1, grid_dim) # Flatten to (H*W, 2) + + # Process x and y coordinates separately + emb_x = make_sincos_pos_embed(embed_dim // 2, pos_flat[:, 0], omega_0=omega_0) # [1, H*W, D/2] + emb_y = make_sincos_pos_embed(embed_dim // 2, pos_flat[:, 1], omega_0=omega_0) # [1, H*W, D/2] + + # Combine and reshape + emb = torch.cat([emb_x, emb_y], dim=-1) # [1, H*W, D] + + return emb.view(H, W, embed_dim) # [H, W, D] + + +def make_sincos_pos_embed(embed_dim: int, pos: torch.Tensor, omega_0: float = 100) -> torch.Tensor: + """ + This function generates a 1D positional embedding from a given grid using sine and cosine functions. + + Args: + - embed_dim: The embedding dimension. + - pos: The position to generate the embedding from. + + Returns: + - emb: The generated 1D positional embedding. + """ + assert embed_dim % 2 == 0 + device = pos.device + omega = torch.arange(embed_dim // 2, dtype=torch.float32 if device.type == "mps" else torch.double, device=device) + omega /= embed_dim / 2.0 + omega = 1.0 / omega_0**omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = torch.einsum("m,d->md", pos, omega) # (M, D/2), outer product + + emb_sin = torch.sin(out) # (M, D/2) + emb_cos = torch.cos(out) # (M, D/2) + + emb = torch.cat([emb_sin, emb_cos], dim=1) # (M, D) + return emb.float() + + +# Inspired by https://github.com/microsoft/moge + + +def create_uv_grid( + width: int, height: int, aspect_ratio: float = None, dtype: torch.dtype = None, device: torch.device = None +) -> torch.Tensor: + """ + Create a normalized UV grid of shape (width, height, 2). + + The grid spans horizontally and vertically according to an aspect ratio, + ensuring the top-left corner is at (-x_span, -y_span) and the bottom-right + corner is at (x_span, y_span), normalized by the diagonal of the plane. + + Args: + width (int): Number of points horizontally. + height (int): Number of points vertically. + aspect_ratio (float, optional): Width-to-height ratio. Defaults to width/height. + dtype (torch.dtype, optional): Data type of the resulting tensor. + device (torch.device, optional): Device on which the tensor is created. + + Returns: + torch.Tensor: A (width, height, 2) tensor of UV coordinates. + """ + # Derive aspect ratio if not explicitly provided + if aspect_ratio is None: + aspect_ratio = float(width) / float(height) + + # Compute normalized spans for X and Y + diag_factor = (aspect_ratio**2 + 1.0) ** 0.5 + span_x = aspect_ratio / diag_factor + span_y = 1.0 / diag_factor + + # Establish the linspace boundaries + left_x = -span_x * (width - 1) / width + right_x = span_x * (width - 1) / width + top_y = -span_y * (height - 1) / height + bottom_y = span_y * (height - 1) / height + + # Generate 1D coordinates + x_coords = torch.linspace(left_x, right_x, steps=width, dtype=dtype, device=device) + y_coords = torch.linspace(top_y, bottom_y, steps=height, dtype=dtype, device=device) + + # Create 2D meshgrid (width x height) and stack into UV + uu, vv = torch.meshgrid(x_coords, y_coords, indexing="xy") + uv_grid = torch.stack((uu, vv), dim=-1) + + return uv_grid + + + +def reorder_by_reference(x: torch.Tensor, b_idx: torch.Tensor) -> torch.Tensor: + """Reorder tensor views to place the selected reference view at the first position (index 0), + while keeping the remaining views in their original order (excluding the reference view). + + Args: + x: Input tensor with shape (B, S, ...) where B = batch size, S = number of views, + and trailing dimensions can be arbitrary (e.g., N, C for patch tokens). + b_idx: 1D tensor of shape (B,) containing the index of the reference view for each batch element, + each value must be in the range [0, S-1]. + + Returns: + Reordered tensor with the same shape as input, where the reference view is at position 0 + and other views retain their original order (skipping the reference view). + + Example: + If B=1, S=5, b_idx=[2], input view order is [0,1,2,3,4], + output order becomes [2,0,1,3,4]. + """ + # Extract batch size (B) and number of views (S) from input shape + B, S = x.shape[0], x.shape[1] + + # No reordering needed if only one view exists + if S <= 1: + return x + + # Generate base index matrix (B, S): each row is [0, 1, ..., S-1] (same across batches) + idx = torch.arange(S, device=x.device).expand(B, -1) + + # Create mask to exclude reference view indices (True for non-reference positions) + mask = idx != b_idx.unsqueeze(1) + + # Build reorder indices: [reference_idx] + [all non-reference indices in original order] + # Reshape non-reference indices to (B, S-1) to match batch dimension, then concatenate + reorder_idx = torch.cat([b_idx.unsqueeze(1), idx[mask].reshape(B, S-1)], dim=1) + + # Advanced indexing to reorder: batch indices (B,1) paired with reorder indices (B,S) + return x[torch.arange(B).unsqueeze(1), reorder_idx] \ No newline at end of file diff --git a/argus/layers/__init__.py b/argus/layers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..53d561a132f8d50f20ef0d9718600cfa0b0fde1a --- /dev/null +++ b/argus/layers/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +from .mlp import Mlp +from .patch_embed import PatchEmbed diff --git a/argus/layers/attention.py b/argus/layers/attention.py new file mode 100644 index 0000000000000000000000000000000000000000..8e823b4b7a93cca75e4cbab1cdfbbc3121a316fa --- /dev/null +++ b/argus/layers/attention.py @@ -0,0 +1,93 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the Apache License, Version 2.0 +# found in the LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py + +import logging +import os +import warnings + +from torch import Tensor +from torch import nn +import torch.nn.functional as F + +XFORMERS_AVAILABLE = False + + +class Attention(nn.Module): + def __init__( + self, + dim: int, + num_heads: int = 8, + qkv_bias: bool = True, + proj_bias: bool = True, + attn_drop: float = 0.0, + proj_drop: float = 0.0, + norm_layer: nn.Module = nn.LayerNorm, + qk_norm: bool = False, + fused_attn: bool = True, # use F.scaled_dot_product_attention or not + rope=None, + ) -> None: + super().__init__() + assert dim % num_heads == 0, "dim should be divisible by num_heads" + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.scale = self.head_dim**-0.5 + self.fused_attn = fused_attn + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.q_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity() + self.k_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity() + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim, bias=proj_bias) + self.proj_drop = nn.Dropout(proj_drop) + self.rope = rope + + def forward(self, x: Tensor, pos=None) -> Tensor: + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4) + q, k, v = qkv.unbind(0) + q, k = self.q_norm(q), self.k_norm(k) + + if self.rope is not None: + q = self.rope(q, pos) + k = self.rope(k, pos) + + if self.fused_attn: + x = F.scaled_dot_product_attention(q, k, v, dropout_p=self.attn_drop.p if self.training else 0.0) + else: + q = q * self.scale + attn = q @ k.transpose(-2, -1) + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + x = attn @ v + + x = x.transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class MemEffAttention(Attention): + def forward(self, x: Tensor, attn_bias=None, pos=None) -> Tensor: + assert pos is None + if not XFORMERS_AVAILABLE: + if attn_bias is not None: + raise AssertionError("xFormers is required for using nested tensors") + return super().forward(x) + + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads) + + q, k, v = unbind(qkv, 2) + + x = memory_efficient_attention(q, k, v, attn_bias=attn_bias) + x = x.reshape([B, N, C]) + + x = self.proj(x) + x = self.proj_drop(x) + return x diff --git a/argus/layers/block.py b/argus/layers/block.py new file mode 100644 index 0000000000000000000000000000000000000000..fc5847352a1f8f5d63da28c99e94270e50ccf3aa --- /dev/null +++ b/argus/layers/block.py @@ -0,0 +1,247 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the Apache License, Version 2.0 +# found in the LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py + +import logging +import os +from typing import Callable, List, Any, Tuple, Dict +import warnings + +import torch +from torch import nn, Tensor + +from .attention import Attention +from .drop_path import DropPath +from .layer_scale import LayerScale +from .mlp import Mlp + + +XFORMERS_AVAILABLE = False + + +class Block(nn.Module): + def __init__( + self, + dim: int, + num_heads: int, + mlp_ratio: float = 4.0, + qkv_bias: bool = True, + proj_bias: bool = True, + ffn_bias: bool = True, + drop: float = 0.0, + attn_drop: float = 0.0, + init_values=None, + drop_path: float = 0.0, + act_layer: Callable[..., nn.Module] = nn.GELU, + norm_layer: Callable[..., nn.Module] = nn.LayerNorm, + attn_class: Callable[..., nn.Module] = Attention, + ffn_layer: Callable[..., nn.Module] = Mlp, + qk_norm: bool = False, + fused_attn: bool = True, # use F.scaled_dot_product_attention or not + rope=None, + ) -> None: + super().__init__() + + self.norm1 = norm_layer(dim) + + self.attn = attn_class( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + proj_bias=proj_bias, + attn_drop=attn_drop, + proj_drop=drop, + qk_norm=qk_norm, + fused_attn=fused_attn, + rope=rope, + ) + + self.ls1 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity() + self.drop_path1 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = ffn_layer( + in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop, bias=ffn_bias + ) + self.ls2 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity() + self.drop_path2 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + self.sample_drop_ratio = drop_path + + def forward(self, x: Tensor, pos=None) -> Tensor: + def attn_residual_func(x: Tensor, pos=None) -> Tensor: + return self.ls1(self.attn(self.norm1(x), pos=pos)) + + def ffn_residual_func(x: Tensor) -> Tensor: + return self.ls2(self.mlp(self.norm2(x))) + + if self.training and self.sample_drop_ratio > 0.1: + # the overhead is compensated only for a drop path rate larger than 0.1 + x = drop_add_residual_stochastic_depth( + x, pos=pos, residual_func=attn_residual_func, sample_drop_ratio=self.sample_drop_ratio + ) + x = drop_add_residual_stochastic_depth( + x, residual_func=ffn_residual_func, sample_drop_ratio=self.sample_drop_ratio + ) + elif self.training and self.sample_drop_ratio > 0.0: + x = x + self.drop_path1(attn_residual_func(x, pos=pos)) + x = x + self.drop_path1(ffn_residual_func(x)) # FIXME: drop_path2 + else: + x = x + attn_residual_func(x, pos=pos) + x = x + ffn_residual_func(x) + return x + + +def drop_add_residual_stochastic_depth( + x: Tensor, residual_func: Callable[[Tensor], Tensor], sample_drop_ratio: float = 0.0, pos=None +) -> Tensor: + # 1) extract subset using permutation + b, n, d = x.shape + sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1) + brange = (torch.randperm(b, device=x.device))[:sample_subset_size] + x_subset = x[brange] + + # 2) apply residual_func to get residual + if pos is not None: + # if necessary, apply rope to the subset + pos = pos[brange] + residual = residual_func(x_subset, pos=pos) + else: + residual = residual_func(x_subset) + + x_flat = x.flatten(1) + residual = residual.flatten(1) + + residual_scale_factor = b / sample_subset_size + + # 3) add the residual + x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor) + return x_plus_residual.view_as(x) + + +def get_branges_scales(x, sample_drop_ratio=0.0): + b, n, d = x.shape + sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1) + brange = (torch.randperm(b, device=x.device))[:sample_subset_size] + residual_scale_factor = b / sample_subset_size + return brange, residual_scale_factor + + +def add_residual(x, brange, residual, residual_scale_factor, scaling_vector=None): + if scaling_vector is None: + x_flat = x.flatten(1) + residual = residual.flatten(1) + x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor) + else: + x_plus_residual = scaled_index_add( + x, brange, residual.to(dtype=x.dtype), scaling=scaling_vector, alpha=residual_scale_factor + ) + return x_plus_residual + + +attn_bias_cache: Dict[Tuple, Any] = {} + + +def get_attn_bias_and_cat(x_list, branges=None): + """ + this will perform the index select, cat the tensors, and provide the attn_bias from cache + """ + batch_sizes = [b.shape[0] for b in branges] if branges is not None else [x.shape[0] for x in x_list] + all_shapes = tuple((b, x.shape[1]) for b, x in zip(batch_sizes, x_list)) + if all_shapes not in attn_bias_cache.keys(): + seqlens = [] + for b, x in zip(batch_sizes, x_list): + for _ in range(b): + seqlens.append(x.shape[1]) + attn_bias = fmha.BlockDiagonalMask.from_seqlens(seqlens) + attn_bias._batch_sizes = batch_sizes + attn_bias_cache[all_shapes] = attn_bias + + if branges is not None: + cat_tensors = index_select_cat([x.flatten(1) for x in x_list], branges).view(1, -1, x_list[0].shape[-1]) + else: + tensors_bs1 = tuple(x.reshape([1, -1, *x.shape[2:]]) for x in x_list) + cat_tensors = torch.cat(tensors_bs1, dim=1) + + return attn_bias_cache[all_shapes], cat_tensors + + +def drop_add_residual_stochastic_depth_list( + x_list: List[Tensor], + residual_func: Callable[[Tensor, Any], Tensor], + sample_drop_ratio: float = 0.0, + scaling_vector=None, +) -> Tensor: + # 1) generate random set of indices for dropping samples in the batch + branges_scales = [get_branges_scales(x, sample_drop_ratio=sample_drop_ratio) for x in x_list] + branges = [s[0] for s in branges_scales] + residual_scale_factors = [s[1] for s in branges_scales] + + # 2) get attention bias and index+concat the tensors + attn_bias, x_cat = get_attn_bias_and_cat(x_list, branges) + + # 3) apply residual_func to get residual, and split the result + residual_list = attn_bias.split(residual_func(x_cat, attn_bias=attn_bias)) # type: ignore + + outputs = [] + for x, brange, residual, residual_scale_factor in zip(x_list, branges, residual_list, residual_scale_factors): + outputs.append(add_residual(x, brange, residual, residual_scale_factor, scaling_vector).view_as(x)) + return outputs + + +class NestedTensorBlock(Block): + def forward_nested(self, x_list: List[Tensor]) -> List[Tensor]: + """ + x_list contains a list of tensors to nest together and run + """ + assert isinstance(self.attn, MemEffAttention) + + if self.training and self.sample_drop_ratio > 0.0: + + def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor: + return self.attn(self.norm1(x), attn_bias=attn_bias) + + def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor: + return self.mlp(self.norm2(x)) + + x_list = drop_add_residual_stochastic_depth_list( + x_list, + residual_func=attn_residual_func, + sample_drop_ratio=self.sample_drop_ratio, + scaling_vector=(self.ls1.gamma if isinstance(self.ls1, LayerScale) else None), + ) + x_list = drop_add_residual_stochastic_depth_list( + x_list, + residual_func=ffn_residual_func, + sample_drop_ratio=self.sample_drop_ratio, + scaling_vector=(self.ls2.gamma if isinstance(self.ls1, LayerScale) else None), + ) + return x_list + else: + + def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor: + return self.ls1(self.attn(self.norm1(x), attn_bias=attn_bias)) + + def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor: + return self.ls2(self.mlp(self.norm2(x))) + + attn_bias, x = get_attn_bias_and_cat(x_list) + x = x + attn_residual_func(x, attn_bias=attn_bias) + x = x + ffn_residual_func(x) + return attn_bias.split(x) + + def forward(self, x_or_x_list): + if isinstance(x_or_x_list, Tensor): + return super().forward(x_or_x_list) + elif isinstance(x_or_x_list, list): + if not XFORMERS_AVAILABLE: + raise AssertionError("xFormers is required for using nested tensors") + return self.forward_nested(x_or_x_list) + else: + raise AssertionError diff --git a/argus/layers/drop_path.py b/argus/layers/drop_path.py new file mode 100644 index 0000000000000000000000000000000000000000..1d640e0b969b8dcba96260243473700b4e5b24b5 --- /dev/null +++ b/argus/layers/drop_path.py @@ -0,0 +1,34 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the Apache License, Version 2.0 +# found in the LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/drop.py + + +from torch import nn + + +def drop_path(x, drop_prob: float = 0.0, training: bool = False): + if drop_prob == 0.0 or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = x.new_empty(shape).bernoulli_(keep_prob) + if keep_prob > 0.0: + random_tensor.div_(keep_prob) + output = x * random_tensor + return output + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" + + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) diff --git a/argus/layers/layer_scale.py b/argus/layers/layer_scale.py new file mode 100644 index 0000000000000000000000000000000000000000..4ddfc51c3d87370d50175f5b4e649dac1c614ff9 --- /dev/null +++ b/argus/layers/layer_scale.py @@ -0,0 +1,22 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the Apache License, Version 2.0 +# found in the LICENSE file in the root directory of this source tree. + +# Modified from: https://github.com/huggingface/pytorch-image-models/blob/main/timm/models/vision_transformer.py#L103-L110 + +from typing import Union + +import torch +from torch import Tensor +from torch import nn + + +class LayerScale(nn.Module): + def __init__(self, dim: int, init_values: Union[float, Tensor] = 1e-5, inplace: bool = False) -> None: + super().__init__() + self.inplace = inplace + self.gamma = nn.Parameter(init_values * torch.ones(dim)) + + def forward(self, x: Tensor) -> Tensor: + return x.mul_(self.gamma) if self.inplace else x * self.gamma diff --git a/argus/layers/mlp.py b/argus/layers/mlp.py new file mode 100644 index 0000000000000000000000000000000000000000..bbf9432aae9258612caeae910a7bde17999e328e --- /dev/null +++ b/argus/layers/mlp.py @@ -0,0 +1,40 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the Apache License, Version 2.0 +# found in the LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/mlp.py + + +from typing import Callable, Optional + +from torch import Tensor, nn + + +class Mlp(nn.Module): + def __init__( + self, + in_features: int, + hidden_features: Optional[int] = None, + out_features: Optional[int] = None, + act_layer: Callable[..., nn.Module] = nn.GELU, + drop: float = 0.0, + bias: bool = True, + ) -> None: + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features, bias=bias) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features, bias=bias) + self.drop = nn.Dropout(drop) + + def forward(self, x: Tensor) -> Tensor: + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x diff --git a/argus/layers/patch_embed.py b/argus/layers/patch_embed.py new file mode 100644 index 0000000000000000000000000000000000000000..bc19605e4d6e88d06355ae3b1afddc76f595aafe --- /dev/null +++ b/argus/layers/patch_embed.py @@ -0,0 +1,85 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the Apache License, Version 2.0 +# found in the LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py + +from typing import Callable, Optional, Tuple, Union + +from torch import Tensor +import torch.nn as nn + + +def make_2tuple(x): + if isinstance(x, tuple): + assert len(x) == 2 + return x + + assert isinstance(x, int) + return (x, x) + + +class PatchEmbed(nn.Module): + """ + 2D image to patch embedding: (B,C,H,W) -> (B,N,D) + + Args: + img_size: Image size. + patch_size: Patch token size. + in_chans: Number of input image channels. + embed_dim: Number of linear projection output channels. + norm_layer: Normalization layer. + """ + + def __init__( + self, + img_size: Union[int, Tuple[int, int]] = 224, + patch_size: Union[int, Tuple[int, int]] = 16, + in_chans: int = 3, + embed_dim: int = 768, + norm_layer: Optional[Callable] = None, + flatten_embedding: bool = True, + ) -> None: + super().__init__() + + image_HW = make_2tuple(img_size) + patch_HW = make_2tuple(patch_size) + patch_grid_size = (image_HW[0] // patch_HW[0], image_HW[1] // patch_HW[1]) + + self.img_size = image_HW + self.patch_size = patch_HW + self.patches_resolution = patch_grid_size + self.num_patches = patch_grid_size[0] * patch_grid_size[1] + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.flatten_embedding = flatten_embedding + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_HW, stride=patch_HW) + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + def forward(self, x: Tensor) -> Tensor: + _, _, H, W = x.shape + patch_H, patch_W = self.patch_size + + assert H % patch_H == 0, f"Input image height {H} is not a multiple of patch height {patch_H}" + assert W % patch_W == 0, f"Input image width {W} is not a multiple of patch width: {patch_W}" + + x = self.proj(x) # B C H W + H, W = x.size(2), x.size(3) + x = x.flatten(2).transpose(1, 2) # B HW C + x = self.norm(x) + if not self.flatten_embedding: + x = x.reshape(-1, H, W, self.embed_dim) # B H W C + return x + + def flops(self) -> float: + Ho, Wo = self.patches_resolution + flops = Ho * Wo * self.embed_dim * self.in_chans * (self.patch_size[0] * self.patch_size[1]) + if self.norm is not None: + flops += Ho * Wo * self.embed_dim + return flops diff --git a/argus/layers/rope.py b/argus/layers/rope.py new file mode 100644 index 0000000000000000000000000000000000000000..4d5d33304e55dbd05687bd86752a47a80e5f82df --- /dev/null +++ b/argus/layers/rope.py @@ -0,0 +1,188 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the Apache License, Version 2.0 +# found in the LICENSE file in the root directory of this source tree. + + +# Implementation of 2D Rotary Position Embeddings (RoPE). + +# This module provides a clean implementation of 2D Rotary Position Embeddings, +# which extends the original RoPE concept to handle 2D spatial positions. + +# Inspired by: +# https://github.com/meta-llama/codellama/blob/main/llama/model.py +# https://github.com/naver-ai/rope-vit + + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from typing import Dict, Tuple + + +class PositionGetter: + """Generates and caches 2D spatial positions for patches in a grid. + + This class efficiently manages the generation of spatial coordinates for patches + in a 2D grid, caching results to avoid redundant computations. + + Attributes: + position_cache: Dictionary storing precomputed position tensors for different + grid dimensions. + """ + + def __init__(self): + """Initializes the position generator with an empty cache.""" + self.position_cache: Dict[Tuple[int, int], torch.Tensor] = {} + + def __call__(self, batch_size: int, height: int, width: int, device: torch.device) -> torch.Tensor: + """Generates spatial positions for a batch of patches. + + Args: + batch_size: Number of samples in the batch. + height: Height of the grid in patches. + width: Width of the grid in patches. + device: Target device for the position tensor. + + Returns: + Tensor of shape (batch_size, height*width, 2) containing y,x coordinates + for each position in the grid, repeated for each batch item. + """ + if (height, width) not in self.position_cache: + y_coords = torch.arange(height, device=device) + x_coords = torch.arange(width, device=device) + positions = torch.cartesian_prod(y_coords, x_coords) + self.position_cache[height, width] = positions + + cached_positions = self.position_cache[height, width] + return cached_positions.view(1, height * width, 2).expand(batch_size, -1, -1).clone() + + +class RotaryPositionEmbedding2D(nn.Module): + """2D Rotary Position Embedding implementation. + + This module applies rotary position embeddings to input tokens based on their + 2D spatial positions. It handles the position-dependent rotation of features + separately for vertical and horizontal dimensions. + + Args: + frequency: Base frequency for the position embeddings. Default: 100.0 + scaling_factor: Scaling factor for frequency computation. Default: 1.0 + + Attributes: + base_frequency: Base frequency for computing position embeddings. + scaling_factor: Factor to scale the computed frequencies. + frequency_cache: Cache for storing precomputed frequency components. + """ + + def __init__(self, frequency: float = 100.0, scaling_factor: float = 1.0): + """Initializes the 2D RoPE module.""" + super().__init__() + self.base_frequency = frequency + self.scaling_factor = scaling_factor + self.frequency_cache: Dict[Tuple, Tuple[torch.Tensor, torch.Tensor]] = {} + + def _compute_frequency_components( + self, dim: int, seq_len: int, device: torch.device, dtype: torch.dtype + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Computes frequency components for rotary embeddings. + + Args: + dim: Feature dimension (must be even). + seq_len: Maximum sequence length. + device: Target device for computations. + dtype: Data type for the computed tensors. + + Returns: + Tuple of (cosine, sine) tensors for frequency components. + """ + cache_key = (dim, seq_len, device, dtype) + if cache_key not in self.frequency_cache: + # Compute frequency bands + exponents = torch.arange(0, dim, 2, device=device).float() / dim + inv_freq = 1.0 / (self.base_frequency**exponents) + + # Generate position-dependent frequencies + positions = torch.arange(seq_len, device=device, dtype=inv_freq.dtype) + angles = torch.einsum("i,j->ij", positions, inv_freq) + + # Compute and cache frequency components + angles = angles.to(dtype) + angles = torch.cat((angles, angles), dim=-1) + cos_components = angles.cos().to(dtype) + sin_components = angles.sin().to(dtype) + self.frequency_cache[cache_key] = (cos_components, sin_components) + + return self.frequency_cache[cache_key] + + @staticmethod + def _rotate_features(x: torch.Tensor) -> torch.Tensor: + """Performs feature rotation by splitting and recombining feature dimensions. + + Args: + x: Input tensor to rotate. + + Returns: + Rotated feature tensor. + """ + feature_dim = x.shape[-1] + x1, x2 = x[..., : feature_dim // 2], x[..., feature_dim // 2 :] + return torch.cat((-x2, x1), dim=-1) + + def _apply_1d_rope( + self, tokens: torch.Tensor, positions: torch.Tensor, cos_comp: torch.Tensor, sin_comp: torch.Tensor + ) -> torch.Tensor: + """Applies 1D rotary position embeddings along one dimension. + + Args: + tokens: Input token features. + positions: Position indices. + cos_comp: Cosine components for rotation. + sin_comp: Sine components for rotation. + + Returns: + Tokens with applied rotary position embeddings. + """ + # Embed positions with frequency components + cos = F.embedding(positions, cos_comp)[:, None, :, :] + sin = F.embedding(positions, sin_comp)[:, None, :, :] + + # Apply rotation + return (tokens * cos) + (self._rotate_features(tokens) * sin) + + def forward(self, tokens: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: + """Applies 2D rotary position embeddings to input tokens. + + Args: + tokens: Input tensor of shape (batch_size, n_heads, n_tokens, dim). + The feature dimension (dim) must be divisible by 4. + positions: Position tensor of shape (batch_size, n_tokens, 2) containing + the y and x coordinates for each token. + + Returns: + Tensor of same shape as input with applied 2D rotary position embeddings. + + Raises: + AssertionError: If input dimensions are invalid or positions are malformed. + """ + # Validate inputs + assert tokens.size(-1) % 2 == 0, "Feature dimension must be even" + assert positions.ndim == 3 and positions.shape[-1] == 2, "Positions must have shape (batch_size, n_tokens, 2)" + + # Compute feature dimension for each spatial direction + feature_dim = tokens.size(-1) // 2 + + # Get frequency components + max_position = int(positions.max()) + 1 + cos_comp, sin_comp = self._compute_frequency_components(feature_dim, max_position, tokens.device, tokens.dtype) + + # Split features for vertical and horizontal processing + vertical_features, horizontal_features = tokens.chunk(2, dim=-1) + + # Apply RoPE separately for each dimension + vertical_features = self._apply_1d_rope(vertical_features, positions[..., 0], cos_comp, sin_comp) + horizontal_features = self._apply_1d_rope(horizontal_features, positions[..., 1], cos_comp, sin_comp) + + # Combine processed features + return torch.cat((vertical_features, horizontal_features), dim=-1) diff --git a/argus/layers/swiglu_ffn.py b/argus/layers/swiglu_ffn.py new file mode 100644 index 0000000000000000000000000000000000000000..1dd991e1deb87141ccd282098d4b9d38fed6ef25 --- /dev/null +++ b/argus/layers/swiglu_ffn.py @@ -0,0 +1,67 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the Apache License, Version 2.0 +# found in the LICENSE file in the root directory of this source tree. + +import os +from typing import Callable, Optional +import warnings + +from torch import Tensor, nn +import torch.nn.functional as F + + +class SwiGLUFFN(nn.Module): + def __init__( + self, + in_features: int, + hidden_features: Optional[int] = None, + out_features: Optional[int] = None, + act_layer: Callable[..., nn.Module] = None, + drop: float = 0.0, + bias: bool = True, + ) -> None: + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.w12 = nn.Linear(in_features, 2 * hidden_features, bias=bias) + self.w3 = nn.Linear(hidden_features, out_features, bias=bias) + + def forward(self, x: Tensor) -> Tensor: + x12 = self.w12(x) + x1, x2 = x12.chunk(2, dim=-1) + hidden = F.silu(x1) * x2 + return self.w3(hidden) + + +XFORMERS_ENABLED = os.environ.get("XFORMERS_DISABLED") is None +# try: +# if XFORMERS_ENABLED: +# from xformers.ops import SwiGLU + +# XFORMERS_AVAILABLE = True +# warnings.warn("xFormers is available (SwiGLU)") +# else: +# warnings.warn("xFormers is disabled (SwiGLU)") +# raise ImportError +# except ImportError: +SwiGLU = SwiGLUFFN +XFORMERS_AVAILABLE = False + +# warnings.warn("xFormers is not available (SwiGLU)") + + +class SwiGLUFFNFused(SwiGLU): + def __init__( + self, + in_features: int, + hidden_features: Optional[int] = None, + out_features: Optional[int] = None, + act_layer: Callable[..., nn.Module] = None, + drop: float = 0.0, + bias: bool = True, + ) -> None: + out_features = out_features or in_features + hidden_features = hidden_features or in_features + hidden_features = (int(hidden_features * 2 / 3) + 7) // 8 * 8 + super().__init__(in_features=in_features, hidden_features=hidden_features, out_features=out_features, bias=bias) diff --git a/argus/layers/vision_transformer.py b/argus/layers/vision_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..5563719af3e9851ba5a3b0e312f201e739636152 --- /dev/null +++ b/argus/layers/vision_transformer.py @@ -0,0 +1,401 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the Apache License, Version 2.0 +# found in the LICENSE file in the root directory of this source tree. + +# References: +# https://github.com/facebookresearch/dino/blob/main/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py + +from functools import partial +import math +import logging +from typing import Sequence, Tuple, Union, Callable + +import torch +import torch.nn as nn +from torch.utils.checkpoint import checkpoint +from torch.nn.init import trunc_normal_ +from .mlp import Mlp +from .patch_embed import PatchEmbed +from .swiglu_ffn import SwiGLUFFNFused +from .attention import MemEffAttention +from .block import NestedTensorBlock as Block + +logger = logging.getLogger("dinov2") + + +def named_apply(fn: Callable, module: nn.Module, name="", depth_first=True, include_root=False) -> nn.Module: + if not depth_first and include_root: + fn(module=module, name=name) + for child_name, child_module in module.named_children(): + child_name = ".".join((name, child_name)) if name else child_name + named_apply(fn=fn, module=child_module, name=child_name, depth_first=depth_first, include_root=True) + if depth_first and include_root: + fn(module=module, name=name) + return module + + +class BlockChunk(nn.ModuleList): + def forward(self, x): + for b in self: + x = b(x) + return x + + +class DinoVisionTransformer(nn.Module): + def __init__( + self, + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4.0, + qkv_bias=True, + ffn_bias=True, + proj_bias=True, + drop_path_rate=0.0, + drop_path_uniform=False, + init_values=None, # for layerscale: None or 0 => no layerscale + embed_layer=PatchEmbed, + act_layer=nn.GELU, + block_fn=Block, + ffn_layer="mlp", + block_chunks=1, + num_register_tokens=0, + interpolate_antialias=False, + interpolate_offset=0.1, + qk_norm=False, + ): + """ + Args: + img_size (int, tuple): input image size + patch_size (int, tuple): patch size + in_chans (int): number of input channels + embed_dim (int): embedding dimension + depth (int): depth of transformer + num_heads (int): number of attention heads + mlp_ratio (int): ratio of mlp hidden dim to embedding dim + qkv_bias (bool): enable bias for qkv if True + proj_bias (bool): enable bias for proj in attn if True + ffn_bias (bool): enable bias for ffn if True + drop_path_rate (float): stochastic depth rate + drop_path_uniform (bool): apply uniform drop rate across blocks + weight_init (str): weight init scheme + init_values (float): layer-scale init values + embed_layer (nn.Module): patch embedding layer + act_layer (nn.Module): MLP activation layer + block_fn (nn.Module): transformer block class + ffn_layer (str): "mlp", "swiglu", "swiglufused" or "identity" + block_chunks: (int) split block sequence into block_chunks units for FSDP wrap + num_register_tokens: (int) number of extra cls tokens (so-called "registers") + interpolate_antialias: (str) flag to apply anti-aliasing when interpolating positional embeddings + interpolate_offset: (float) work-around offset to apply when interpolating positional embeddings + """ + super().__init__() + norm_layer = partial(nn.LayerNorm, eps=1e-6) + + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + self.num_tokens = 1 + self.n_blocks = depth + self.num_heads = num_heads + self.patch_size = patch_size + self.num_register_tokens = num_register_tokens + self.interpolate_antialias = interpolate_antialias + self.interpolate_offset = interpolate_offset + self.use_reentrant = False # hardcoded to False + + self.patch_embed = embed_layer(img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) + num_patches = self.patch_embed.num_patches + + self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) + self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim)) + assert num_register_tokens >= 0 + self.register_tokens = ( + nn.Parameter(torch.zeros(1, num_register_tokens, embed_dim)) if num_register_tokens else None + ) + + if drop_path_uniform is True: + dpr = [drop_path_rate] * depth + else: + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule + + if ffn_layer == "mlp": + logger.info("using MLP layer as FFN") + ffn_layer = Mlp + elif ffn_layer == "swiglufused" or ffn_layer == "swiglu": + logger.info("using SwiGLU layer as FFN") + ffn_layer = SwiGLUFFNFused + elif ffn_layer == "identity": + logger.info("using Identity layer as FFN") + + def f(*args, **kwargs): + return nn.Identity() + + ffn_layer = f + else: + raise NotImplementedError + + blocks_list = [ + block_fn( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + proj_bias=proj_bias, + ffn_bias=ffn_bias, + drop_path=dpr[i], + norm_layer=norm_layer, + act_layer=act_layer, + ffn_layer=ffn_layer, + init_values=init_values, + qk_norm=qk_norm, + ) + for i in range(depth) + ] + if block_chunks > 0: + self.chunked_blocks = True + chunked_blocks = [] + chunksize = depth // block_chunks + for i in range(0, depth, chunksize): + # this is to keep the block index consistent if we chunk the block list + chunked_blocks.append([nn.Identity()] * i + blocks_list[i : i + chunksize]) + self.blocks = nn.ModuleList([BlockChunk(p) for p in chunked_blocks]) + else: + self.chunked_blocks = False + self.blocks = nn.ModuleList(blocks_list) + + self.norm = norm_layer(embed_dim) + self.head = nn.Identity() + + self.mask_token = nn.Parameter(torch.zeros(1, embed_dim)) + + self.init_weights() + + def init_weights(self): + trunc_normal_(self.pos_embed, std=0.02) + nn.init.normal_(self.cls_token, std=1e-6) + if self.register_tokens is not None: + nn.init.normal_(self.register_tokens, std=1e-6) + named_apply(init_weights_vit_timm, self) + + def interpolate_pos_encoding(self, x, w, h): + previous_dtype = x.dtype + npatch = x.shape[1] - 1 + N = self.pos_embed.shape[1] - 1 + if npatch == N and w == h: + return self.pos_embed + pos_embed = self.pos_embed.float() + class_pos_embed = pos_embed[:, 0] + patch_pos_embed = pos_embed[:, 1:] + dim = x.shape[-1] + w0 = w // self.patch_size + h0 = h // self.patch_size + M = int(math.sqrt(N)) # Recover the number of patches in each dimension + assert N == M * M + kwargs = {} + if self.interpolate_offset: + # Historical kludge: add a small number to avoid floating point error in the interpolation, see https://github.com/facebookresearch/dino/issues/8 + # Note: still needed for backward-compatibility, the underlying operators are using both output size and scale factors + sx = float(w0 + self.interpolate_offset) / M + sy = float(h0 + self.interpolate_offset) / M + kwargs["scale_factor"] = (sx, sy) + else: + # Simply specify an output size instead of a scale factor + kwargs["size"] = (w0, h0) + patch_pos_embed = nn.functional.interpolate( + patch_pos_embed.reshape(1, M, M, dim).permute(0, 3, 1, 2), + mode="bicubic", + antialias=self.interpolate_antialias, + **kwargs, + ) + assert (w0, h0) == patch_pos_embed.shape[-2:] + patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim) + return torch.cat((class_pos_embed.unsqueeze(0), patch_pos_embed), dim=1).to(previous_dtype) + + def prepare_tokens_with_masks(self, x, masks=None): + B, nc, w, h = x.shape + x = self.patch_embed(x) + if masks is not None: + x = torch.where(masks.unsqueeze(-1), self.mask_token.to(x.dtype).unsqueeze(0), x) + + x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1) + x = x + self.interpolate_pos_encoding(x, w, h) + + if self.register_tokens is not None: + x = torch.cat((x[:, :1], self.register_tokens.expand(x.shape[0], -1, -1), x[:, 1:]), dim=1) + + return x + + def forward_features_list(self, x_list, masks_list): + x = [self.prepare_tokens_with_masks(x, masks) for x, masks in zip(x_list, masks_list)] + + for blk in self.blocks: + if self.training: + x = checkpoint(blk, x, use_reentrant=self.use_reentrant) + else: + x = blk(x) + + all_x = x + output = [] + for x, masks in zip(all_x, masks_list): + x_norm = self.norm(x) + output.append( + { + "x_norm_clstoken": x_norm[:, 0], + "x_norm_regtokens": x_norm[:, 1 : self.num_register_tokens + 1], + "x_norm_patchtokens": x_norm[:, self.num_register_tokens + 1 :], + "x_prenorm": x, + "masks": masks, + } + ) + return output + + def forward_features(self, x, masks=None): + if isinstance(x, list): + return self.forward_features_list(x, masks) + + x = self.prepare_tokens_with_masks(x, masks) + + for blk in self.blocks: + if self.training: + x = checkpoint(blk, x, use_reentrant=self.use_reentrant) + else: + x = blk(x) + + x_norm = self.norm(x) + return { + "x_norm_clstoken": x_norm[:, 0], + "x_norm_regtokens": x_norm[:, 1 : self.num_register_tokens + 1], + "x_norm_patchtokens": x_norm[:, self.num_register_tokens + 1 :], + "x_prenorm": x, + "masks": masks, + } + + def _get_intermediate_layers_not_chunked(self, x, n=1): + x = self.prepare_tokens_with_masks(x) + # If n is an int, take the n last blocks. If it's a list, take them + output, total_block_len = [], len(self.blocks) + blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n + for i, blk in enumerate(self.blocks): + x = blk(x) + if i in blocks_to_take: + output.append(x) + assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found" + return output + + def _get_intermediate_layers_chunked(self, x, n=1): + x = self.prepare_tokens_with_masks(x) + output, i, total_block_len = [], 0, len(self.blocks[-1]) + # If n is an int, take the n last blocks. If it's a list, take them + blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n + for block_chunk in self.blocks: + for blk in block_chunk[i:]: # Passing the nn.Identity() + x = blk(x) + if i in blocks_to_take: + output.append(x) + i += 1 + assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found" + return output + + def get_intermediate_layers( + self, + x: torch.Tensor, + n: Union[int, Sequence] = 1, # Layers or n last layers to take + reshape: bool = False, + return_class_token: bool = False, + norm=True, + ) -> Tuple[Union[torch.Tensor, Tuple[torch.Tensor]]]: + if self.chunked_blocks: + outputs = self._get_intermediate_layers_chunked(x, n) + else: + outputs = self._get_intermediate_layers_not_chunked(x, n) + if norm: + outputs = [self.norm(out) for out in outputs] + class_tokens = [out[:, 0] for out in outputs] + outputs = [out[:, 1 + self.num_register_tokens :] for out in outputs] + if reshape: + B, _, w, h = x.shape + outputs = [ + out.reshape(B, w // self.patch_size, h // self.patch_size, -1).permute(0, 3, 1, 2).contiguous() + for out in outputs + ] + if return_class_token: + return tuple(zip(outputs, class_tokens)) + return tuple(outputs) + + def forward(self, *args, is_training=True, **kwargs): + ret = self.forward_features(*args, **kwargs) + if is_training: + return ret + else: + return self.head(ret["x_norm_clstoken"]) + + +def init_weights_vit_timm(module: nn.Module, name: str = ""): + """ViT weight initialization, original timm impl (for reproducibility)""" + if isinstance(module, nn.Linear): + trunc_normal_(module.weight, std=0.02) + if module.bias is not None: + nn.init.zeros_(module.bias) + + +def vit_small(patch_size=16, num_register_tokens=0, **kwargs): + model = DinoVisionTransformer( + patch_size=patch_size, + embed_dim=384, + depth=12, + num_heads=6, + mlp_ratio=4, + block_fn=partial(Block, attn_class=MemEffAttention), + num_register_tokens=num_register_tokens, + **kwargs, + ) + return model + + +def vit_base(patch_size=16, num_register_tokens=0, **kwargs): + model = DinoVisionTransformer( + patch_size=patch_size, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4, + block_fn=partial(Block, attn_class=MemEffAttention), + num_register_tokens=num_register_tokens, + **kwargs, + ) + return model + + +def vit_large(patch_size=16, num_register_tokens=0, **kwargs): + model = DinoVisionTransformer( + patch_size=patch_size, + embed_dim=1024, + depth=24, + num_heads=16, + mlp_ratio=4, + block_fn=partial(Block, attn_class=MemEffAttention), + num_register_tokens=num_register_tokens, + **kwargs, + ) + return model + + +def vit_giant2(patch_size=16, num_register_tokens=0, **kwargs): + """ + Close to ViT-giant, with embed-dim 1536 and 24 heads => embed-dim per head 64 + """ + model = DinoVisionTransformer( + patch_size=patch_size, + embed_dim=1536, + depth=40, + num_heads=24, + mlp_ratio=4, + block_fn=partial(Block, attn_class=MemEffAttention), + num_register_tokens=num_register_tokens, + **kwargs, + ) + return model diff --git a/argus/models/__init__.py b/argus/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7b3cdba8f0417592fec0c4b198244c248ea0130f --- /dev/null +++ b/argus/models/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2026 Realsee. All rights reserved. +# Licensed under the Apache License, Version 2.0. diff --git a/argus/models/aggregator.py b/argus/models/aggregator.py new file mode 100644 index 0000000000000000000000000000000000000000..43d83dbfd7715f02e2ad93b842b13794161f3996 --- /dev/null +++ b/argus/models/aggregator.py @@ -0,0 +1,502 @@ +import logging +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.utils.checkpoint import checkpoint +from typing import Optional, Tuple, Union, List, Dict, Any +from argus.layers import Mlp +from argus.layers import PatchEmbed +from argus.layers.block import Block +from argus.layers.rope import RotaryPositionEmbedding2D, PositionGetter +from argus.layers.vision_transformer import vit_small, vit_base, vit_large, vit_giant2 +from argus.heads.utils import reorder_by_reference + +logger = logging.getLogger(__name__) + +_RESNET_MEAN = [0.485, 0.456, 0.406] +_RESNET_STD = [0.229, 0.224, 0.225] + + +class Aggregator(nn.Module): + """ + Args: + img_size (int): Image size in pixels. + patch_size (int): Size of each patch for PatchEmbed. + embed_dim (int): Dimension of the token embeddings. + depth (int): Number of blocks. + num_heads (int): Number of attention heads. + mlp_ratio (float): Ratio of MLP hidden dim to embedding dim. + num_register_tokens (int): Number of register tokens. + block_fn (nn.Module): The block type used for attention (Block by default). + qkv_bias (bool): Whether to include bias in QKV projections. + proj_bias (bool): Whether to include bias in the output projection. + ffn_bias (bool): Whether to include bias in MLP layers. + patch_embed (str): Type of patch embed. e.g., "conv" or "dinov2_vitl14_reg". + aa_order (list[str]): The order of alternating attention, e.g. ["frame", "global"]. + aa_block_size (int): How many blocks to group under each attention type before switching. If not necessary, set to 1. + qk_norm (bool): Whether to apply QK normalization. + rope_freq (int): Base frequency for rotary embedding. -1 to disable. + init_values (float): Init scale for layer scale. + reorder_by_learning_ref (bool): Whether to reorder features by learning reference view index. + ref_aa_block_num (int): Number of aa blocks for reference view learning. + """ + + def __init__( + self, + img_size=518, + patch_size=14, + embed_dim=1024, + depth=24, + num_heads=16, + mlp_ratio=4.0, + num_register_tokens=4, + block_fn=Block, + qkv_bias=True, + proj_bias=True, + ffn_bias=True, + patch_embed="dinov2_vitl14_reg", + aa_order=["frame", "global"], + aa_block_size=1, + qk_norm=True, + rope_freq=100, + init_values=0.01, + reorder_by_learning_ref=True, + ref_aa_block_num=2, + save_inference_memory=True, + ): + super().__init__() + + self.reorder_by_learning_ref = reorder_by_learning_ref + self.save_inference_memory = save_inference_memory + + self.__build_patch_embed__(patch_embed, img_size, patch_size, num_register_tokens, embed_dim=embed_dim) + + # Initialize rotary position embedding if frequency > 0 + self.rope = RotaryPositionEmbedding2D(frequency=rope_freq) if rope_freq > 0 else None + self.position_getter = PositionGetter() if self.rope is not None else None + + self.frame_blocks = nn.ModuleList( + [ + block_fn( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + proj_bias=proj_bias, + ffn_bias=ffn_bias, + init_values=init_values, + qk_norm=qk_norm, + rope=self.rope, + ) + for _ in range(depth) + ] + ) + + self.global_blocks = nn.ModuleList( + [ + block_fn( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + proj_bias=proj_bias, + ffn_bias=ffn_bias, + init_values=init_values, + qk_norm=qk_norm, + rope=self.rope, + ) + for _ in range(depth) + ] + ) + + self.depth = depth + self.aa_order = aa_order + self.patch_size = patch_size + self.aa_block_size = aa_block_size + + # Validate that depth is divisible by aa_block_size + if self.depth % self.aa_block_size != 0: + raise ValueError(f"depth ({depth}) must be divisible by aa_block_size ({aa_block_size})") + + self.aa_block_num = self.depth // self.aa_block_size + + + # Reference Learning Network + if self.reorder_by_learning_ref: + self.ref_aa_block_num = ref_aa_block_num + self.ref_frame_blocks = nn.ModuleList( + [ + block_fn( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + proj_bias=proj_bias, + ffn_bias=ffn_bias, + init_values=init_values, + qk_norm=qk_norm, + rope=self.rope, + ) + for _ in range(self.ref_aa_block_num) + ] + ) + + self.ref_global_blocks = nn.ModuleList( + [ + block_fn( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + proj_bias=proj_bias, + ffn_bias=ffn_bias, + init_values=init_values, + qk_norm=qk_norm, + rope=self.rope, + ) + for _ in range(self.ref_aa_block_num) + ] + ) + + # Note: We have two camera tokens, one for the first frame and one for the rest + # The same applies for register tokens + self.camera_token = nn.Parameter(torch.randn(1, 2, 1, embed_dim)) + self.register_token = nn.Parameter(torch.randn(1, 2, num_register_tokens, embed_dim)) + + if self.reorder_by_learning_ref: + # describe the covisibility of the current frame with other frames + self.covisibility_token = nn.Parameter(torch.randn(1, 1, 1, embed_dim)) + + # The patch tokens start after the camera and register tokens + self.patch_start_idx = 1 + num_register_tokens + + # Initialize parameters with small values + nn.init.normal_(self.camera_token, std=1e-6) + nn.init.normal_(self.register_token, std=1e-6) + if self.reorder_by_learning_ref: + nn.init.normal_(self.covisibility_token, std=1e-6) + + + # Register normalization constants as buffers + for name, value in (("_resnet_mean", _RESNET_MEAN), ("_resnet_std", _RESNET_STD)): + self.register_buffer(name, torch.FloatTensor(value).view(1, 1, 3, 1, 1), persistent=False) + + self.use_reentrant = False # hardcoded to False + + def __build_patch_embed__( + self, + patch_embed, + img_size, + patch_size, + num_register_tokens, + interpolate_antialias=True, + interpolate_offset=0.0, + block_chunks=0, + init_values=1.0, + embed_dim=1024, + ): + """ + Build the patch embed layer. If 'conv', we use a + simple PatchEmbed conv layer. Otherwise, we use a vision transformer. + """ + + if "conv" in patch_embed: + self.patch_embed = PatchEmbed(img_size=img_size, patch_size=patch_size, in_chans=3, embed_dim=embed_dim) + else: + vit_models = { + "dinov2_vitl14_reg": vit_large, + "dinov2_vitb14_reg": vit_base, + "dinov2_vits14_reg": vit_small, + "dinov2_vitg2_reg": vit_giant2, + } + + self.patch_embed = vit_models[patch_embed]( + img_size=img_size, + patch_size=patch_size, + num_register_tokens=num_register_tokens, + interpolate_antialias=interpolate_antialias, + interpolate_offset=interpolate_offset, + block_chunks=block_chunks, + init_values=init_values, + ) + + # Disable gradient updates for mask token + if hasattr(self.patch_embed, "mask_token"): + # self.patch_embed.mask_token.requires_grad_(False) + del self.patch_embed.mask_token + + + + # covisibility head + if self.reorder_by_learning_ref: + self.token_norm = nn.LayerNorm(embed_dim * 2) + self.covisibility_head = Mlp(in_features=embed_dim * 2, hidden_features=embed_dim * 2 // 2, out_features=1, drop=0) + + def forward(self, images: torch.Tensor) -> Tuple[List[torch.Tensor], int]: + """ + Args: + images (torch.Tensor): Input images with shape [B, S, 3, H, W], in range [0, 1]. + B: batch size, S: sequence length, 3: RGB channels, H: height, W: width + + Returns: + (list[torch.Tensor], int): + The list of outputs from the attention blocks, + and the patch_start_idx indicating where patch tokens begin. + """ + B, S, C_in, H, W = images.shape + + if C_in != 3: + raise ValueError(f"Expected 3 input channels, got {C_in}") + + # Normalize images and reshape for patch embed + images = (images - self._resnet_mean) / self._resnet_std + + # Reshape to [B*S, C, H, W] for patch embedding + images = images.view(B * S, C_in, H, W) + patch_tokens = self.patch_embed(images) + + if isinstance(patch_tokens, dict): + patch_tokens = patch_tokens["x_norm_patchtokens"] + + _, P, C = patch_tokens.shape + + ################# ref learning + covisibility_scores = None + ref_idx = None + if self.reorder_by_learning_ref: + # expand covisibility token to match batch size and sequence length + covisibility_token = self.covisibility_token.expand(B, S, 1, C).view(B * S, 1, C).contiguous() + # Concatenate covisibility token with patch tokens + covisibility_patch_tokens = torch.cat([covisibility_token, patch_tokens], dim=1) # [BS,1+HW,C] + + covisibility_pos = None + if self.rope is not None: + covisibility_pos = self.position_getter(B * S, H // self.patch_size, W // self.patch_size, device=images.device) + + # do not use position embedding for special covisibility_token + # so set pos to 0 for the special tokens + covisibility_pos = covisibility_pos + 1 + covisibility_pos_special = torch.zeros(B * S, 1, 2).to(images.device).to(covisibility_pos.dtype) + covisibility_pos = torch.cat([covisibility_pos_special, covisibility_pos], dim=1) # [BS, 1+HW, 2] + + + # update P because we added special tokens + _, P_covis, C_covis = covisibility_patch_tokens.shape + + frame_idx = 0 + global_idx = 0 + output_list = [] + + for ref_block_i in range(self.ref_aa_block_num): + for attn_type in self.aa_order: + if attn_type == "frame": + covisibility_patch_tokens, frame_idx, frame_intermediates = self._ref_process_frame_attention( + covisibility_patch_tokens, B, S, P_covis, C_covis, frame_idx, pos=covisibility_pos + ) + elif attn_type == "global": + covisibility_patch_tokens, global_idx, global_intermediates = self._ref_process_global_attention( + covisibility_patch_tokens, B, S, P_covis, C_covis, global_idx, pos=covisibility_pos + ) + else: + raise ValueError(f"Unknown attention type: {attn_type}") + + for i in range(len(frame_intermediates)): + # concat frame and global intermediates, [B x S x P x 2C] + concat_inter = torch.cat([frame_intermediates[-1], global_intermediates[-1]], dim=-1) + output_list.append(concat_inter) + + last_covisibility_patch_tokens = output_list[-1][:,:,0,:] # [B, S, C] + # normalize + last_covisibility_patch_tokens = self.token_norm(last_covisibility_patch_tokens) + + covisibility_scores = self.covisibility_head(last_covisibility_patch_tokens).squeeze(-1) # [B, S] + # # cos + # feat_norm = F.normalize(covisibility_features, p=2, dim=-1, eps=1e-8) # [B, S, D] + # covisibility_scores = feat_norm @ feat_norm.transpose(-1, -2) + + + ref_idx = covisibility_scores.argmax(-1) # [B, S] -> [B] + patch_tokens = patch_tokens.view(B,S,P,C) + patch_tokens = reorder_by_reference(patch_tokens, ref_idx) + patch_tokens = patch_tokens.view(B*S,P,C).contiguous() + + #################### + # Expand camera and register tokens to match batch size and sequence length + camera_token = slice_expand_and_flatten(self.camera_token, B, S) + register_token = slice_expand_and_flatten(self.register_token, B, S) + # Concatenate special tokens with patch tokens + tokens = torch.cat([camera_token, register_token, patch_tokens], dim=1) # [BS,1+4+HW,C] + + pos = None + if self.rope is not None: + pos = self.position_getter(B * S, H // self.patch_size, W // self.patch_size, device=images.device) + + if self.patch_start_idx > 0: + # do not use position embedding for special tokens (camera and register tokens) + # so set pos to 0 for the special tokens + pos = pos + 1 + pos_special = torch.zeros(B * S, self.patch_start_idx, 2).to(images.device).to(pos.dtype) + pos = torch.cat([pos_special, pos], dim=1) # [BS, 1+4+HW, 2] + + + + # update P because we added special tokens + _, P, C = tokens.shape + + frame_idx = 0 + global_idx = 0 + output_list = [] + + for block_i in range(self.aa_block_num): + for attn_type in self.aa_order: + if attn_type == "frame": + tokens, frame_idx, frame_intermediates = self._process_frame_attention( + tokens, B, S, P, C, frame_idx, pos=pos + ) + elif attn_type == "global": + tokens, global_idx, global_intermediates = self._process_global_attention( + tokens, B, S, P, C, global_idx, pos=pos + ) + else: + raise ValueError(f"Unknown attention type: {attn_type}") + + for i in range(len(frame_intermediates)): + concat_inter = torch.cat([frame_intermediates[i], global_intermediates[i]], dim=-1) + if (not self.training ) and (self.save_inference_memory) and (block_i not in [4,11,17,23]): + # only save the useful indices of intermediates + output_list.append(torch.tensor(0)) + else: + # concat frame and global intermediates, [B x S x P x 2C] + output_list.append(concat_inter) + + del concat_inter + del frame_intermediates + del global_intermediates + return output_list, self.patch_start_idx, covisibility_scores, ref_idx + + def _process_frame_attention(self, tokens, B, S, P, C, frame_idx, pos=None): + """ + Process frame attention blocks. We keep tokens in shape (B*S, P, C). + """ + # If needed, reshape tokens or positions: + if tokens.shape != (B * S, P, C): + tokens = tokens.view(B, S, P, C).view(B * S, P, C) + + if pos is not None and pos.shape != (B * S, P, 2): + pos = pos.view(B, S, P, 2).view(B * S, P, 2) + + intermediates = [] + + # by default, self.aa_block_size=1, which processes one block at a time + for _ in range(self.aa_block_size): + if self.training: + tokens = checkpoint(self.frame_blocks[frame_idx], tokens, pos, use_reentrant=self.use_reentrant) + else: + tokens = self.frame_blocks[frame_idx](tokens, pos=pos) + frame_idx += 1 + intermediates.append(tokens.view(B, S, P, C)) + + return tokens, frame_idx, intermediates + + def _process_global_attention(self, tokens, B, S, P, C, global_idx, pos=None): + """ + Process global attention blocks. We keep tokens in shape (B, S*P, C). + """ + if tokens.shape != (B, S * P, C): + tokens = tokens.view(B, S, P, C).view(B, S * P, C) + + if pos is not None and pos.shape != (B, S * P, 2): + pos = pos.view(B, S, P, 2).view(B, S * P, 2) + + intermediates = [] + + + # by default, self.aa_block_size=1, which processes one block at a time + for _ in range(self.aa_block_size): + if self.training: + tokens = checkpoint(self.global_blocks[global_idx], tokens, pos, use_reentrant=self.use_reentrant) + else: + tokens = self.global_blocks[global_idx](tokens, pos=pos) + global_idx += 1 + + intermediates.append(tokens.view(B, S, P, C)) + + + + return tokens, global_idx, intermediates + + def _ref_process_frame_attention(self, tokens, B, S, P, C, frame_idx, pos=None): + """ + Process frame attention blocks. We keep tokens in shape (B*S, P, C). + """ + # If needed, reshape tokens or positions: + if tokens.shape != (B * S, P, C): + tokens = tokens.view(B, S, P, C).view(B * S, P, C) + + if pos is not None and pos.shape != (B * S, P, 2): + pos = pos.view(B, S, P, 2).view(B * S, P, 2) + + intermediates = [] + + # by default, self.aa_block_size=1, which processes one block at a time + for _ in range(self.aa_block_size): + if self.training: + tokens = checkpoint(self.ref_frame_blocks[frame_idx], tokens, pos, use_reentrant=self.use_reentrant) + else: + tokens = self.ref_frame_blocks[frame_idx](tokens, pos=pos) + frame_idx += 1 + intermediates.append(tokens.view(B, S, P, C)) + + return tokens, frame_idx, intermediates + + def _ref_process_global_attention(self, tokens, B, S, P, C, global_idx, pos=None): + """ + Process global attention blocks. We keep tokens in shape (B, S*P, C). + """ + if tokens.shape != (B, S * P, C): + tokens = tokens.view(B, S, P, C).view(B, S * P, C) + + if pos is not None and pos.shape != (B, S * P, 2): + pos = pos.view(B, S, P, 2).view(B, S * P, 2) + + intermediates = [] + + + # by default, self.aa_block_size=1, which processes one block at a time + for _ in range(self.aa_block_size): + if self.training: + tokens = checkpoint(self.ref_global_blocks[global_idx], tokens, pos, use_reentrant=self.use_reentrant) + else: + tokens = self.ref_global_blocks[global_idx](tokens, pos=pos) + global_idx += 1 + + intermediates.append(tokens.view(B, S, P, C)) + + + + return tokens, global_idx, intermediates + +def slice_expand_and_flatten(token_tensor, B, S): + """ + Processes specialized tokens with shape (1, 2, X, C) for multi-frame processing: + 1) Uses the first position (index=0) for the first frame only + 2) Uses the second position (index=1) for all remaining frames (S-1 frames) + 3) Expands both to match batch size B + 4) Concatenates to form (B, S, X, C) where each sequence has 1 first-position token + followed by (S-1) second-position tokens + 5) Flattens to (B*S, X, C) for processing + + Returns: + torch.Tensor: Processed tokens with shape (B*S, X, C) + """ + + # Slice out the "query" tokens => shape (1, 1, ...) + query = token_tensor[:, 0:1, ...].expand(B, 1, *token_tensor.shape[2:]) + # Slice out the "other" tokens => shape (1, S-1, ...) + others = token_tensor[:, 1:, ...].expand(B, S - 1, *token_tensor.shape[2:]) + # Concatenate => shape (B, S, ...) + combined = torch.cat([query, others], dim=1) + + # Finally flatten => shape (B*S, ...) + combined = combined.view(B * S, *combined.shape[2:]) + return combined diff --git a/argus/models/argus.py b/argus/models/argus.py new file mode 100644 index 0000000000000000000000000000000000000000..731727a30d32995a64a85f4620b665162847521b --- /dev/null +++ b/argus/models/argus.py @@ -0,0 +1,234 @@ +import torch +import torch.nn as nn +from typing import Optional, Dict +from huggingface_hub import PyTorchModelHubMixin + +# Import model components +from argus.models.aggregator import Aggregator +from argus.heads.camera_head import CameraHead +from argus.heads.dpt_head import DPTHead +from argus.heads.utils import reorder_by_reference + + +class Argus(nn.Module, PyTorchModelHubMixin): + """ + Argus multi-task vision model for camera pose estimation, depth prediction, and 3D points. + + Integrates an aggregator backbone with task-specific heads for: + - Camera pose encoding + - Depth map prediction + - 3D camera/rotated/world point prediction + + Args: + img_size: Input image size (height/width, assumes square) (default: 518) + patch_size: Patch size for vision transformer backbone (default: 14) + embed_dim: Embedding dimension for transformer features (default: 1024) + enable_camera: Enable camera pose estimation head (default: True) + enable_depth: Enable depth prediction head (default: True) + enable_cam_point: Enable camera coordinate 3D point prediction head (default: False) + enable_rotated_point: Enable rotated 3D point prediction head (default: False) + enable_point: Enable world coordinate 3D point prediction head (default: False, Please do not set it to True during training) + + Note: + All heads share the same aggregated transformer features from the Aggregator backbone. + Each DPT-based head outputs both predictions and confidence scores. + """ + def __init__( + self, + img_size: int = 518, + patch_size: int = 14, + embed_dim: int = 1024, + enable_camera: bool = True, + enable_depth: bool = True, + enable_cam_point: bool = False, + enable_rotated_point: bool = False, + enable_point: bool = False, + reorder_by_learning_ref: bool = True, + restore_metric_scale: bool = False + ) -> None: + super().__init__() + # For inference + self.restore_metric_scale = restore_metric_scale + self.reorder_by_learning_ref = reorder_by_learning_ref + + # Backbone and geometry transformer + self.aggregator = Aggregator( + img_size=img_size, + patch_size=patch_size, + embed_dim=embed_dim, + reorder_by_learning_ref=reorder_by_learning_ref, + ) + + # Task-specific prediction heads (lazy initialization based on flags) + self.camera_head: Optional[CameraHead] = CameraHead(dim_in=2 * embed_dim) if enable_camera else None + self.depth_head: Optional[DPTHead] = DPTHead( + dim_in=2 * embed_dim, + output_dim=2, + activation="exp", + conf_activation="expp1" + ) if enable_depth else None + + # 3D point prediction heads (shared architecture, different output semantics) + self.cam_point_head: Optional[DPTHead] = DPTHead( + dim_in=2 * embed_dim, + output_dim=4, + activation="inv_log", + conf_activation="expp1" + ) if enable_cam_point else None + + self.rotated_point_head: Optional[DPTHead] = DPTHead( + dim_in=2 * embed_dim, + output_dim=4, + activation="inv_log", + conf_activation="expp1" + ) if enable_rotated_point else None + + self.point_head: Optional[DPTHead] = DPTHead( + dim_in=2 * embed_dim, + output_dim=4, + activation="inv_log", + conf_activation="expp1" + ) if enable_point else None + + def forward( + self, + images: torch.Tensor, + ) -> Dict[str, torch.Tensor]: + """ + Forward pass of the Argus model. + + Automatically adds batch dimension if missing and processes multi-task predictions. + + Args: + images: Input RGB images with shape: + - [S, 3, H, W] (sequence without batch) or + - [B, S, 3, H, W] (batch of sequences) + Values in range [0, 1], where: + - B: batch size + - S: sequence length (number of frames) + - 3: RGB channels + - H/W: image height/width (matches img_size) + + Returns: + Dictionary of model predictions with task-specific outputs: + Common outputs: + - covisibility_scores: Covisibility scores from aggregator (shape varies) + - ref_idx: Reference frame indices (shape varies) + + Camera head outputs (if enabled): + - pose_enc: Final camera pose encoding [B, S, 9] + - pose_enc_list: List of pose encodings from all iterations [List[torch.Tensor]] + + Depth head outputs (if enabled): + - depth: Predicted depth maps [B, S, H, W, 1] + - depth_conf: Depth prediction confidence [B, S, H, W] + + Camera point head outputs (if enabled): + - cam_points: 3D camera coordinates per pixel [B, S, H, W, 3] + - cam_points_conf: Camera point confidence [B, S, H, W] + + Rotated point head outputs (if enabled): + - rotated_points: Rotated 3D coordinates per pixel [B, S, H, W, 3] + - rotated_points_conf: Rotated point confidence [B, S, H, W] + + World point head outputs (if enabled): + - world_points: 3D world coordinates per pixel [B, S, H, W, 3] + - world_points_conf: World point confidence [B, S, H, W] + + Inference-only outputs (not training): + - images: Original input images (for visualization) [B, S, 3, H, W] + """ + # Add batch dimension if missing (handle [S,3,H,W] -> [1,S,3,H,W]) + if len(images.shape) == 4: + images = images.unsqueeze(0) + + # Extract aggregated features from backbone + ( + aggregated_tokens_list, # List of aggregated transformer tokens across iterations + patch_start_idx, # Patch start indices for feature reconstruction + covisibility_scores, # Covisibility scores between frames + ref_idx # Reference frame indices + ) = self.aggregator(images) + + # Initialize prediction dictionary + predictions: Dict[str, torch.Tensor] = {} + + # Disable mixed precision for precise prediction calculations + with torch.amp.autocast("cuda", enabled=False): + # Add aggregator outputs to predictions + if covisibility_scores is not None: + predictions["covisibility_scores"] = covisibility_scores + if ref_idx is not None: + predictions["ref_idx"] = ref_idx + + # Camera pose prediction (if enabled) + if self.camera_head is not None: + pose_enc_list = self.camera_head(aggregated_tokens_list) + predictions["pose_enc"] = pose_enc_list[-1] # Use final iteration encoding + predictions["pose_enc_list"] = pose_enc_list # Mutil-layer supervision + + # Depth prediction (if enabled) + if self.depth_head is not None: + depth, depth_conf = self.depth_head( + aggregated_tokens_list, + images=images, + patch_start_idx=patch_start_idx + ) + predictions["depth"] = depth + predictions["depth_conf"] = depth_conf + + # Camera 3D point prediction (if enabled) + if self.cam_point_head is not None: + cam_pts3d, cam_pts3d_conf = self.cam_point_head( + aggregated_tokens_list, + images=images, + patch_start_idx=patch_start_idx + ) + predictions["cam_points"] = cam_pts3d + predictions["cam_points_conf"] = cam_pts3d_conf + + # Rotated 3D point prediction (if enabled) + if self.rotated_point_head is not None: + rotated_pts3d, rotated_pts3d_conf = self.rotated_point_head( + aggregated_tokens_list, + images=images, + patch_start_idx=patch_start_idx + ) + predictions["rotated_points"] = rotated_pts3d + predictions["rotated_points_conf"] = rotated_pts3d_conf + + # World 3D point prediction (if enabled) + if self.point_head is not None: + world_pts3d, world_pts3d_conf = self.point_head( + aggregated_tokens_list, + images=images, + patch_start_idx=patch_start_idx + ) + predictions["world_points"] = world_pts3d + predictions["world_points_conf"] = world_pts3d_conf + + + # Store input images for visualization during inference (skip in training) + if not self.training: + predictions["images"] = images + if "ref_idx" in predictions: + ref_idx = predictions["ref_idx"].detach() + # Reorder all spatial/temporal data (exclude adjacency matrix and IDs) + predictions["images"] = reorder_by_reference(predictions["images"], ref_idx) + + if self.restore_metric_scale: + # Restore metric scale + abs_scale = 10.0 + if self.camera_head is not None: + predictions["pose_enc"][...,:3] *= abs_scale + if self.depth_head is not None: + predictions["depth"] *= abs_scale + if self.cam_point_head is not None: + predictions["cam_points"] *= abs_scale + if self.rotated_point_head is not None: + predictions["rotated_points"] *= abs_scale + if self.point_head is not None: + predictions["world_points"] *= abs_scale + + + return predictions \ No newline at end of file diff --git a/argus/utils/__init__.py b/argus/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7b3cdba8f0417592fec0c4b198244c248ea0130f --- /dev/null +++ b/argus/utils/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2026 Realsee. All rights reserved. +# Licensed under the Apache License, Version 2.0. diff --git a/argus/utils/data_io.py b/argus/utils/data_io.py new file mode 100644 index 0000000000000000000000000000000000000000..587842726cd8cf2c21eab7d38f835d73ba66d895 --- /dev/null +++ b/argus/utils/data_io.py @@ -0,0 +1,152 @@ +# Copyright 2026 Realsee. All rights reserved. +# Licensed under the Apache License, Version 2.0. + +""" +Shared I/O and preprocessing utilities for panoramic image data. + +These functions are used by both evaluation and training pipelines. +""" + +import os + +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" +import cv2 +import numpy as np + + +def read_image_cv2_360(path: str, rgb: bool = True, shape=(560, 280)) -> np.ndarray: + """Read and resize a 360 panorama image. + + Args: + path: Path to the image file. + rgb: If True, convert BGR to RGB (default: True). + shape: Target (width, height) tuple. + + Returns: + Image as numpy array with shape (H, W, 3). + """ + img = cv2.imread(path) + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + if img.shape[1] != shape[0]: + img = cv2.resize(img, shape, interpolation=cv2.INTER_AREA) + return img + + +def read_depth_360(path: str, depth_scale=5000.0, shape=(560, 280)) -> np.ndarray: + """Read and normalize a 360 depth map. + + Args: + path: Path to the depth image file. + depth_scale: Scale factor to convert raw depth to meters. + shape: Target (width, height) tuple. + + Returns: + Depth map as float32 numpy array with shape (H, W). + """ + d = cv2.imread(path, cv2.IMREAD_UNCHANGED) + if d.shape[1] != shape[0]: + d = cv2.resize(d, shape, interpolation=cv2.INTER_NEAREST) + d = d.astype(np.float32) / depth_scale + return d + + +def random_rotate_theta(W=560, max_shift_percent=0.5): + """Generate a random rotation angle for panorama augmentation. + + Args: + W: Panorama width in pixels. + max_shift_percent: Maximum horizontal shift as fraction of width. + + Returns: + Rotation angle in radians. + """ + max_shift = int(W * max_shift_percent) + shift_pixels = np.random.randint(-max_shift, max_shift + 1) + theta = (shift_pixels * 2 * np.pi) / W + return theta + + +def rotate_y(theta): + """Create a 3x3 rotation matrix around the Y-axis. + + Args: + theta: Rotation angle in radians. + + Returns: + 3x3 rotation matrix as float64 numpy array. + """ + cos_theta = np.cos(theta) + sin_theta = np.sin(theta) + return np.array( + [[cos_theta, 0, -sin_theta], [0, 1, 0], [sin_theta, 0, cos_theta]], + dtype=np.float64, + ) + + +def pano_depth_to_points(depth_map, pano_shape=(560, 280), crop=True, crop_ratio=0.15): + """Convert a panorama depth map to 3D point cloud. + + Args: + depth_map: 2D depth map (H, W) or flattened array. + pano_shape: Original panorama (width, height) tuple. + crop: Whether the depth map has been vertically cropped. + crop_ratio: Crop ratio applied to top and bottom. + + Returns: + Point cloud as numpy array with shape (N, 3). + """ + w, h = pano_shape + + if not crop: + px = np.tile(np.arange(w), int(h)) + py = np.arange(0, int(h)).repeat(w) + else: + px = np.tile(np.arange(w), int(h * (1 - 2 * crop_ratio))) + py = np.arange(int(crop_ratio * h), int((1 - crop_ratio) * h)).repeat(w) + + dist = depth_map.reshape(-1) + + lat = (py / h - 0.5) * np.pi + long = (px / w - 0.5) * np.pi * 2.0 + + y = dist * np.sin(lat) + tmp = dist * np.cos(lat) + x = tmp * np.sin(long) + z = tmp * np.cos(long) + + point_map = np.concatenate([i.reshape(-1, 1) for i in (x, y, z)], axis=-1) + + return point_map # (h*w, 3) + + +def crop_panorama(pano, crop_ratio=0.15): + """Crop the top and bottom of a panorama by a given ratio. + + Args: + pano: Input panorama array with shape (H, W, ...). + crop_ratio: Fraction to crop from top and bottom. + + Returns: + Cropped panorama. + """ + H, W = pano.shape[:2] + crop_H_top = int(crop_ratio * H) + crop_H_bottom = H - int(crop_ratio * H) + crop_pano = pano[crop_H_top:crop_H_bottom, ...] + return crop_pano + + +def rotate_panorama(panorama, theta): + """Horizontally rotate a panorama by shifting pixels. + + Args: + panorama: Input panorama array with shape (H, W, ...). + theta: Rotation angle in radians. + + Returns: + Shifted panorama. + """ + H, W = panorama.shape[:2] + shift_pixels = int((theta * W) / (2 * np.pi)) + shifted = np.roll(panorama, shift_pixels, axis=1) + return shifted diff --git a/argus/utils/geometry.py b/argus/utils/geometry.py new file mode 100644 index 0000000000000000000000000000000000000000..c2d92f4bdf1d886f01d960c291cb511c0980ab80 --- /dev/null +++ b/argus/utils/geometry.py @@ -0,0 +1,201 @@ +import torch +import numpy as np + + +def closed_form_inverse_se3(se3, R=None, T=None): + """ + Compute the inverse of each 4x4 (or 3x4) SE3 matrix in a batch. + + If `R` and `T` are provided, they must correspond to the rotation and translation + components of `se3`. Otherwise, they will be extracted from `se3`. + + Args: + se3: Nx4x4 or Nx3x4 array or tensor of SE3 matrices. + R (optional): Nx3x3 array or tensor of rotation matrices. + T (optional): Nx3x1 array or tensor of translation vectors. + + Returns: + Inverted SE3 matrices with the same type and device as `se3`. + + Shapes: + se3: (N, 4, 4) + R: (N, 3, 3) + T: (N, 3, 1) + """ + # Check if se3 is a numpy array or a torch tensor + is_numpy = isinstance(se3, np.ndarray) + + # Validate shapes + if se3.shape[-2:] != (4, 4) and se3.shape[-2:] != (3, 4): + raise ValueError(f"se3 must be of shape (N,4,4), got {se3.shape}.") + + # Extract R and T if not provided + if R is None: + R = se3[:, :3, :3] # (N,3,3) + if T is None: + T = se3[:, :3, 3:] # (N,3,1) + + # Transpose R + if is_numpy: + # Compute the transpose of the rotation for NumPy + R_transposed = np.transpose(R, (0, 2, 1)) + # -R^T t for NumPy + top_right = -np.matmul(R_transposed, T) + inverted_matrix = np.tile(np.eye(4), (len(R), 1, 1)) + else: + R_transposed = R.transpose(1, 2) # (N,3,3) + top_right = -torch.bmm(R_transposed, T) # (N,3,1) + inverted_matrix = torch.eye(4, 4)[None].repeat(len(R), 1, 1) + inverted_matrix = inverted_matrix.to(R.dtype).to(R.device) + + inverted_matrix[:, :3, :3] = R_transposed + inverted_matrix[:, :3, 3:] = top_right + + return inverted_matrix + + +def pano_depth_to_points(depth_map, original_pano_shape=(560, 280), crop_ratio=0.15): + """ + Convert batched cropped panoramic depth maps to 3D point clouds (PyTorch implementation). + Assumption: Input depth maps are already cropped by crop_ratio on top and bottom. + + Args: + depth_map (torch.Tensor): Input cropped depth map, shape [B, S, H_crop, W, 1] + original_pano_shape (tuple): Original uncropped panorama size (W_ori, H_ori), default (560, 280) + crop_ratio (float): Crop ratio of original panorama (top and bottom respectively), default 0.15 + + Returns: + torch.Tensor: 3D point cloud with shape [B, S, H_crop, W, 3] + """ + # Validate input shape + assert depth_map.dim() == 5 and depth_map.shape[-1] == 1, \ + f"Input must be [B, S, H_crop, W, 1], got {depth_map.shape}" + + B, S, H_crop, W, _ = depth_map.shape + W_ori, H_ori = original_pano_shape + device = depth_map.device # Align tensor device automatically + + # Generate pixel grid coordinates (H_crop, W) + px_grid, py_grid = torch.meshgrid( + torch.arange(W, device=device), + torch.arange(H_crop, device=device), + indexing='xy' # Consistent with numpy's meshgrid + ) + + # Restore to original panorama y-coordinates (compensate for cropping) + crop_top = int(crop_ratio * H_ori) + py_ori = py_grid + crop_top + + # Compute spherical coordinates (lat: latitude, long: longitude) + lat = (py_ori / H_ori - 0.5) * torch.pi + long = (px_grid / W_ori - 0.5) * 2 * torch.pi + + # Remove channel dim and compute 3D Cartesian coordinates + dist = depth_map.squeeze(-1) # [B, S, H_crop, W] + y = dist * torch.sin(lat) + tmp = dist * torch.cos(lat) + x = tmp * torch.sin(long) + z = tmp * torch.cos(long) + + # Concatenate to form 3D point cloud + point_cloud = torch.stack([x, y, z], dim=-1) + + return point_cloud + + +def points_to_pano_depth(points): + """ + Convert 3D point cloud back to ray panoramic depth map. + Ignore the error in direction. + + Args: + points (torch.Tensor): Input 3D point cloud, shape [B, S, H, W, 3] + + Returns: + torch.Tensor: panoramic depth map, shape [B, S, H, W, 1] + """ + # Validate input shape and fill mode + assert points.dim() == 5 and points.shape[-1] == 3, \ + f"Input point cloud must be [B, S, H, W, 3], got {points.shape}" + + # Compute radial depth (dist = sqrt(x² + y² + z²)) + dist = torch.norm(points, dim=-1, keepdim=True) # [B, S, H, W, 1] + + return dist + + +def camera_points_to_rotated_points(cam_points, R): + """ + Rotate batched panoramic camera point clouds with corresponding rotation matrices. + + Args: + cam_points (torch.Tensor): Input camera 3D point cloud, shape [B, S, H, W, 3] + R (torch.Tensor): Corresponding rotation matrices, shape [B, S, 3, 3] + + Returns: + torch.Tensor: Rotated 3D point cloud, shape [B, S, H, W, 3] (same as input cam_points) + """ + # Validate input shapes and dimensions matching + assert cam_points.dim() == 5 and cam_points.shape[-1] == 3, \ + f"Camera points must be [B, S, H, W, 3], got {cam_points.shape}" + assert R.dim() == 4 and R.shape[2:] == (3, 3), \ + f"Rotation matrices R must be [B, S, 3, 3], got {R.shape}" + assert cam_points.shape[:2] == R.shape[:2], \ + f"Batch/Sequence dim mismatch: cam_points {cam_points.shape[:2]} vs R {R.shape[:2]}" + + # Expand dimensions for broadcasting (align spatial dimensions H, W) + cam_points_expanded = cam_points.unsqueeze(-1) # [B, S, H, W, 3, 1] + R_expanded = R.unsqueeze(2).unsqueeze(2) # [B, S, 1, 1, 3, 3] + + # Batch matrix multiplication: R @ p (rotation operation) + rotated_points_expanded = torch.matmul(R_expanded, cam_points_expanded) + + # Squeeze redundant dimension to recover original shape + rotated_points = rotated_points_expanded.squeeze(-1) + + return rotated_points + + +def rotated_points_to_world_points(rotated_points, t): + """ + Transform rotated camera points to world coordinates by adding translation vector. + + Args: + rotated_points (torch.Tensor): Rotated 3D point cloud, shape [B, S, H, W, 3] + t (torch.Tensor): Translation vector, shape [B, S, 3] (per batch-sequence translation) + + Returns: + torch.Tensor: World-coordinate 3D point cloud, shape [B, S, H, W, 3] (same as input) + """ + # Validate input shapes and dimension matching + assert rotated_points.dim() == 5 and rotated_points.shape[-1] == 3, \ + f"Rotated points must be [B, S, H, W, 3], got {rotated_points.shape}" + assert t.dim() == 3 and t.shape[-1] == 3, \ + f"Translation t must be [B, S, 3], got {t.shape}" + assert rotated_points.shape[:2] == t.shape[:2], \ + f"Batch/Sequence dim mismatch: rotated_points {rotated_points.shape[:2]} vs t {t.shape[:2]}" + + # Expand translation dimensions for broadcasting with spatial dimensions (H, W) + # t: [B, S, 3] -> [B, S, 1, 1, 3] (broadcast to H and W) + t_expanded = t.unsqueeze(2).unsqueeze(2) + + # Add translation (broadcasting automatically applies t to all H×W points per B-S pair) + world_points = rotated_points + t_expanded + + return world_points + + + +def unproject_depth_to_world_points(depth, extrinsic, size=560): + ''' + Args: + depth: [S, H, W, 1] + extrinsic: [S, 4, 4] + Returns: + world_points: [S, H, W, 3] + ''' + camera_points = pano_depth_to_points(depth, original_pano_shape=(size, size//2)) + rotated_points = camera_points_to_rotated_points(camera_points, extrinsic[:, :, :3, :3]) + world_points = rotated_points_to_world_points(rotated_points, extrinsic[:, :, :3, 3]) + + return world_points diff --git a/argus/utils/normalization.py b/argus/utils/normalization.py new file mode 100644 index 0000000000000000000000000000000000000000..bc0bda7dae900b7616e6357bf3037d3a4cbbd6a4 --- /dev/null +++ b/argus/utils/normalization.py @@ -0,0 +1,65 @@ +import torch +from typing import Optional, Tuple +from argus.utils.geometry import closed_form_inverse_se3 + +def cal_scale_by_points(points: torch.Tensor, point_masks: torch.Tensor) -> torch.Tensor: + # Calculate average distance of valid 3D points (batch-wise) + dist = points.norm(dim=-1) + dist_sum = (dist * point_masks).sum(dim=[1, 2, 3]) # Shape: [B,] + valid_count = point_masks.sum(dim=[1, 2, 3]) + avg_scale = (dist_sum / (valid_count + 1e-3)).clamp(min=1e-6, max=1e6) + return avg_scale + +def normalize_camera_extrinsics_and_points_batch( + extrinsics: torch.Tensor, + cam_points: torch.Tensor, + depths: torch.Tensor, + point_masks: torch.Tensor, + scale_mode: str = "none", +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + # Basic input validation + assert extrinsics.ndim == 4 and extrinsics.shape[2:] == (4, 4), \ + f"Extrinsics must be (B, S, 4, 4), got {extrinsics.shape}" + B, S = extrinsics.shape[:2] + device = extrinsics.device + + # Step 1: Transform all extrinsics to reference frame (1st frame of each batch) + ref_extrinsics = extrinsics[:,0,:,:] # (B, 4, 4) + ref_extr_inv = closed_form_inverse_se3(ref_extrinsics) + new_extrinsics = torch.matmul(ref_extr_inv.unsqueeze(1), extrinsics) # (B, S, 4, 4) world coordinate + + # Step 2: Clone tensors to avoid in-place modification + new_depths = depths.clone() + new_cam_points = cam_points.clone() + + # Step 3: Compute rotated/world points from new extrinsics + R_new = new_extrinsics[:, :, :3, :3] # (B, S, 3, 3) + t_new = new_extrinsics[:, :, :3, 3] # (B, S, 3) + new_rotated_points = torch.matmul(R_new.unsqueeze(2).unsqueeze(3), new_cam_points.unsqueeze(-1)).squeeze(-1) # (B,S,1,1,3,3) × (B,S,H,W,3,1) -> (B,S,H,W,3) + new_world_points = new_rotated_points + t_new.unsqueeze(2).unsqueeze(3) + + # Step 4: Apply scene scaling + if scale_mode == "avg_dist": + avg_scale = cal_scale_by_points(new_world_points, point_masks) # (B,) + # Reshape scale for broadcasting with different tensor shapes + scale_3d = avg_scale.view(-1, 1, 1) # For extrinsics (B, S, 4, 4) + scale_4d = avg_scale.view(-1, 1, 1, 1) # For depths (B, S, H, W) + scale_5d = avg_scale.view(-1, 1, 1, 1, 1) # For 3D points (B, S, H, W, 3) + new_extrinsics[:, :, :3, 3] /= scale_3d + new_depths /= scale_4d + new_cam_points /= scale_5d + new_rotated_points /= scale_5d + new_world_points /= scale_5d + elif scale_mode == "abs": + metric_scale = 10.0 + new_extrinsics[:, :, :3, 3] /= metric_scale + new_depths /= metric_scale + new_cam_points /= metric_scale + new_rotated_points /= metric_scale + new_world_points /= metric_scale + elif scale_mode == "none": + pass + else: + raise ValueError(f"Unknown scale_mode: {scale_mode}") + + return new_extrinsics, new_cam_points, new_rotated_points, new_world_points, new_depths \ No newline at end of file diff --git a/argus/utils/pose_enc.py b/argus/utils/pose_enc.py new file mode 100644 index 0000000000000000000000000000000000000000..11d19401e5e75c45818ebdf810c6c1b9d464f7f8 --- /dev/null +++ b/argus/utils/pose_enc.py @@ -0,0 +1,105 @@ +import torch +from typing import Tuple, Union +from .rotation import quat_to_mat, mat_to_quat + + +def extri_to_pose_encoding360( + extrinsics: torch.Tensor, + pose_encoding_type: Union[str, "absT_quaR"] = "absT_quaR" +) -> torch.Tensor: + """ + Convert camera extrinsic parameters to a compact pose encoding (absolute translation + quaternion rotation). + + Transforms OpenCV-style camera extrinsics (3x4 [R|t] matrix) into a flattened encoding format + suitable for machine learning tasks like pose prediction or representation learning. + + Args: + extrinsics: Camera extrinsic matrices with shape [B, S, 3, 4] or [B, S, 4, 4] + - B: Batch size + - S: Sequence length (number of frames) + - 3x4/4x4: Extrinsic matrix in OpenCV coordinate system (x-right, y-down, z-forward) + representing the transformation from world to camera space ([R|t] where R=3x3 rotation, t=3x1 translation) + pose_encoding_type: Type of pose encoding format (only "absT_quaR" supported): + - "absT_quaR": Absolute translation (3D) + quaternion rotation (4D) + + Returns: + Encoded pose tensor with shape [B, S, 7] + - [:3]: Absolute translation vector (T) in world coordinates + - [3:7]: Rotation represented as unit quaternion (quat) + """ + # Extract rotation matrix (R) and translation vector (T) from extrinsics + # Handle both 3x4 and 4x4 extrinsic matrix inputs + R = extrinsics[:, :, :3, :3] # [B, S, 3, 3] - rotation matrix + T = extrinsics[:, :, :3, 3] # [B, S, 3] - translation vector + + if pose_encoding_type == "absT_quaR": + # Convert rotation matrix to quaternion (4D) + quat = mat_to_quat(R) + + # Concatenate translation and quaternion to form compact pose encoding + pose_encoding = torch.cat([T, quat], dim=-1).float() + else: + raise NotImplementedError(f"Pose encoding type '{pose_encoding_type}' not supported. Only 'absT_quaR' is implemented.") + + return pose_encoding + + +def pose_encoding_to_extri360( + pose_encoding: torch.Tensor, + pose_encoding_type: Union[str, "absT_quaR"] = "absT_quaR" +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Convert compact pose encoding back to full camera extrinsic parameters (inverse of extri_to_pose_encoding360). + + Reconstructs the 4x4 homogeneous extrinsic matrix from the flattened pose encoding, + including extraction of confidence scores from the encoding's extra dimensions. + + Args: + pose_encoding: Encoded pose tensor with shape [B, S, 9] + - B: Batch size + - S: Sequence length (number of frames) + - [:3]: Absolute translation vector (T) + - [3:7]: Rotation quaternion (quat) + - [-2:]: Confidence scores for translation and rotation + pose_encoding_type: Type of pose encoding format (only "absT_quaR" supported): + - "absT_quaR": Absolute translation (3D) + quaternion rotation (4D) + + Returns: + Tuple containing: + 1. extrinsics: Reconstructed camera extrinsic matrices with shape [B, S, 4, 4] + (homogeneous matrix in OpenCV coordinate system: [R|t; 0 0 0 1]) + 2. conf: Confidence scores with shape [B, S, 2] + - [:, :, 0]: Translation confidence + - [:, :, 1]: Rotation confidence + + Raises: + NotImplementedError: If unsupported pose encoding type is provided + """ + if pose_encoding_type == "absT_quaR": + # Extract translation (T) and rotation quaternion (quat) from pose encoding + T = pose_encoding[..., :3] # [B, S, 3] - translation vector + quat = pose_encoding[..., 3:7] # [B, S, 4] - rotation quaternion + + # Convert quaternion back to rotation matrix (3x3) + R = quat_to_mat(quat) # [B, S, 3, 3] + + # Reconstruct 3x4 [R|t] matrix (rotation + translation) + extri_3x4 = torch.cat([R, T[..., None]], dim=-1) # [B, S, 3, 4] + + # Add homogeneous row [0, 0, 0, 1] to form 4x4 extrinsic matrix + batch_size, seq_len = extri_3x4.shape[:2] + homogenous_row = torch.tensor( + [0, 0, 0, 1], + device=extri_3x4.device, + dtype=extri_3x4.dtype + ).expand(batch_size, seq_len, 1, 4) # [B, S, 1, 4] + + # Combine to form 4x4 homogeneous extrinsic matrix + extrinsics = torch.cat((extri_3x4, homogenous_row), dim=2) # [B, S, 4, 4] + + # Extract confidence scores (last two dimensions of pose encoding) + conf = pose_encoding[..., -2:] # [B, S, 2] + + return extrinsics, conf + + raise NotImplementedError(f"Pose encoding type '{pose_encoding_type}' not supported. Only 'absT_quaR' is implemented.") \ No newline at end of file diff --git a/argus/utils/rotation.py b/argus/utils/rotation.py new file mode 100644 index 0000000000000000000000000000000000000000..19b7c6bd4454b4bd53dbbba3b0b0d387a7347800 --- /dev/null +++ b/argus/utils/rotation.py @@ -0,0 +1,118 @@ +import torch +import numpy as np +import torch.nn.functional as F + + +def quat_to_mat(quaternions: torch.Tensor) -> torch.Tensor: + """ + Quaternion Order: XYZW or say ijkr, scalar-last + + Convert rotations given as quaternions to rotation matrices. + Args: + quaternions: quaternions with real part last, + as tensor of shape (..., 4). + + Returns: + Rotation matrices as tensor of shape (..., 3, 3). + """ + # Normalize quaternions to unit length + quaternions = F.normalize(quaternions, dim=-1) + + i, j, k, r = torch.unbind(quaternions, -1) + two_s = 2.0 / (quaternions * quaternions).sum(-1) + + o = torch.stack( + ( + 1 - two_s * (j * j + k * k), + two_s * (i * j - k * r), + two_s * (i * k + j * r), + two_s * (i * j + k * r), + 1 - two_s * (i * i + k * k), + two_s * (j * k - i * r), + two_s * (i * k - j * r), + two_s * (j * k + i * r), + 1 - two_s * (i * i + j * j), + ), + -1, + ) + return o.reshape(quaternions.shape[:-1] + (3, 3)) + + +def mat_to_quat(matrix: torch.Tensor) -> torch.Tensor: + """ + Convert rotations given as rotation matrices to quaternions. + + Args: + matrix: Rotation matrices as tensor of shape (..., 3, 3). + + Returns: + quaternions with real part last, as tensor of shape (..., 4). + Quaternion Order: XYZW or say ijkr, scalar-last + """ + if matrix.size(-1) != 3 or matrix.size(-2) != 3: + raise ValueError(f"Invalid rotation matrix shape {matrix.shape}.") + + batch_dim = matrix.shape[:-2] + m00, m01, m02, m10, m11, m12, m20, m21, m22 = torch.unbind(matrix.reshape(batch_dim + (9,)), dim=-1) + + q_abs = _sqrt_positive_part( + torch.stack( + [1.0 + m00 + m11 + m22, 1.0 + m00 - m11 - m22, 1.0 - m00 + m11 - m22, 1.0 - m00 - m11 + m22], dim=-1 + ) + ) + + # we produce the desired quaternion multiplied by each of r, i, j, k + quat_by_rijk = torch.stack( + [ + torch.stack([q_abs[..., 0] ** 2, m21 - m12, m02 - m20, m10 - m01], dim=-1), + torch.stack([m21 - m12, q_abs[..., 1] ** 2, m10 + m01, m02 + m20], dim=-1), + torch.stack([m02 - m20, m10 + m01, q_abs[..., 2] ** 2, m12 + m21], dim=-1), + torch.stack([m10 - m01, m20 + m02, m21 + m12, q_abs[..., 3] ** 2], dim=-1), + ], + dim=-2, + ) + + # We floor here at 0.1 but the exact level is not important; if q_abs is small, + # the candidate won't be picked. + flr = torch.tensor(0.1).to(dtype=q_abs.dtype, device=q_abs.device) + quat_candidates = quat_by_rijk / (2.0 * q_abs[..., None].max(flr)) + + # if not for numerical problems, quat_candidates[i] should be same (up to a sign), + # forall i; we pick the best-conditioned one (with the largest denominator) + out = quat_candidates[F.one_hot(q_abs.argmax(dim=-1), num_classes=4) > 0.5, :].reshape(batch_dim + (4,)) + + # Convert from rijk to ijkr + out = out[..., [1, 2, 3, 0]] + + out = standardize_quaternion(out) + + return out + + +def _sqrt_positive_part(x: torch.Tensor) -> torch.Tensor: + """ + Returns torch.sqrt(torch.max(0, x)) + but with a zero subgradient where x is 0. + """ + ret = torch.zeros_like(x) + positive_mask = x > 0 + if torch.is_grad_enabled(): + ret[positive_mask] = torch.sqrt(x[positive_mask]) + else: + ret = torch.where(positive_mask, torch.sqrt(x), ret) + return ret + + +def standardize_quaternion(quaternions: torch.Tensor) -> torch.Tensor: + """ + Convert a unit quaternion to a standard form: one in which the real + part is non negative. + + Args: + quaternions: Quaternions with real part last, + as tensor of shape (..., 4). + + Returns: + Standardized quaternions as tensor of shape (..., 4). + """ + return torch.where(quaternions[..., 3:4] < 0, -quaternions, quaternions) diff --git a/assets/argus_logo.png b/assets/argus_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..5155208d1878a1f1925f0f8e2369e24368621c73 --- /dev/null +++ b/assets/argus_logo.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84ce97bea0cd15bca4e4655fa32b2e0a37b82c904963e3a79afd1f521ab89173 +size 108218 diff --git a/examples/far_4/0.jpg b/examples/far_4/0.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1327b41edcde5bb7fb43c4d7c9b02fe37273ae92 --- /dev/null +++ b/examples/far_4/0.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20ef38597f7839149d86ed294f7c878db26e9c895a36f40c48777e7d6c6d367f +size 102083 diff --git a/examples/far_4/1.jpg b/examples/far_4/1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..60f44e311257e80d9d4474b06187c545048d6971 --- /dev/null +++ b/examples/far_4/1.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2283054165720ace4163571dd8360d1f7df9dd94ca58e9b6f5c3d81f1c3e4220 +size 90741 diff --git a/examples/far_4/2.jpg b/examples/far_4/2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..55a3aa0ba66458e76083fb841d1713cbb743c768 --- /dev/null +++ b/examples/far_4/2.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8408263518733a6ee177771601977b8b99ebbb802155abf093bbfe3b033a2f69 +size 89894 diff --git a/examples/far_4/3.jpg b/examples/far_4/3.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4a87c43307847c2581a5df710cf7e7fc9088613e --- /dev/null +++ b/examples/far_4/3.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84694d2d5d75f82f5e9ecdc523d932e77179989990fc49f926021d26aea2877e +size 87241 diff --git a/examples/scene_00008/1757748389.jpg b/examples/scene_00008/1757748389.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2bc3eb3ba573f0379e2f7a901f13a3b01f191690 --- /dev/null +++ b/examples/scene_00008/1757748389.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30ff019ea8b37e6d7337b9b524bc91194ab8ff4e0ac9cffb33d985674b160740 +size 76807 diff --git a/examples/scene_00008/1757748429.jpg b/examples/scene_00008/1757748429.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a589251445740faf13a59bbe8f171ee03f26820b --- /dev/null +++ b/examples/scene_00008/1757748429.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd4f6ee92bd6f883078e7002b8d72e21100134ffa4bed9feb1b039cdd0002c19 +size 74557 diff --git a/examples/scene_00008/1757748477.jpg b/examples/scene_00008/1757748477.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6548e4c5b793befeeda1bad3ba3d5c8c14c45b5a --- /dev/null +++ b/examples/scene_00008/1757748477.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4912e213784d15c4415216b5a13288722e5ef509a349084f170fa3920dede8b +size 84383 diff --git a/examples/scene_00008/1757748528.jpg b/examples/scene_00008/1757748528.jpg new file mode 100644 index 0000000000000000000000000000000000000000..31a49f4624d7e8ac6a2a805f4caa0690aea6ade4 --- /dev/null +++ b/examples/scene_00008/1757748528.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47c0fecc84e80012faee6e9ec6d1ce8b7841934f8ddcdcbb8d1bed60b2d0fd7a +size 96323 diff --git a/examples/scene_00008/1757748562.jpg b/examples/scene_00008/1757748562.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5aa069247751ce46394aa1c94c54be4c9c166567 --- /dev/null +++ b/examples/scene_00008/1757748562.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c04a1704951d53733755b4c3444694325cd8576c87501bbd7aa679a8a61d5fc +size 105270 diff --git a/examples/scene_00008/1757748600.jpg b/examples/scene_00008/1757748600.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9c21e195c7d8367226070ad2c99c363d20506c94 --- /dev/null +++ b/examples/scene_00008/1757748600.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3db952b9929da9f0f608d4509de43c3aa10dd7e5882c8f64a091c4d553b8be5f +size 85718 diff --git a/examples/scene_00008/1757748638.jpg b/examples/scene_00008/1757748638.jpg new file mode 100644 index 0000000000000000000000000000000000000000..17a9900ab7ea4671cf618fdc27a1f7322665aad4 --- /dev/null +++ b/examples/scene_00008/1757748638.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ae1ddc73cc7cc85534b8ef0156baab344ee254e9bc7f02955ffc5c314a5e09c +size 75312 diff --git a/examples/scene_00008/1757748685.jpg b/examples/scene_00008/1757748685.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e45b35091e51c594314af91db5579da52a4cd7f2 --- /dev/null +++ b/examples/scene_00008/1757748685.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d879dd182f6b747cbff39bf1104dcd194880212c356f421db3fca2102621571 +size 119699 diff --git a/examples/scene_00008/1757748728.jpg b/examples/scene_00008/1757748728.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1b4d221abd7503060ba4484e91ce367267cc3caf --- /dev/null +++ b/examples/scene_00008/1757748728.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9ec8b77c76757f695aa7f6bfaf5a1dbecd328c4b231a1daa0fbad77dc596115 +size 75865 diff --git a/examples/scene_00008/1757748770.jpg b/examples/scene_00008/1757748770.jpg new file mode 100644 index 0000000000000000000000000000000000000000..232dff9a8ad39d7e7c34d95ce6e5916cca1cb367 --- /dev/null +++ b/examples/scene_00008/1757748770.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83535100f57b9ea09ee6b0e47da2ed827957dd2027c531727940a136cbb34719 +size 91940 diff --git a/examples/scene_00008/1757748817.jpg b/examples/scene_00008/1757748817.jpg new file mode 100644 index 0000000000000000000000000000000000000000..af7ed4b1f88bc727d8b9fc76d749fd73c4502bed --- /dev/null +++ b/examples/scene_00008/1757748817.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d5bd095592ee99cbe7f2646a60d86f65a1e57767b479f81981d853f816d0939 +size 87476 diff --git a/examples/scene_00008/1757748866.jpg b/examples/scene_00008/1757748866.jpg new file mode 100644 index 0000000000000000000000000000000000000000..01583e83a413b1c8fbf1513f56b9a1370b16fdc5 --- /dev/null +++ b/examples/scene_00008/1757748866.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:238d31b343b59023773ffd19a4f64e3663ceab052ee800c2b78548bd3484b9c4 +size 76254 diff --git a/examples/scene_00008/1757748907.jpg b/examples/scene_00008/1757748907.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e93ab1ff34d467b9d662fdd4aae4dfc30368f850 --- /dev/null +++ b/examples/scene_00008/1757748907.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd3e611f2473a49d071e51d75302c419ef63c48d15339adb2988f7ebd581f11d +size 80401 diff --git a/examples/scene_00008/1757748959.jpg b/examples/scene_00008/1757748959.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cf314a6d754863a7576bb61b439f56012c41be88 --- /dev/null +++ b/examples/scene_00008/1757748959.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b157fccda7e9b51e908ce990976b38374b2baa01de4d5437fecbc49f1751b067 +size 95163 diff --git a/examples/scene_00008/1757749004.jpg b/examples/scene_00008/1757749004.jpg new file mode 100644 index 0000000000000000000000000000000000000000..971c583dc0097a82c585b111c360dc62ba2054c8 --- /dev/null +++ b/examples/scene_00008/1757749004.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63f3042464745c37e7eb327308c93174714dfd99f40d644e716b04ee4dac5558 +size 90841 diff --git a/examples/scene_00008/1757749043.jpg b/examples/scene_00008/1757749043.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fff1ff8b740ff81837ba3c6f0864948fc47139ec --- /dev/null +++ b/examples/scene_00008/1757749043.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b2dbc11fc28999b640ecce24fa80e18fcc1758a02bf7240358eec2adcbb9096 +size 63835 diff --git a/examples/scene_00008/1757749091.jpg b/examples/scene_00008/1757749091.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ad8e3061b0308a515486e7d9359129e96285e259 --- /dev/null +++ b/examples/scene_00008/1757749091.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884e574ce5de6f976d17028e5b23798003d746312f504882555bc007ff818bf0 +size 75333 diff --git a/examples/scene_00008/1757749140.jpg b/examples/scene_00008/1757749140.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d83e82430525eb542a7792972fdbb2ed41c53fd3 --- /dev/null +++ b/examples/scene_00008/1757749140.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:198106f9757833a60a0fdb7d2894412c82749edfdcfa8c14a68d3817846d427c +size 101697 diff --git a/examples/scene_00025/1757403850.jpg b/examples/scene_00025/1757403850.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a5763ce205a3ad2e9271f8185bc843630ee3b93c --- /dev/null +++ b/examples/scene_00025/1757403850.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aad378a5ccc3ec7643ca5199cd239e971608439741fec701f7af9e07a8dbbeeb +size 58875 diff --git a/examples/scene_00025/1757403874.jpg b/examples/scene_00025/1757403874.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0a5402359b321c00f0dd7638bf03e0517dd5cd14 --- /dev/null +++ b/examples/scene_00025/1757403874.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea6a560b4b10ce97c1fa8b8cae8c11972b4c0ad4b77d7c2b7f25a547de39a562 +size 72228 diff --git a/examples/scene_00025/1757403900.jpg b/examples/scene_00025/1757403900.jpg new file mode 100644 index 0000000000000000000000000000000000000000..14c1dddc46c479e2ef74382f4d55c4c10f800550 --- /dev/null +++ b/examples/scene_00025/1757403900.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0beaa1ecd60f621df2718ad02a7b9a29aac70ef175034e3451cc7a138348068 +size 115120 diff --git a/examples/scene_00025/1757403924.jpg b/examples/scene_00025/1757403924.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3b47f559b745890b360a11a1b5901bad1b6f8297 --- /dev/null +++ b/examples/scene_00025/1757403924.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8692eb49902083426bb18df7b6df6e9e5de727e8f364692c3112549c1f8772c +size 64744 diff --git a/examples/scene_00025/1757403946.jpg b/examples/scene_00025/1757403946.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7988e9f9f4817cccb7b553234bc84a9eaf2e7430 --- /dev/null +++ b/examples/scene_00025/1757403946.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7de3a8a669b9581c78f90b6f9928d43ed75946e2790c8cf1a7c05643b891e8a +size 55869 diff --git a/examples/scene_00025/1757403967.jpg b/examples/scene_00025/1757403967.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7b9d97f04b81f21f866eb011328eaed5751800b9 --- /dev/null +++ b/examples/scene_00025/1757403967.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57c3855de5c76267cce43414e50e725a28bccb46582989e8595f42381cf25cbf +size 81979 diff --git a/examples/scene_00025/1757403998.jpg b/examples/scene_00025/1757403998.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d07bd91282453a84f6946bfe44a38754be609269 --- /dev/null +++ b/examples/scene_00025/1757403998.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df3407d9b397f085080f5a56aef63022911416c14619b5261f84238d37c1b6c2 +size 121878 diff --git a/examples/scene_00025/1757404033.jpg b/examples/scene_00025/1757404033.jpg new file mode 100644 index 0000000000000000000000000000000000000000..97baf729b99ed787cae60990853cbd95b882120d --- /dev/null +++ b/examples/scene_00025/1757404033.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8f65fc26d93fcb9346c7e01d2ba6a07c2b2a7eaacf3636d8b1804d876be58f2 +size 70308 diff --git a/examples/scene_00025/1757404056.jpg b/examples/scene_00025/1757404056.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f29e5442e50b920abb851c18152151389ef8fa70 --- /dev/null +++ b/examples/scene_00025/1757404056.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2b30f9c26f1a44a17077d553964527df895f85df78268ea3353d8fde1fa65a0 +size 69253 diff --git a/examples/scene_00025/1757404081.jpg b/examples/scene_00025/1757404081.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f4430679f65a84ba9604be0b34407b1989231461 --- /dev/null +++ b/examples/scene_00025/1757404081.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:312cf83254e97fd77a72a503d0a605b39e6dd7dffa2ca9e7972a2262cc8ea593 +size 55192 diff --git a/examples/scene_00025/1757404102.jpg b/examples/scene_00025/1757404102.jpg new file mode 100644 index 0000000000000000000000000000000000000000..74b6e027c18d4d5f45acb41329f7becf04142755 --- /dev/null +++ b/examples/scene_00025/1757404102.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ddc129f0bc93b64d3c90d59f677bb73955ab4312429a2dab64b3ba7cdd7faff +size 52731 diff --git a/examples/scene_00025/1757404127.jpg b/examples/scene_00025/1757404127.jpg new file mode 100644 index 0000000000000000000000000000000000000000..56c6c0136f6070db396cffd848c68d00c16fe807 --- /dev/null +++ b/examples/scene_00025/1757404127.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5d07977a9db0e969792aab8e929edcd7be32f7cc33cc469bbb7273beb173881 +size 61802 diff --git a/examples/scene_00025/1757404158.jpg b/examples/scene_00025/1757404158.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3766304b3c89e270021c9fb9d573b13193d21511 --- /dev/null +++ b/examples/scene_00025/1757404158.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44b4b166449be574baf222cbd3044d7353046ac37660a28e909965da70dec628 +size 61638 diff --git a/examples/synthetic_scene_00067/0.jpg b/examples/synthetic_scene_00067/0.jpg new file mode 100644 index 0000000000000000000000000000000000000000..17dc5c389bf92da5647b298dc0888baaf1f0367e --- /dev/null +++ b/examples/synthetic_scene_00067/0.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d200610bad2649c01684136fc19ad5ae608147e445cd24b207703eafe7155fb8 +size 99122 diff --git a/examples/synthetic_scene_00067/1.jpg b/examples/synthetic_scene_00067/1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5bdca69e9e8c64ab7f7f4abed0cbefe3c353b477 --- /dev/null +++ b/examples/synthetic_scene_00067/1.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fb34a457ebb39b15697bbe54ab05914ee984c79606b7d5c1690390ca6101ab3 +size 105674 diff --git a/examples/synthetic_scene_00067/10.jpg b/examples/synthetic_scene_00067/10.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8cdb29a8b2a14a7aeae8e02990babd6ddcd8aeb9 --- /dev/null +++ b/examples/synthetic_scene_00067/10.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8d9e7c2ce80c0798345e2517082db518c3b3dc064b193aea41b9c3ae174687b +size 102825 diff --git a/examples/synthetic_scene_00067/11.jpg b/examples/synthetic_scene_00067/11.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2fc0de258ab40f06787d256356ae46c4496f0f0d --- /dev/null +++ b/examples/synthetic_scene_00067/11.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24bf6cb384d5cd4bd6f68b6ac76ad6d28cd9259a0799eb1028073f2c8a951dfb +size 86096 diff --git a/examples/synthetic_scene_00067/12.jpg b/examples/synthetic_scene_00067/12.jpg new file mode 100644 index 0000000000000000000000000000000000000000..58cee46ed258f4debeba981a6d40ab441813f532 --- /dev/null +++ b/examples/synthetic_scene_00067/12.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:410b5b9ff1ee9fb0c6cbaafd5c4e09efc23b8e3f8c358f8f0120164dc829ad44 +size 98104 diff --git a/examples/synthetic_scene_00067/13.jpg b/examples/synthetic_scene_00067/13.jpg new file mode 100644 index 0000000000000000000000000000000000000000..629ea0e5ec637c4d27e1858244a2af3b9bf2dba1 --- /dev/null +++ b/examples/synthetic_scene_00067/13.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1f1ebc6a93b86e5b4a6a4d4c93b91481c1443ce5ad0b040bdf956b1fc8be1e1 +size 85946 diff --git a/examples/synthetic_scene_00067/14.jpg b/examples/synthetic_scene_00067/14.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fab30e979d5c74fa638677997378ca91f498af9e --- /dev/null +++ b/examples/synthetic_scene_00067/14.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91362d4ba17a619434b9f307c2fc83b99bd660f23270ad966a183f7595f0e733 +size 102862 diff --git a/examples/synthetic_scene_00067/15.jpg b/examples/synthetic_scene_00067/15.jpg new file mode 100644 index 0000000000000000000000000000000000000000..96d16eeb59671966247ebf6fea03aa29d3822acb --- /dev/null +++ b/examples/synthetic_scene_00067/15.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:043cb0aba9492dff4c7c5c9c806ea27dabe5885760d03769fb248c0a0a55dcfa +size 101929 diff --git a/examples/synthetic_scene_00067/16.jpg b/examples/synthetic_scene_00067/16.jpg new file mode 100644 index 0000000000000000000000000000000000000000..add334cf419050439c6422e22ba5f1ca178134a9 --- /dev/null +++ b/examples/synthetic_scene_00067/16.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:feedcc1a5c0e2a31859abbb5e595f2a4734d08d67ca9e3ea324671019b3bb0c9 +size 89883 diff --git a/examples/synthetic_scene_00067/17.jpg b/examples/synthetic_scene_00067/17.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1a743f3beb6fb433deecab2743eacc64e324e1b2 --- /dev/null +++ b/examples/synthetic_scene_00067/17.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06fd8d148b0bb4b717b53892efc3ebea5bfa0fda8566e18f7645ed3220b61e58 +size 101449 diff --git a/examples/synthetic_scene_00067/18.jpg b/examples/synthetic_scene_00067/18.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6b72c35c58b6ed09ee4076c91a544af617228135 --- /dev/null +++ b/examples/synthetic_scene_00067/18.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dff7693877f2aa1072e9edbb8bf87a78176a44a2895d517f26b9c4f3e141fc79 +size 91280 diff --git a/examples/synthetic_scene_00067/19.jpg b/examples/synthetic_scene_00067/19.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e514c6bd85b6ba167264fdb7c1d2cdfb76fa1a66 --- /dev/null +++ b/examples/synthetic_scene_00067/19.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dac4778e95bca604935052e24bb3d15269359d7fc41ff19941cf51d100c3cb23 +size 80585 diff --git a/examples/synthetic_scene_00067/2.jpg b/examples/synthetic_scene_00067/2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..51820c210b47ccfb5c02bbaf2ef3bd69c75ac898 --- /dev/null +++ b/examples/synthetic_scene_00067/2.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bb5d4088ae733aecd96fe95c572f650a06df904d490dc9e76372368ccdd6375 +size 97659 diff --git a/examples/synthetic_scene_00067/20.jpg b/examples/synthetic_scene_00067/20.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b376488b10cc2c710e8eba2862cd8d2a077fd6de --- /dev/null +++ b/examples/synthetic_scene_00067/20.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5034298966a3202c918620090a620befbcc17faca2063779a7299f021dafc8e +size 102976 diff --git a/examples/synthetic_scene_00067/21.jpg b/examples/synthetic_scene_00067/21.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d1d5f5072daacd6b6cb7bd96a45b7e937e2f4484 --- /dev/null +++ b/examples/synthetic_scene_00067/21.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c495a8d3ab490d6d01e7cf78e514d398745746c4da5fadb96cc303715c43ca9 +size 115802 diff --git a/examples/synthetic_scene_00067/22.jpg b/examples/synthetic_scene_00067/22.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c78ff3e0e19ceba199b07584d6c9e75576c7db67 --- /dev/null +++ b/examples/synthetic_scene_00067/22.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a2e0a5c5ac45952ddec971dcbf9105f3de3717c0959886ee9da4a45118308c9 +size 103107 diff --git a/examples/synthetic_scene_00067/23.jpg b/examples/synthetic_scene_00067/23.jpg new file mode 100644 index 0000000000000000000000000000000000000000..42d113ea1573a934935325011edcb0a96109ff3a --- /dev/null +++ b/examples/synthetic_scene_00067/23.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98457d9711eaa5934efa35d223b6f3d43337e7881866da7ce62b0f5777ea2602 +size 103092 diff --git a/examples/synthetic_scene_00067/24.jpg b/examples/synthetic_scene_00067/24.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a5f78b49b3945e5f8091351076a294297514ad19 --- /dev/null +++ b/examples/synthetic_scene_00067/24.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f0a3aab4cbbf4edffb9da408f1d8610885a865154587c0c21bb19487560816b +size 97930 diff --git a/examples/synthetic_scene_00067/25.jpg b/examples/synthetic_scene_00067/25.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fc2f42154b8ea2d7b60bfa9bc7085d3f92ae2643 --- /dev/null +++ b/examples/synthetic_scene_00067/25.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8e4e7606cb2ed9850d4a0929bf63c81940c2be59084c500f38ae67ad333c8f7 +size 104082 diff --git a/examples/synthetic_scene_00067/26.jpg b/examples/synthetic_scene_00067/26.jpg new file mode 100644 index 0000000000000000000000000000000000000000..21834b77bf76bfcf0d27db4a2250898cd384b0ad --- /dev/null +++ b/examples/synthetic_scene_00067/26.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac60df6473259b15a658f8d637f997cb6191b04bc5a26009439f27818de98bda +size 103922 diff --git a/examples/synthetic_scene_00067/27.jpg b/examples/synthetic_scene_00067/27.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9052e915aa0837209c72712dd31b94e286ea7d24 --- /dev/null +++ b/examples/synthetic_scene_00067/27.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd88aea447ee99cb7d935c222276ba5e4606d3d01299998aa37b674d046644ac +size 115903 diff --git a/examples/synthetic_scene_00067/28.jpg b/examples/synthetic_scene_00067/28.jpg new file mode 100644 index 0000000000000000000000000000000000000000..778764a122304e0e8f2cdc7556458a3100e0b2fd --- /dev/null +++ b/examples/synthetic_scene_00067/28.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a74d4347ed0707a676fcfc9bdf6b9d4af14280fa8f2487930fbb5ec71e3c744 +size 104473 diff --git a/examples/synthetic_scene_00067/29.jpg b/examples/synthetic_scene_00067/29.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d774ca1740c782a1f158813175fd09721f08810b --- /dev/null +++ b/examples/synthetic_scene_00067/29.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab73e3e3c22bc76033a6e413b44571ea9fa94d1c8241b60ad8e1b5074b65c5ad +size 112372 diff --git a/examples/synthetic_scene_00067/3.jpg b/examples/synthetic_scene_00067/3.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d6f34d6aebf3bb383cfb0ec9ebff75c7d8532295 --- /dev/null +++ b/examples/synthetic_scene_00067/3.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e058d379f6b5093ca0731ab3d4382ca972da436fdaf1ad40528f6649469ec049 +size 87268 diff --git a/examples/synthetic_scene_00067/30.jpg b/examples/synthetic_scene_00067/30.jpg new file mode 100644 index 0000000000000000000000000000000000000000..665b41b956861d3bfe012cc19268ea037ef57551 --- /dev/null +++ b/examples/synthetic_scene_00067/30.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dccb90e5068c2aa85a0779b3c700a666b10ae30872ef5820d795b3bdad9a25f +size 106541 diff --git a/examples/synthetic_scene_00067/31.jpg b/examples/synthetic_scene_00067/31.jpg new file mode 100644 index 0000000000000000000000000000000000000000..16107388cc7927ae59896f84f44e82f96f7a89b2 --- /dev/null +++ b/examples/synthetic_scene_00067/31.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cadc062a730c3820fdff03ef33c8f98e37f5dbc367631acc006a4b713c106a8 +size 96024 diff --git a/examples/synthetic_scene_00067/32.jpg b/examples/synthetic_scene_00067/32.jpg new file mode 100644 index 0000000000000000000000000000000000000000..564dbb64900d5c5e964454967b9c4aee93d8a0f9 --- /dev/null +++ b/examples/synthetic_scene_00067/32.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a84a625dea8046c04891ac0b105d7da0cf10bcc5e61671c805528d4c79b2bce +size 96102 diff --git a/examples/synthetic_scene_00067/33.jpg b/examples/synthetic_scene_00067/33.jpg new file mode 100644 index 0000000000000000000000000000000000000000..505f55d4c18012462b7512002c9491db36e6d22e --- /dev/null +++ b/examples/synthetic_scene_00067/33.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e43e1a5dc5fd590576347df8c3837042159d9204c3fec815f0d01b151fc1ce7b +size 102356 diff --git a/examples/synthetic_scene_00067/34.jpg b/examples/synthetic_scene_00067/34.jpg new file mode 100644 index 0000000000000000000000000000000000000000..312bc0d509fb9ca4c85f3112ad38367d56d38441 --- /dev/null +++ b/examples/synthetic_scene_00067/34.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a87cfcd9944d2ba21985ed840d470756239fab256e3ea9531637ce85d02bdd3 +size 110948 diff --git a/examples/synthetic_scene_00067/35.jpg b/examples/synthetic_scene_00067/35.jpg new file mode 100644 index 0000000000000000000000000000000000000000..768edf4cb8c3d145a4ed1d053680096136cc17a7 --- /dev/null +++ b/examples/synthetic_scene_00067/35.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d3dea2c86f5728aab276a70ff766be798ecc2f8b2f1c586ca6b80cee94371e9 +size 104684 diff --git a/examples/synthetic_scene_00067/36.jpg b/examples/synthetic_scene_00067/36.jpg new file mode 100644 index 0000000000000000000000000000000000000000..706c2d070c648771e4a7d6ef4838cef2dec77a40 --- /dev/null +++ b/examples/synthetic_scene_00067/36.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0147c44f9f257db416c9093ea3d8747b8999a4cce4d6a8c209742769bf42c2b6 +size 86460 diff --git a/examples/synthetic_scene_00067/37.jpg b/examples/synthetic_scene_00067/37.jpg new file mode 100644 index 0000000000000000000000000000000000000000..08e6e2a081fabb23ededcd6150a1e6ff4aae8669 --- /dev/null +++ b/examples/synthetic_scene_00067/37.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71b489147e68131c7117f2191c7466dc7fdca4f38751f85e7b4edfe67733fc29 +size 95888 diff --git a/examples/synthetic_scene_00067/38.jpg b/examples/synthetic_scene_00067/38.jpg new file mode 100644 index 0000000000000000000000000000000000000000..523b2f4598eb54bbd8fa449b5ec0739feb7d76ae --- /dev/null +++ b/examples/synthetic_scene_00067/38.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:015b19895281cfadc6b015b022e6f6a96dd04ba4de1461bef3697fa58a90e6bc +size 94044 diff --git a/examples/synthetic_scene_00067/4.jpg b/examples/synthetic_scene_00067/4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a4584bf487fe6c713f777aeec7d7cbc95bfeb122 --- /dev/null +++ b/examples/synthetic_scene_00067/4.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a99acc0f7ce6bfbf9a8a072b664906a2a1b06e43c91173e4030415aee9e1876c +size 97661 diff --git a/examples/synthetic_scene_00067/5.jpg b/examples/synthetic_scene_00067/5.jpg new file mode 100644 index 0000000000000000000000000000000000000000..344c289fe781174dbd20ea8624b6c3840e94e5b2 --- /dev/null +++ b/examples/synthetic_scene_00067/5.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d75c1edd1d44b8d62286a18660fc59580701f8318e9c1fdf30c290208c86b592 +size 114988 diff --git a/examples/synthetic_scene_00067/6.jpg b/examples/synthetic_scene_00067/6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a9deb934873060f02cad2bd9f1fa05e5c1b7d1ce --- /dev/null +++ b/examples/synthetic_scene_00067/6.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c36c8e99713c33a13b4013616e3cd5a0573acacaec9c9a10265145f2960f83a8 +size 100165 diff --git a/examples/synthetic_scene_00067/7.jpg b/examples/synthetic_scene_00067/7.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ba7becf893e01f21de30e454701c23b2200c94b7 --- /dev/null +++ b/examples/synthetic_scene_00067/7.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aefbcd4e54b243eec34229c32bc4b59dd859354cf5f710f22623a352aa7ff865 +size 89076 diff --git a/examples/synthetic_scene_00067/8.jpg b/examples/synthetic_scene_00067/8.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7bc45af58d3561b402e8936acf8daa7eef982616 --- /dev/null +++ b/examples/synthetic_scene_00067/8.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0ce4ff7cd1281e1919543ba0407d0ff4fd0f1d2909a89e895f991fb83a09d73 +size 100774 diff --git a/examples/synthetic_scene_00067/9.jpg b/examples/synthetic_scene_00067/9.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8384a4d5184825adadf5692b5941a998a90519f2 --- /dev/null +++ b/examples/synthetic_scene_00067/9.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6be4e542e2f8807edd401a28df136677b0a1b28729bf0f6a39339059154bbf24 +size 97197 diff --git a/examples/synthetic_scene_00156/0.jpg b/examples/synthetic_scene_00156/0.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4f68ea3683cdb620e651f1a7493e8cfdde6ce00a --- /dev/null +++ b/examples/synthetic_scene_00156/0.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c17db5ab8ec8c45f0a82a311305e7c6e9d8bd954943de376ff4a532165fb7e4c +size 84973 diff --git a/examples/synthetic_scene_00156/1.jpg b/examples/synthetic_scene_00156/1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6bf0920bb9649d842778369c6098d5b68f03c1c6 --- /dev/null +++ b/examples/synthetic_scene_00156/1.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:391ec6ef4f99abb55ed4114062fdeab7c04cc157d4446e1b8bebeeac83434a8e +size 52986 diff --git a/examples/synthetic_scene_00156/10.jpg b/examples/synthetic_scene_00156/10.jpg new file mode 100644 index 0000000000000000000000000000000000000000..01f23a1e71e5023a33ff29f9faa3a609c0e6cf77 --- /dev/null +++ b/examples/synthetic_scene_00156/10.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed72075142cd1a405536832b328f4aacd37e7aefddbfc3e229460635e8d7d15c +size 87286 diff --git a/examples/synthetic_scene_00156/11.jpg b/examples/synthetic_scene_00156/11.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4f570453acb8510df4d693836d8625a50325909a --- /dev/null +++ b/examples/synthetic_scene_00156/11.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ae8051b9eb7c2187844d4f7923f6dcffbc9a0d12a04f5a3c4ec518c96345559 +size 92045 diff --git a/examples/synthetic_scene_00156/12.jpg b/examples/synthetic_scene_00156/12.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1a637131dbed5b654face01a349c4eacff1a1bd0 --- /dev/null +++ b/examples/synthetic_scene_00156/12.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:366a7853fb94f9f5ebba635b6bc681df86f87c8cec83a0b4742c5f1a0fe792d0 +size 72618 diff --git a/examples/synthetic_scene_00156/13.jpg b/examples/synthetic_scene_00156/13.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e25f9687ce5ab3bd862e77efeef43d34cb33d2ea --- /dev/null +++ b/examples/synthetic_scene_00156/13.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b21d8099a9e1e07e7259405af326f68f7cdd2382dd74f88b8fcc675b47b4534a +size 77007 diff --git a/examples/synthetic_scene_00156/14.jpg b/examples/synthetic_scene_00156/14.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bd6c21f99b50d7565cf0ba00640d63c6214fe8c2 --- /dev/null +++ b/examples/synthetic_scene_00156/14.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:126f467747035fa38b9fcf1ddc08a2e57dcb557656e29c51af8eeee4d114eb24 +size 79600 diff --git a/examples/synthetic_scene_00156/15.jpg b/examples/synthetic_scene_00156/15.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2756baebc69ff5a771cd97f045cd378f1fc18367 --- /dev/null +++ b/examples/synthetic_scene_00156/15.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4d6f9ec9e77cd86fc1f21e694ead49391138b500ed74e0fac802804fdc79ef7 +size 77965 diff --git a/examples/synthetic_scene_00156/16.jpg b/examples/synthetic_scene_00156/16.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c5928c7fc39069b3510bf8639300191d2cff3cc9 --- /dev/null +++ b/examples/synthetic_scene_00156/16.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6caadc407d1df451da229ba4263c2bbf47f8fbea61857b153a6c62c7770eeead +size 78669 diff --git a/examples/synthetic_scene_00156/17.jpg b/examples/synthetic_scene_00156/17.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d629e9be4c4d029904628b86ddd316f0ae67957c --- /dev/null +++ b/examples/synthetic_scene_00156/17.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebbac45c72c205fba31cace7606d88101f0a4ede19ad7a1980a221d73cc63b9f +size 87753 diff --git a/examples/synthetic_scene_00156/18.jpg b/examples/synthetic_scene_00156/18.jpg new file mode 100644 index 0000000000000000000000000000000000000000..95a17209ee1fe51f87f70be1c02d1905c62eb075 --- /dev/null +++ b/examples/synthetic_scene_00156/18.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a610e39732125551382817e6062ac5b6cda6e03ba3f361571e736b900d5f8120 +size 85533 diff --git a/examples/synthetic_scene_00156/19.jpg b/examples/synthetic_scene_00156/19.jpg new file mode 100644 index 0000000000000000000000000000000000000000..502816b04675a7d394d89657c4c2f54ebc7c7192 --- /dev/null +++ b/examples/synthetic_scene_00156/19.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4037f7c9a9069f22dbc02bfb9ee4d1878f00532b0de906099a632b02c53746b7 +size 89795 diff --git a/examples/synthetic_scene_00156/2.jpg b/examples/synthetic_scene_00156/2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..17c1ab373079bcdc52e80c9c8517e1c42c4b3ddf --- /dev/null +++ b/examples/synthetic_scene_00156/2.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e7e61012de43d640a54506f1e0d0376877a0bfc445f26840b0d0b1c981c0767 +size 87521 diff --git a/examples/synthetic_scene_00156/20.jpg b/examples/synthetic_scene_00156/20.jpg new file mode 100644 index 0000000000000000000000000000000000000000..663d881dccb5699f2f0998f1ef71874572778f7e --- /dev/null +++ b/examples/synthetic_scene_00156/20.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fa13305af6a532158b10fdeba9d7e6b985cf47b5a1c4c338e724fb594383621 +size 105185 diff --git a/examples/synthetic_scene_00156/21.jpg b/examples/synthetic_scene_00156/21.jpg new file mode 100644 index 0000000000000000000000000000000000000000..63566198d21534a3f0d5420ee90954e846a52db9 --- /dev/null +++ b/examples/synthetic_scene_00156/21.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3efe0a8febf0f8e86fedf850e2de7cf0de8fe9727bff55961e050faa56a022e3 +size 98403 diff --git a/examples/synthetic_scene_00156/22.jpg b/examples/synthetic_scene_00156/22.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3da33344cadc417b06957e3f4d69721f3c8bbc43 --- /dev/null +++ b/examples/synthetic_scene_00156/22.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbbcb9801cb41a99739ada8db0d1b9382ad6167c8f33d33de7c2d285bfe67829 +size 95551 diff --git a/examples/synthetic_scene_00156/23.jpg b/examples/synthetic_scene_00156/23.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3e781cf63b966d957764004e92270509aee4c5ee --- /dev/null +++ b/examples/synthetic_scene_00156/23.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f043576eda0c6c53014a82b046a4712f64abed3978b99d28a951b8ee3bd0cf27 +size 87299 diff --git a/examples/synthetic_scene_00156/24.jpg b/examples/synthetic_scene_00156/24.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c52dbd770c0d019643df4a76f9f097f862dd3123 --- /dev/null +++ b/examples/synthetic_scene_00156/24.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abe22bbe3e134216f60dffe7048ce2871b7250bf01f0ef673079fa3caf1d6133 +size 103599 diff --git a/examples/synthetic_scene_00156/3.jpg b/examples/synthetic_scene_00156/3.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4ae008e50fe6178e9827b72bcf539a45955ce3a2 --- /dev/null +++ b/examples/synthetic_scene_00156/3.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45dd9015094ac2bb9b4a60cf3732e8447f61ce7513f7ffe4433f3a0591247498 +size 63195 diff --git a/examples/synthetic_scene_00156/4.jpg b/examples/synthetic_scene_00156/4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..56818bd9af628edfb2c023ac541f147b1d637ada --- /dev/null +++ b/examples/synthetic_scene_00156/4.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7919bc5d97b12dc6119d7fdb5bbe394d4e7d64efe5c8800048cc33931fe35c5 +size 81038 diff --git a/examples/synthetic_scene_00156/5.jpg b/examples/synthetic_scene_00156/5.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7406810f3f007112c75b460db23114c03178ff7f --- /dev/null +++ b/examples/synthetic_scene_00156/5.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad1536fb2d45413e72ea0342d1aa105d6fe968b88540d5a7dcb873fab95f34f9 +size 65575 diff --git a/examples/synthetic_scene_00156/6.jpg b/examples/synthetic_scene_00156/6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f97b4859d1510b07ec997eba277ed922ad5bb8a3 --- /dev/null +++ b/examples/synthetic_scene_00156/6.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cddc5e3dad96c44d2819082fe70a07774f7e67964c6129b4f505fdd553064f96 +size 84117 diff --git a/examples/synthetic_scene_00156/7.jpg b/examples/synthetic_scene_00156/7.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b0464d7bc5a5ab6e115ac4ddb6537d61314c7512 --- /dev/null +++ b/examples/synthetic_scene_00156/7.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7677cf911036255203aadab2043d1afca5705328f01cebe5c9e63dd36495831 +size 85120 diff --git a/examples/synthetic_scene_00156/8.jpg b/examples/synthetic_scene_00156/8.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fb4fa01a003caf7dd516ad3cca5a56b4a82c67f8 --- /dev/null +++ b/examples/synthetic_scene_00156/8.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d77ab1e4978f74d47257f343cfae574ace0175078cc5a4ab5dc7e5eefc75e9cf +size 80357 diff --git a/examples/synthetic_scene_00156/9.jpg b/examples/synthetic_scene_00156/9.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7437518246ec522e8c95af0f2e4a691a34862476 --- /dev/null +++ b/examples/synthetic_scene_00156/9.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d02f60b5dbfc4f5a489ed7e040399543f0c042b922ff693476f1ea66b5911709 +size 78681 diff --git a/examples/weak_covis_2/0.jpg b/examples/weak_covis_2/0.jpg new file mode 100644 index 0000000000000000000000000000000000000000..db1a89c5f60d4924401c6d1c809bb900c45f9e94 --- /dev/null +++ b/examples/weak_covis_2/0.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6eae8f66e21e06253cd0cf778b856d9c532c218b896b16df959b3e86477da6dd +size 75096 diff --git a/examples/weak_covis_2/15.jpg b/examples/weak_covis_2/15.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6ab2de3d5a6c236f9f1bf89574a39fa68f56b3db --- /dev/null +++ b/examples/weak_covis_2/15.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f174f0ad72afc620d4e0233ea881f164cad5434fe740464b14b36ab2ac95363 +size 72437 diff --git a/examples/z0/panoImage_2.jpg b/examples/z0/panoImage_2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..952fd915dc05c03cdc9983d8e93329da64be19bc --- /dev/null +++ b/examples/z0/panoImage_2.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fedad6f6c772affc2936500aa71833256c51f52213a63f8fe2f9f21becaacb18 +size 78333 diff --git a/examples/z1/1.jpeg b/examples/z1/1.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..522b07a5581a48b00eaf48e1bf7c2e24055d26cd --- /dev/null +++ b/examples/z1/1.jpeg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34170cc8f85d963caf80226baa1456eeacd6f92c2991ffe567271b80638d5025 +size 92348 diff --git a/examples/z2/1.jpeg b/examples/z2/1.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..a74d52f6774021a1eeb98dfc2f4ceb254ca94d04 --- /dev/null +++ b/examples/z2/1.jpeg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:886fffcb66e4f59a8123abf3bf330a5b6d828630a89687aa04345861e4ceddf4 +size 87569 diff --git a/examples/z3/1.jpeg b/examples/z3/1.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..fa072f9c736d1b45d7226b92d4bc323548290884 --- /dev/null +++ b/examples/z3/1.jpeg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e74b88985883f1c659081a3bdb0fa053a60c852eaf3e515692be8ac3e06d2d61 +size 86824 diff --git a/examples/z4/12.jpg b/examples/z4/12.jpg new file mode 100644 index 0000000000000000000000000000000000000000..84c124c7e3f9801070effdcbcced00e46eb6aecf --- /dev/null +++ b/examples/z4/12.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecf558205078a2ee483fc138d7db34bad3cfa0b8304f80d295a9afeb5cdfeb36 +size 89930 diff --git a/examples/z5/panoImage_orig.jpg b/examples/z5/panoImage_orig.jpg new file mode 100644 index 0000000000000000000000000000000000000000..df789542571a4f0786cc63f11471206d13d9b90e --- /dev/null +++ b/examples/z5/panoImage_orig.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98aba8fc253c71f9ad91c19778e4902139c12a7ae4b880218672c54d1c52a051 +size 79993 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52ebaf28c5d589d8050c8c84bf4d17e37dd2f48d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +torch +torchvision +numpy==1.26.1 +opencv-python-headless +trimesh +scipy +matplotlib +gradio>=4.0 +huggingface_hub