Spaces:
Running on Zero
Running on Zero
Download llava/model/multimodal_spatial_encoder/vggt_spatial_encoder.py from hugging-apps/mvrd-spatial-reasoning: direct link, hf CLI and curl.
- Browser
- Download file 12.8 kB
-
https://huggingface.co/spaces/hugging-apps/mvrd-spatial-reasoning/resolve/main/llava/model/multimodal_spatial_encoder/vggt_spatial_encoder.py
- Command line
-
hf download hf://spaces/hugging-apps/mvrd-spatial-reasoning/llava/model/multimodal_spatial_encoder/vggt_spatial_encoder.py
-
curl -L -o vggt_spatial_encoder.py https://huggingface.co/spaces/hugging-apps/mvrd-spatial-reasoning/resolve/main/llava/model/multimodal_spatial_encoder/vggt_spatial_encoder.py
12.8 kB
| import torch | |
| import torch.nn as nn | |
| from transformers import PretrainedConfig, PreTrainedModel | |
| from transformers.modeling_outputs import BaseModelOutputWithPooling, BaseModelOutput | |
| from typing import Union, Optional, Tuple | |
| import os | |
| from llava.utils import rank0_print | |
| from einops import rearrange | |
| import sys | |
| # TODO: Verify this relative path is correct for the project structure | |
| vggt_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'vggt')) | |
| if vggt_path not in sys.path: | |
| sys.path.append(vggt_path) | |
| from vggt.models.vggt import VGGT | |
| import numpy as np | |
| class VGGTSpatialConfig(PretrainedConfig): | |
| model_type = "vggt_spatial_model" | |
| def __init__( | |
| self, | |
| weights_path: str = "VGGT-1B", # Default relative name | |
| **kwargs, | |
| ): | |
| super().__init__(**kwargs) | |
| self.weights_path = weights_path | |
| def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": | |
| cls._set_token_in_kwargs(kwargs) | |
| config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) | |
| # get the spatial config dict if we are loading from Cut3rSpatialConfig | |
| if config_dict.get("model_type") == "vggt_spatial_model": # Or check if it's a base VGGT config? | |
| # Assuming a base VGGT config might not have 'spatial_config'. Adjust logic if needed. | |
| # If loading from a parent config that wraps this, you might need: | |
| # config_dict = config_dict["spatial_config"] | |
| pass # Keep config_dict as is if it's already the correct type | |
| if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type: | |
| print(f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " f"{cls.model_type}. This is not supported for all configurations of models and can yield errors.") | |
| return cls.from_dict(config_dict, **kwargs) | |
| class VGGTSpatialPreTrainedModel(PreTrainedModel): | |
| """ | |
| An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained | |
| models. | |
| """ | |
| config_class = VGGTSpatialConfig | |
| base_model_prefix = "vggt" | |
| supports_gradient_checkpointing = True | |
| def _init_weights(self, module): | |
| """Initialize the weights""" | |
| pass | |
| def prepare_input(pixel_values, modality): | |
| # pixel_values: (F, C, H, W) | |
| if modality == "video": | |
| pixel_values = nn.functional.interpolate(pixel_values, size=(196, 196), mode='bilinear') | |
| elif modality == "image": | |
| pixel_values = nn.functional.interpolate(pixel_values, size=(378, 378), mode='bilinear') | |
| else: | |
| raise ValueError(f"Unsupported modality: {modality}") | |
| # convert to 0-1 | |
| pixel_values = pixel_values * 0.5 + 0.5 | |
| pixel_values = pixel_values.unsqueeze(0) # 1 frame 3 h w | |
| return pixel_values | |
| class VGGT_Encoder(nn.Module): | |
| def __init__(self, config: VGGTSpatialConfig, **kwargs): | |
| super().__init__() | |
| # Load model using the path from the config | |
| rank0_print(f"Loading VGGT from: {config.weights_path}") | |
| self.vggt = VGGT.from_pretrained(config.weights_path) | |
| self.vggt.eval() | |
| for param in self.vggt.parameters(): | |
| param.requires_grad = False | |
| def forward( | |
| self, | |
| pixel_values, | |
| modality, | |
| attention_mask: Optional[torch.Tensor] = None, | |
| output_attentions: Optional[bool] = None, | |
| output_hidden_states: Optional[bool] = None, | |
| return_dict: Optional[bool] = None, | |
| point_cloud_output_paths: Optional[list[str]] = None # Add for API consistency if needed, though not used here | |
| ) -> Union[Tuple, BaseModelOutputWithPooling]: | |
| # FIXME: can't process batch | |
| views = prepare_input(pixel_values=pixel_values, modality=modality) | |
| with torch.no_grad(): | |
| with torch.cuda.amp.autocast(dtype=views.dtype): | |
| aggregated_tokens_list, ps_idx = self.vggt.aggregator(views) | |
| # just need the patch tokens of the last layer | |
| spatial_feat = aggregated_tokens_list[-1] # 1, num_frame, num_patch, token_feature | |
| spatial_feat = spatial_feat.to(pixel_values.dtype) | |
| spatial_feat = spatial_feat.squeeze(0) # num_frame, num_patch, token_feature | |
| camera_token = spatial_feat[:, 0:1, :] | |
| patch_tokens = spatial_feat[:, ps_idx:, :] | |
| # for debug(visualize point cloud) | |
| # pts3d_pred, pts3d_conf = self.vggt.point_head( | |
| # aggregated_tokens_list, images=views, patch_start_idx=ps_idx | |
| # ) | |
| return (camera_token, patch_tokens) | |
| class VGGT_SpatialTransformer(nn.Module): | |
| def __init__(self, config: VGGTSpatialConfig, **kwargs): | |
| super().__init__() | |
| self.config = config | |
| self.encoder = VGGT_Encoder(config=config, **kwargs) | |
| def forward( | |
| self, | |
| pixel_values, | |
| output_attentions: Optional[bool] = None, | |
| output_hidden_states: Optional[bool] = None, | |
| return_dict: Optional[bool] = None, | |
| modality: Optional[str] = None, | |
| ) -> Union[Tuple, BaseModelOutputWithPooling]: | |
| r""" | |
| Returns: | |
| """ | |
| output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions | |
| output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states | |
| return_dict = return_dict if return_dict is not None else self.config.use_return_dict | |
| encoder_outputs = self.encoder( | |
| pixel_values=pixel_values, | |
| modality=modality, | |
| output_attentions=output_attentions, | |
| output_hidden_states=output_hidden_states, | |
| return_dict=return_dict, | |
| ) | |
| return encoder_outputs | |
| class VGGT_SpatialModel(VGGTSpatialPreTrainedModel): | |
| config_class = VGGTSpatialConfig | |
| main_input_name = "pixel_values" | |
| _no_split_modules = ["VGGTSpatialEncoderLayer"] | |
| def __init__(self, config: VGGTSpatialConfig, **kwargs): | |
| super().__init__(config) | |
| self.spatial_model = VGGT_SpatialTransformer(config, **kwargs) | |
| # Initialize weights and apply final processing | |
| self.post_init() | |
| def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], config: VGGTSpatialConfig, **kwargs): | |
| # Use the provided config object | |
| model = cls(config=config, **kwargs) | |
| # Potentially load state dict here if pretrained_model_name_or_path points to a checkpoint | |
| # containing the wrapper model's state, not just the base VGGT model. | |
| # For now, it assumes the config dictates the base model loading within VGGT_SpatialTransformer. | |
| return model | |
| def forward( | |
| self, | |
| pixel_values, | |
| modality, | |
| output_attentions: Optional[bool] = None, | |
| output_hidden_states: Optional[bool] = None, | |
| return_dict: Optional[bool] = None, | |
| point_cloud_output_paths: Optional[list[str]] = None # Add for API consistency | |
| ) -> Union[Tuple, BaseModelOutputWithPooling]: | |
| return_dict = return_dict if return_dict is not None else self.config.use_return_dict | |
| return self.spatial_model( | |
| pixel_values=pixel_values, | |
| modality=modality, | |
| output_attentions=output_attentions, | |
| output_hidden_states=output_hidden_states, | |
| return_dict=return_dict, | |
| # Pass point_cloud_output_paths if the underlying model needs it, though VGGT_SpatialTransformer doesn't use it currently | |
| # point_cloud_output_paths=point_cloud_output_paths | |
| ) | |
| class VGGTSpatialTower(nn.Module): | |
| def __init__(self, spatial_tower, spatial_tower_cfg, delay_load=True): | |
| super().__init__() | |
| self.is_loaded = False | |
| # Dynamically determine the weights path | |
| script_dir = os.path.dirname(os.path.abspath(__file__)) | |
| # Assumes vggt repo is cloned sibling to the main project repo | |
| # ../../../ -> goes up from multimodal_spatial_encoder -> model -> llava -> project root | |
| root = os.path.abspath(os.path.join(script_dir, '..', '..', '..')) | |
| # Adjust the relative path 'vggt/VGGT-1B' as needed based on where VGGT weights are stored | |
| default_weights_name = "VGGT-1B" # Or whatever the default identifier is | |
| dynamic_weights_path = os.path.join(root, 'vggt', default_weights_name) # Construct full path | |
| # Create config with the dynamic path | |
| self.config = VGGTSpatialConfig( | |
| weights_path=dynamic_weights_path, | |
| # Pass other relevant configs from spatial_tower_cfg if needed | |
| # Example: output_attentions=getattr(spatial_tower_cfg, 'output_attentions', False), | |
| ) | |
| self.spatial_tower_name = spatial_tower # Keep track of the logical name/identifier | |
| if not delay_load: | |
| rank0_print(f"Loading spatial tower: {spatial_tower} using weights from {self.config.weights_path}") | |
| self.load_model() | |
| elif getattr(spatial_tower_cfg, "unfreeze_mm_spatial_tower", False): | |
| # TODO: better detector is needed. | |
| rank0_print(f"The checkpoint seems to contain `spatial_tower` weights: `unfreeze_mm_spatial_tower`: True.") | |
| self.load_model() | |
| elif hasattr(spatial_tower_cfg, "mm_tunable_parts") and "mm_spatial_tower" in spatial_tower_cfg.mm_tunable_parts: | |
| rank0_print(f"The checkpoint seems to contain `spatial_tower` weights: `mm_tunable_parts` contains `mm_spatial_tower`.") | |
| self.load_model() | |
| else: | |
| # Store the config even if not loading immediately | |
| self.cfg_only = self.config | |
| def load_model(self, device_map=None): | |
| if self.is_loaded: | |
| rank0_print("{} is already loaded, `load_model` called again, skipping.".format(self.spatial_tower_name)) | |
| return | |
| # Load the VGGT_SpatialModel using the config which contains the weights_path | |
| # The spatial_tower_name might be used here if it points to a checkpoint containing the *entire* VGGTSpatialModel state, | |
| # rather than just the base VGGT weights. Adjust logic based on how checkpoints are saved/loaded. | |
| # Assuming spatial_tower_name is mainly an identifier and config.weights_path points to base VGGT weights. | |
| rank0_print(f"Instantiating VGGT_SpatialModel with config pointing to: {self.config.weights_path}") | |
| self.spatial_tower = VGGT_SpatialModel.from_pretrained( | |
| pretrained_model_name_or_path=self.spatial_tower_name, # Or potentially self.config.weights_path if appropriate | |
| config=self.config, | |
| device_map=device_map | |
| ) | |
| self.spatial_tower.requires_grad_(False) | |
| self.is_loaded = True | |
| def forward(self, images, modality): | |
| if type(images) is list: | |
| image_features = [] | |
| for image in images: | |
| # Pass point_cloud_output_paths=None if needed by the forward signature | |
| image_forward_out = self.spatial_tower(image.to(device=self.device, dtype=self.dtype).unsqueeze(0), output_hidden_states=True, modality=modality) | |
| # Assuming image_forward_out is now a tuple (camera_token, patch_features) | |
| # We likely only need the patch features for downstream tasks | |
| patch_features = image_forward_out[1] # Get patch features | |
| # image_feature = image_forward_out.last_hidden_state.to(image.dtype) # Original line | |
| image_feature = patch_features.to(image.dtype) | |
| image_features.append(image_feature) | |
| else: | |
| # Pass point_cloud_output_paths=None if needed by the forward signature | |
| image_features = self.spatial_tower(images.to(device=self.device, dtype=self.dtype), output_hidden_states=True, modality=modality) | |
| return image_features | |
| def dummy_feature(self): | |
| return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype) | |
| def dtype(self): | |
| for p in self.spatial_tower.parameters(): | |
| return p.dtype | |
| def device(self): | |
| for p in self.spatial_tower.parameters(): | |
| return p.device | |
| def hidden_size(self): | |
| return self.config.hidden_size | |
| def num_patches(self): | |
| return (self.config.image_size // self.config.patch_size) ** 2 | |
| def num_patches_per_side(self): | |
| return self.config.image_size // self.config.patch_size | |
| def image_size(self): | |
| return self.config.image_size |