multimodalart's picture
multimodalart HF Staff
Upload folder using huggingface_hub
8a2c145 verified
Raw History Blame Contribute Delete
12.8 kB
import torch
import torch.nn as nn
from transformers import PretrainedConfig, PreTrainedModel
from transformers.modeling_outputs import BaseModelOutputWithPooling, BaseModelOutput
from typing import Union, Optional, Tuple
import os
from llava.utils import rank0_print
from einops import rearrange
import sys
# TODO: Verify this relative path is correct for the project structure
vggt_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'vggt'))
if vggt_path not in sys.path:
sys.path.append(vggt_path)
from vggt.models.vggt import VGGT
import numpy as np
class VGGTSpatialConfig(PretrainedConfig):
model_type = "vggt_spatial_model"
def __init__(
self,
weights_path: str = "VGGT-1B", # Default relative name
**kwargs,
):
super().__init__(**kwargs)
self.weights_path = weights_path
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
cls._set_token_in_kwargs(kwargs)
config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
# get the spatial config dict if we are loading from Cut3rSpatialConfig
if config_dict.get("model_type") == "vggt_spatial_model": # Or check if it's a base VGGT config?
# Assuming a base VGGT config might not have 'spatial_config'. Adjust logic if needed.
# If loading from a parent config that wraps this, you might need:
# config_dict = config_dict["spatial_config"]
pass # Keep config_dict as is if it's already the correct type
if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
print(f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " f"{cls.model_type}. This is not supported for all configurations of models and can yield errors.")
return cls.from_dict(config_dict, **kwargs)
class VGGTSpatialPreTrainedModel(PreTrainedModel):
"""
An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
models.
"""
config_class = VGGTSpatialConfig
base_model_prefix = "vggt"
supports_gradient_checkpointing = True
def _init_weights(self, module):
"""Initialize the weights"""
pass
def prepare_input(pixel_values, modality):
# pixel_values: (F, C, H, W)
if modality == "video":
pixel_values = nn.functional.interpolate(pixel_values, size=(196, 196), mode='bilinear')
elif modality == "image":
pixel_values = nn.functional.interpolate(pixel_values, size=(378, 378), mode='bilinear')
else:
raise ValueError(f"Unsupported modality: {modality}")
# convert to 0-1
pixel_values = pixel_values * 0.5 + 0.5
pixel_values = pixel_values.unsqueeze(0) # 1 frame 3 h w
return pixel_values
class VGGT_Encoder(nn.Module):
def __init__(self, config: VGGTSpatialConfig, **kwargs):
super().__init__()
# Load model using the path from the config
rank0_print(f"Loading VGGT from: {config.weights_path}")
self.vggt = VGGT.from_pretrained(config.weights_path)
self.vggt.eval()
for param in self.vggt.parameters():
param.requires_grad = False
def forward(
self,
pixel_values,
modality,
attention_mask: Optional[torch.Tensor] = None,
output_attentions: Optional[bool] = None,
output_hidden_states: Optional[bool] = None,
return_dict: Optional[bool] = None,
point_cloud_output_paths: Optional[list[str]] = None # Add for API consistency if needed, though not used here
) -> Union[Tuple, BaseModelOutputWithPooling]:
# FIXME: can't process batch
views = prepare_input(pixel_values=pixel_values, modality=modality)
with torch.no_grad():
with torch.cuda.amp.autocast(dtype=views.dtype):
aggregated_tokens_list, ps_idx = self.vggt.aggregator(views)
# just need the patch tokens of the last layer
spatial_feat = aggregated_tokens_list[-1] # 1, num_frame, num_patch, token_feature
spatial_feat = spatial_feat.to(pixel_values.dtype)
spatial_feat = spatial_feat.squeeze(0) # num_frame, num_patch, token_feature
camera_token = spatial_feat[:, 0:1, :]
patch_tokens = spatial_feat[:, ps_idx:, :]
# for debug(visualize point cloud)
# pts3d_pred, pts3d_conf = self.vggt.point_head(
# aggregated_tokens_list, images=views, patch_start_idx=ps_idx
# )
return (camera_token, patch_tokens)
class VGGT_SpatialTransformer(nn.Module):
def __init__(self, config: VGGTSpatialConfig, **kwargs):
super().__init__()
self.config = config
self.encoder = VGGT_Encoder(config=config, **kwargs)
def forward(
self,
pixel_values,
output_attentions: Optional[bool] = None,
output_hidden_states: Optional[bool] = None,
return_dict: Optional[bool] = None,
modality: Optional[str] = None,
) -> Union[Tuple, BaseModelOutputWithPooling]:
r"""
Returns:
"""
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
output_hidden_states = output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
encoder_outputs = self.encoder(
pixel_values=pixel_values,
modality=modality,
output_attentions=output_attentions,
output_hidden_states=output_hidden_states,
return_dict=return_dict,
)
return encoder_outputs
class VGGT_SpatialModel(VGGTSpatialPreTrainedModel):
config_class = VGGTSpatialConfig
main_input_name = "pixel_values"
_no_split_modules = ["VGGTSpatialEncoderLayer"]
def __init__(self, config: VGGTSpatialConfig, **kwargs):
super().__init__(config)
self.spatial_model = VGGT_SpatialTransformer(config, **kwargs)
# Initialize weights and apply final processing
self.post_init()
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], config: VGGTSpatialConfig, **kwargs):
# Use the provided config object
model = cls(config=config, **kwargs)
# Potentially load state dict here if pretrained_model_name_or_path points to a checkpoint
# containing the wrapper model's state, not just the base VGGT model.
# For now, it assumes the config dictates the base model loading within VGGT_SpatialTransformer.
return model
def forward(
self,
pixel_values,
modality,
output_attentions: Optional[bool] = None,
output_hidden_states: Optional[bool] = None,
return_dict: Optional[bool] = None,
point_cloud_output_paths: Optional[list[str]] = None # Add for API consistency
) -> Union[Tuple, BaseModelOutputWithPooling]:
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
return self.spatial_model(
pixel_values=pixel_values,
modality=modality,
output_attentions=output_attentions,
output_hidden_states=output_hidden_states,
return_dict=return_dict,
# Pass point_cloud_output_paths if the underlying model needs it, though VGGT_SpatialTransformer doesn't use it currently
# point_cloud_output_paths=point_cloud_output_paths
)
class VGGTSpatialTower(nn.Module):
def __init__(self, spatial_tower, spatial_tower_cfg, delay_load=True):
super().__init__()
self.is_loaded = False
# Dynamically determine the weights path
script_dir = os.path.dirname(os.path.abspath(__file__))
# Assumes vggt repo is cloned sibling to the main project repo
# ../../../ -> goes up from multimodal_spatial_encoder -> model -> llava -> project root
root = os.path.abspath(os.path.join(script_dir, '..', '..', '..'))
# Adjust the relative path 'vggt/VGGT-1B' as needed based on where VGGT weights are stored
default_weights_name = "VGGT-1B" # Or whatever the default identifier is
dynamic_weights_path = os.path.join(root, 'vggt', default_weights_name) # Construct full path
# Create config with the dynamic path
self.config = VGGTSpatialConfig(
weights_path=dynamic_weights_path,
# Pass other relevant configs from spatial_tower_cfg if needed
# Example: output_attentions=getattr(spatial_tower_cfg, 'output_attentions', False),
)
self.spatial_tower_name = spatial_tower # Keep track of the logical name/identifier
if not delay_load:
rank0_print(f"Loading spatial tower: {spatial_tower} using weights from {self.config.weights_path}")
self.load_model()
elif getattr(spatial_tower_cfg, "unfreeze_mm_spatial_tower", False):
# TODO: better detector is needed.
rank0_print(f"The checkpoint seems to contain `spatial_tower` weights: `unfreeze_mm_spatial_tower`: True.")
self.load_model()
elif hasattr(spatial_tower_cfg, "mm_tunable_parts") and "mm_spatial_tower" in spatial_tower_cfg.mm_tunable_parts:
rank0_print(f"The checkpoint seems to contain `spatial_tower` weights: `mm_tunable_parts` contains `mm_spatial_tower`.")
self.load_model()
else:
# Store the config even if not loading immediately
self.cfg_only = self.config
def load_model(self, device_map=None):
if self.is_loaded:
rank0_print("{} is already loaded, `load_model` called again, skipping.".format(self.spatial_tower_name))
return
# Load the VGGT_SpatialModel using the config which contains the weights_path
# The spatial_tower_name might be used here if it points to a checkpoint containing the *entire* VGGTSpatialModel state,
# rather than just the base VGGT weights. Adjust logic based on how checkpoints are saved/loaded.
# Assuming spatial_tower_name is mainly an identifier and config.weights_path points to base VGGT weights.
rank0_print(f"Instantiating VGGT_SpatialModel with config pointing to: {self.config.weights_path}")
self.spatial_tower = VGGT_SpatialModel.from_pretrained(
pretrained_model_name_or_path=self.spatial_tower_name, # Or potentially self.config.weights_path if appropriate
config=self.config,
device_map=device_map
)
self.spatial_tower.requires_grad_(False)
self.is_loaded = True
def forward(self, images, modality):
if type(images) is list:
image_features = []
for image in images:
# Pass point_cloud_output_paths=None if needed by the forward signature
image_forward_out = self.spatial_tower(image.to(device=self.device, dtype=self.dtype).unsqueeze(0), output_hidden_states=True, modality=modality)
# Assuming image_forward_out is now a tuple (camera_token, patch_features)
# We likely only need the patch features for downstream tasks
patch_features = image_forward_out[1] # Get patch features
# image_feature = image_forward_out.last_hidden_state.to(image.dtype) # Original line
image_feature = patch_features.to(image.dtype)
image_features.append(image_feature)
else:
# Pass point_cloud_output_paths=None if needed by the forward signature
image_features = self.spatial_tower(images.to(device=self.device, dtype=self.dtype), output_hidden_states=True, modality=modality)
return image_features
@property
def dummy_feature(self):
return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype)
@property
def dtype(self):
for p in self.spatial_tower.parameters():
return p.dtype
@property
def device(self):
for p in self.spatial_tower.parameters():
return p.device
@property
def hidden_size(self):
return self.config.hidden_size
@property
def num_patches(self):
return (self.config.image_size // self.config.patch_size) ** 2
@property
def num_patches_per_side(self):
return self.config.image_size // self.config.patch_size
@property
def image_size(self):
return self.config.image_size