Spaces:
Paused
Paused
Download src/trellis2/modules/image_feature_extractor.py from jackmowden/TRELLIS.2: direct link, hf CLI and curl.
- Browser
- Download file 5.68 kB
-
https://huggingface.co/spaces/jackmowden/TRELLIS.2/resolve/main/src/trellis2/modules/image_feature_extractor.py
- Command line
-
hf download hf://spaces/jackmowden/TRELLIS.2/src/trellis2/modules/image_feature_extractor.py
-
curl -L -o image_feature_extractor.py https://huggingface.co/spaces/jackmowden/TRELLIS.2/resolve/main/src/trellis2/modules/image_feature_extractor.py
5.68 kB
| import os | |
| from typing import * | |
| import torch | |
| import torch.nn.functional as F | |
| from torchvision import transforms | |
| from transformers import DINOv3ViTModel | |
| import numpy as np | |
| from PIL import Image | |
| DINO_V3_FALLBACK_MODELS = { | |
| "facebook/dinov3-vitl16-pretrain-lvd1689m": "camenduru/dinov3-vitl16-pretrain-lvd1689m", | |
| } | |
| class DinoV2FeatureExtractor: | |
| """ | |
| Feature extractor for DINOv2 models. | |
| """ | |
| def __init__(self, model_name: str): | |
| self.model_name = model_name | |
| self.model = torch.hub.load( | |
| "facebookresearch/dinov2", model_name, pretrained=True | |
| ) | |
| self.model.eval() | |
| self.transform = transforms.Compose( | |
| [ | |
| transforms.Normalize( | |
| mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] | |
| ), | |
| ] | |
| ) | |
| def to(self, device): | |
| self.model.to(device) | |
| def cuda(self): | |
| self.model.cuda() | |
| def cpu(self): | |
| self.model.cpu() | |
| def _prepare_image_tensor( | |
| self, image: Union[torch.Tensor, List[Image.Image]] | |
| ) -> torch.Tensor: | |
| model_device = next(self.model.parameters()).device | |
| model_dtype = next(self.model.parameters()).dtype | |
| if isinstance(image, torch.Tensor): | |
| assert image.ndim == 4, "Image tensor should be batched (B, C, H, W)" | |
| elif isinstance(image, list): | |
| assert all(isinstance(i, Image.Image) for i in image), ( | |
| "Image list should be list of PIL images" | |
| ) | |
| image = [i.resize((518, 518), Image.LANCZOS) for i in image] | |
| image = [np.array(i.convert("RGB")).astype(np.float32) / 255 for i in image] | |
| image = [torch.from_numpy(i).permute(2, 0, 1).float() for i in image] | |
| image = torch.stack(image) | |
| else: | |
| raise ValueError(f"Unsupported type of image: {type(image)}") | |
| image = self.transform(image.cpu()).to(device=model_device, dtype=model_dtype) | |
| return image | |
| def __call__(self, image: Union[torch.Tensor, List[Image.Image]]) -> torch.Tensor: | |
| """ | |
| Extract features from the image. | |
| Args: | |
| image: A batch of images as a tensor of shape (B, C, H, W) or a list of PIL images. | |
| Returns: | |
| A tensor of shape (B, N, D) where N is the number of patches and D is the feature dimension. | |
| """ | |
| image = self._prepare_image_tensor(image) | |
| features = self.model(image, is_training=True)["x_prenorm"] | |
| patchtokens = F.layer_norm(features, features.shape[-1:]) | |
| return patchtokens | |
| class DinoV3FeatureExtractor: | |
| """ | |
| Feature extractor for DINOv3 models. | |
| """ | |
| def __init__(self, model_name: str, image_size=512): | |
| requested_model_name = os.getenv("TRELLIS_DINOV3_MODEL", model_name) | |
| fallback_model_name = DINO_V3_FALLBACK_MODELS.get(requested_model_name) | |
| self.model_name = requested_model_name | |
| try: | |
| self.model = DINOv3ViTModel.from_pretrained(requested_model_name) | |
| except OSError as exc: | |
| gated_error = ( | |
| "gated repo" in str(exc).lower() | |
| or "403 client error" in str(exc).lower() | |
| ) | |
| if not gated_error or fallback_model_name is None: | |
| raise | |
| self.model = DINOv3ViTModel.from_pretrained(fallback_model_name) | |
| self.model_name = fallback_model_name | |
| self.model.eval() | |
| self.image_size = image_size | |
| self.transform = transforms.Compose( | |
| [ | |
| transforms.Normalize( | |
| mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] | |
| ), | |
| ] | |
| ) | |
| def to(self, device): | |
| self.model.to(device) | |
| def cuda(self): | |
| self.model.cuda() | |
| def cpu(self): | |
| self.model.cpu() | |
| def _prepare_image_tensor( | |
| self, image: Union[torch.Tensor, List[Image.Image]] | |
| ) -> torch.Tensor: | |
| model_device = next(self.model.parameters()).device | |
| model_dtype = next(self.model.parameters()).dtype | |
| if isinstance(image, torch.Tensor): | |
| assert image.ndim == 4, "Image tensor should be batched (B, C, H, W)" | |
| elif isinstance(image, list): | |
| assert all(isinstance(i, Image.Image) for i in image), ( | |
| "Image list should be list of PIL images" | |
| ) | |
| image = [ | |
| i.resize((self.image_size, self.image_size), Image.LANCZOS) | |
| for i in image | |
| ] | |
| image = [np.array(i.convert("RGB")).astype(np.float32) / 255 for i in image] | |
| image = [torch.from_numpy(i).permute(2, 0, 1).float() for i in image] | |
| image = torch.stack(image) | |
| else: | |
| raise ValueError(f"Unsupported type of image: {type(image)}") | |
| image = self.transform(image.cpu()).to(device=model_device, dtype=model_dtype) | |
| return image | |
| def extract_features(self, image: torch.Tensor) -> torch.Tensor: | |
| outputs = self.model(image, bool_masked_pos=None) | |
| return outputs.last_hidden_state | |
| def __call__(self, image: Union[torch.Tensor, List[Image.Image]]) -> torch.Tensor: | |
| """ | |
| Extract features from the image. | |
| Args: | |
| image: A batch of images as a tensor of shape (B, C, H, W) or a list of PIL images. | |
| Returns: | |
| A tensor of shape (B, N, D) where N is the number of patches and D is the feature dimension. | |
| """ | |
| image = self._prepare_image_tensor(image) | |
| features = self.extract_features(image) | |
| return features | |