TRELLIS.2 / src /trellis2 /modules /image_feature_extractor.py
Benjamin Auquite
Match image conditioner input dtype
8e436bb verified
Raw History Blame Contribute Delete
5.68 kB
import os
from typing import *
import torch
import torch.nn.functional as F
from torchvision import transforms
from transformers import DINOv3ViTModel
import numpy as np
from PIL import Image
DINO_V3_FALLBACK_MODELS = {
"facebook/dinov3-vitl16-pretrain-lvd1689m": "camenduru/dinov3-vitl16-pretrain-lvd1689m",
}
class DinoV2FeatureExtractor:
"""
Feature extractor for DINOv2 models.
"""
def __init__(self, model_name: str):
self.model_name = model_name
self.model = torch.hub.load(
"facebookresearch/dinov2", model_name, pretrained=True
)
self.model.eval()
self.transform = transforms.Compose(
[
transforms.Normalize(
mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
),
]
)
def to(self, device):
self.model.to(device)
def cuda(self):
self.model.cuda()
def cpu(self):
self.model.cpu()
def _prepare_image_tensor(
self, image: Union[torch.Tensor, List[Image.Image]]
) -> torch.Tensor:
model_device = next(self.model.parameters()).device
model_dtype = next(self.model.parameters()).dtype
if isinstance(image, torch.Tensor):
assert image.ndim == 4, "Image tensor should be batched (B, C, H, W)"
elif isinstance(image, list):
assert all(isinstance(i, Image.Image) for i in image), (
"Image list should be list of PIL images"
)
image = [i.resize((518, 518), Image.LANCZOS) for i in image]
image = [np.array(i.convert("RGB")).astype(np.float32) / 255 for i in image]
image = [torch.from_numpy(i).permute(2, 0, 1).float() for i in image]
image = torch.stack(image)
else:
raise ValueError(f"Unsupported type of image: {type(image)}")
image = self.transform(image.cpu()).to(device=model_device, dtype=model_dtype)
return image
@torch.no_grad()
def __call__(self, image: Union[torch.Tensor, List[Image.Image]]) -> torch.Tensor:
"""
Extract features from the image.
Args:
image: A batch of images as a tensor of shape (B, C, H, W) or a list of PIL images.
Returns:
A tensor of shape (B, N, D) where N is the number of patches and D is the feature dimension.
"""
image = self._prepare_image_tensor(image)
features = self.model(image, is_training=True)["x_prenorm"]
patchtokens = F.layer_norm(features, features.shape[-1:])
return patchtokens
class DinoV3FeatureExtractor:
"""
Feature extractor for DINOv3 models.
"""
def __init__(self, model_name: str, image_size=512):
requested_model_name = os.getenv("TRELLIS_DINOV3_MODEL", model_name)
fallback_model_name = DINO_V3_FALLBACK_MODELS.get(requested_model_name)
self.model_name = requested_model_name
try:
self.model = DINOv3ViTModel.from_pretrained(requested_model_name)
except OSError as exc:
gated_error = (
"gated repo" in str(exc).lower()
or "403 client error" in str(exc).lower()
)
if not gated_error or fallback_model_name is None:
raise
self.model = DINOv3ViTModel.from_pretrained(fallback_model_name)
self.model_name = fallback_model_name
self.model.eval()
self.image_size = image_size
self.transform = transforms.Compose(
[
transforms.Normalize(
mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
),
]
)
def to(self, device):
self.model.to(device)
def cuda(self):
self.model.cuda()
def cpu(self):
self.model.cpu()
def _prepare_image_tensor(
self, image: Union[torch.Tensor, List[Image.Image]]
) -> torch.Tensor:
model_device = next(self.model.parameters()).device
model_dtype = next(self.model.parameters()).dtype
if isinstance(image, torch.Tensor):
assert image.ndim == 4, "Image tensor should be batched (B, C, H, W)"
elif isinstance(image, list):
assert all(isinstance(i, Image.Image) for i in image), (
"Image list should be list of PIL images"
)
image = [
i.resize((self.image_size, self.image_size), Image.LANCZOS)
for i in image
]
image = [np.array(i.convert("RGB")).astype(np.float32) / 255 for i in image]
image = [torch.from_numpy(i).permute(2, 0, 1).float() for i in image]
image = torch.stack(image)
else:
raise ValueError(f"Unsupported type of image: {type(image)}")
image = self.transform(image.cpu()).to(device=model_device, dtype=model_dtype)
return image
def extract_features(self, image: torch.Tensor) -> torch.Tensor:
outputs = self.model(image, bool_masked_pos=None)
return outputs.last_hidden_state
@torch.no_grad()
def __call__(self, image: Union[torch.Tensor, List[Image.Image]]) -> torch.Tensor:
"""
Extract features from the image.
Args:
image: A batch of images as a tensor of shape (B, C, H, W) or a list of PIL images.
Returns:
A tensor of shape (B, N, D) where N is the number of patches and D is the feature dimension.
"""
image = self._prepare_image_tensor(image)
features = self.extract_features(image)
return features