Nadeem-Shoukath115113's picture
download
raw
2.57 kB
import torch
import torch.nn as nn
from transformers import SiglipProcessor, SiglipModel
from typing import List
from PIL import Image
def _as_embedding(output) -> torch.Tensor:
"""Return the pooled embedding tensor from a SigLIP feature call.
transformers <5 returned the pooled tensor directly from
`get_image_features` / `get_text_features`; transformers >=5 returns a
`BaseModelOutputWithPooling`. Support both.
"""
if isinstance(output, torch.Tensor):
return output
for attr in ("pooler_output", "last_hidden_state"):
value = getattr(output, attr, None)
if isinstance(value, torch.Tensor):
return value if attr == "pooler_output" else value.mean(dim=1)
raise TypeError(f"unexpected SigLIP output type: {type(output).__name__}")
class FrozenSiglipEncoder(nn.Module):
def __init__(self, ckpt: str = "google/siglip-so400m-patch14-384", device: torch.device = torch.device("cpu")):
super().__init__()
self.device = device
# Load SigLIP model and processor
self.model = SiglipModel.from_pretrained(ckpt).to(device).eval()
self.processor = SiglipProcessor.from_pretrained(ckpt)
# Freeze parameters
for p in self.model.parameters():
p.requires_grad_(False)
def encode_text(self, texts: List[str]) -> torch.Tensor:
"""
Convert texts to SigLIP embeddings.
Returns: (B, L) tensor, where L depends on the model variant (e.g., 768 or 1152)
"""
inputs = self.processor(text=texts,
return_tensors="pt",
padding="max_length",
max_length=64,
truncation=True).to(self.device)
with torch.no_grad():
# SigLIP uses get_text_features just like CLIP
text_embeds = self.model.get_text_features(**inputs)
return _as_embedding(text_embeds)
def encode_image(self, images: List[Image.Image]) -> torch.Tensor:
"""
Convert PIL images to SigLIP embeddings.
"""
inputs = self.processor.image_processor(
images=images,
return_tensors="pt",
do_rescale=False
).to(self.device)
with torch.no_grad():
image_embeds = self.model.get_image_features(**inputs)
return _as_embedding(image_embeds)

Xet Storage Details

Size:
2.57 kB
·
Xet hash:
edc33a17445e67a8f8ab0c4da956cfbfc8a75bdbeb5ad768b37068d671605fbf

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.