Buckets:
| import torch | |
| import torch.nn as nn | |
| from transformers import SiglipProcessor, SiglipModel | |
| from typing import List | |
| from PIL import Image | |
| def _as_embedding(output) -> torch.Tensor: | |
| """Return the pooled embedding tensor from a SigLIP feature call. | |
| transformers <5 returned the pooled tensor directly from | |
| `get_image_features` / `get_text_features`; transformers >=5 returns a | |
| `BaseModelOutputWithPooling`. Support both. | |
| """ | |
| if isinstance(output, torch.Tensor): | |
| return output | |
| for attr in ("pooler_output", "last_hidden_state"): | |
| value = getattr(output, attr, None) | |
| if isinstance(value, torch.Tensor): | |
| return value if attr == "pooler_output" else value.mean(dim=1) | |
| raise TypeError(f"unexpected SigLIP output type: {type(output).__name__}") | |
| class FrozenSiglipEncoder(nn.Module): | |
| def __init__(self, ckpt: str = "google/siglip-so400m-patch14-384", device: torch.device = torch.device("cpu")): | |
| super().__init__() | |
| self.device = device | |
| # Load SigLIP model and processor | |
| self.model = SiglipModel.from_pretrained(ckpt).to(device).eval() | |
| self.processor = SiglipProcessor.from_pretrained(ckpt) | |
| # Freeze parameters | |
| for p in self.model.parameters(): | |
| p.requires_grad_(False) | |
| def encode_text(self, texts: List[str]) -> torch.Tensor: | |
| """ | |
| Convert texts to SigLIP embeddings. | |
| Returns: (B, L) tensor, where L depends on the model variant (e.g., 768 or 1152) | |
| """ | |
| inputs = self.processor(text=texts, | |
| return_tensors="pt", | |
| padding="max_length", | |
| max_length=64, | |
| truncation=True).to(self.device) | |
| with torch.no_grad(): | |
| # SigLIP uses get_text_features just like CLIP | |
| text_embeds = self.model.get_text_features(**inputs) | |
| return _as_embedding(text_embeds) | |
| def encode_image(self, images: List[Image.Image]) -> torch.Tensor: | |
| """ | |
| Convert PIL images to SigLIP embeddings. | |
| """ | |
| inputs = self.processor.image_processor( | |
| images=images, | |
| return_tensors="pt", | |
| do_rescale=False | |
| ).to(self.device) | |
| with torch.no_grad(): | |
| image_embeds = self.model.get_image_features(**inputs) | |
| return _as_embedding(image_embeds) |
Xet Storage Details
- Size:
- 2.57 kB
- Xet hash:
- edc33a17445e67a8f8ab0c4da956cfbfc8a75bdbeb5ad768b37068d671605fbf
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.