Nadeem-Shoukath115113's picture
download
raw
1.33 kB
import torch
import torch.nn as nn
from transformers import CLIPProcessor, CLIPModel
from typing import List
from PIL import Image
class FrozenCLIPEncoder(nn.Module):
def __init__(self, ckpt: str, device: torch.device):
super().__init__()
self.device = device
self.model = CLIPModel.from_pretrained(ckpt).to(device).eval()
self.processor = CLIPProcessor.from_pretrained(ckpt)
for p in self.model.parameters():
p.requires_grad_(False)
def encode_text(self, texts: List[str]) -> torch.Tensor:
"""
texts: list[str], length B
returns: (B, 512) CLIP text embeddings
"""
inputs = self.processor(text=texts, return_tensors="pt", padding=True, truncation=True).to(self.device)
with torch.no_grad():
text_embeds = self.model.get_text_features(**inputs)
return text_embeds
def encode_image(self, images: List[Image.Image], do_rescale=False) -> torch.Tensor:
"""
images: list of PIL Images, length B
returns: (B, 512) CLIP image embeddings
"""
inputs = self.processor(images=images, return_tensors="pt", do_rescale=do_rescale).to(self.device)
with torch.no_grad():
image_embeds = self.model.get_image_features(**inputs)
return image_embeds

Xet Storage Details

Size:
1.33 kB
·
Xet hash:
ab1df4629c3efc178d55f88976dbe45d93e6f4c1337b087e00856e8da348d9aa

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.