Spaces:
Runtime error
Runtime error
File size: 2,311 Bytes
22d4865 4e15d28 22d4865 4e15d28 22d4865 4e15d28 22d4865 4e15d28 c674e24 4e15d28 c674e24 4e15d28 c674e24 4e15d28 c674e24 22d4865 c674e24 22d4865 4e15d28 c674e24 4e15d28 c674e24 4e15d28 22d4865 c674e24 22d4865 c674e24 4e15d28 22d4865 4e15d28 22d4865 4e15d28 c674e24 4e15d28 c674e24 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | import re
import torch
import torch.nn.functional as F
from PIL import Image
from fastapi import Request
from .BaseControllers import BaseControllers
from transformers import CLIPProcessor, CLIPModel
class FeatureExtractionControllers(BaseControllers):
def __init__(self, request: Request):
super().__init__()
self.request = request
def process_image(
self,
image: Image,
processor: CLIPProcessor
):
input = processor(images=image, return_tensors="pt")["pixel_values"]
return input
def extract_image_features(
self,
image: Image,
model: CLIPModel,
processor: CLIPProcessor
):
input = self.process_image(image, processor)
with torch.no_grad():
image_features = model.get_image_features(input)
# convert the features to unit vector
# cosine similarity only care about the angle between the feature vectors
image_features = F.normalize(image_features, dim=-1)
return image_features
def clean_text(
self,
text: str
):
# Remove punctuation
text = re.sub(r'[^\w\s]', ' ', text)
# Remove extra spaces
text = re.sub(r'\s+', ' ', text).strip()
# Remove non-alphabetic characters
text = re.sub(r'[^a-zA-Z\s]', '', text)
doc = self.request.app.nlp(text.lower())
tokens = [
token.lemma_
for token in doc
]
return " ".join(tokens)
def text_processor(
self,
description: str,
processor: CLIPProcessor
):
description = self.clean_text(description)
input = processor(text=description, return_tensors="pt")["input_ids"]
return input
def extract_text_features(
self,
description: str,
model: CLIPModel,
processor: CLIPProcessor
):
input = self.text_processor(description, processor)
with torch.no_grad():
text_features = model.get_text_features(input)
# convert the features to unit vector
# cosine similarity only care about the angle between the feature vectors
text_features = F.normalize(text_features, dim=-1)
return text_features
|