File size: 2,914 Bytes
a2c5870 7dd5a74 b66ea6d 7dd5a74 37c9551 7dd5a74 a2c5870 e046c4e b66ea6d a2c5870 b66ea6d a2c5870 b66ea6d 7dd5a74 a2c5870 e046c4e 37c9551 7dd5a74 37c9551 7dd5a74 e046c4e 7dd5a74 e046c4e 7dd5a74 e046c4e 7dd5a74 e046c4e 7dd5a74 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | from typing import Dict, List, Any
from transformers import AutoProcessor, AutoModel
import torch
import torch.nn.functional as F
from PIL import Image
import requests
import base64
from io import BytesIO
class EndpointHandler:
def __init__(self, path=""):
self.processor = AutoProcessor.from_pretrained(path, trust_remote_code=True, use_fast=True)
self.model = AutoModel.from_pretrained(
path,
trust_remote_code=True,
torch_dtype=torch.float16,
device_map="auto"
)
self.model.eval()
self.query_text = "This image shows a component that is used in HVAC system"
def __call__(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
inputs_data = data.get("inputs", [])
if not isinstance(inputs_data, list):
inputs_data = [inputs_data]
results = []
for item in inputs_data:
if isinstance(item, dict):
image = item.get("image")
text = item.get("text", self.query_text)
else:
image = None
text = item if isinstance(item, str) else self.query_text
if image:
if isinstance(image, str) and image.startswith("data:image"):
img_data = image.split(",")[1]
pil_img = Image.open(BytesIO(base64.b64decode(img_data)))
elif isinstance(image, str):
pil_img = Image.open(requests.get(image, stream=True).raw)
else:
pil_img = image
messages = [{"role": "user", "content": [
{"type": "image", "image": pil_img},
{"type": "text", "text": text},
]}]
prompt = self.processor.apply_chat_template(
messages, tokenize=False, add_generation_prompt=False
)
inputs = self.processor(
text=[prompt], images=[pil_img],
return_tensors="pt", padding=True
).to(self.model.device)
else:
messages = [{"role": "user", "content": [{"type": "text", "text": text}]}]
prompt = self.processor.apply_chat_template(
messages, tokenize=False, add_generation_prompt=False
)
inputs = self.processor(
text=[prompt], return_tensors="pt", padding=True
).to(self.model.device)
with torch.no_grad():
out = self.model(**inputs, output_hidden_states=True)
hidden = out.last_hidden_state if hasattr(out, 'last_hidden_state') else out[0]
emb = hidden.mean(dim=1)
emb = F.normalize(emb, p=2, dim=-1)
results.append({"embedding": emb[0].cpu().float().tolist()})
return results |