Download handler.py from saveenergysystems/vision-embedding-model-SES: direct link, hf CLI and curl.
- Browser
- Download file 2.91 kB
-
https://huggingface.co/saveenergysystems/vision-embedding-model-SES/resolve/main/handler.py
- Command line
-
hf download hf://saveenergysystems/vision-embedding-model-SES/handler.py
-
curl -L -o handler.py https://huggingface.co/saveenergysystems/vision-embedding-model-SES/resolve/main/handler.py
2.91 kB
| from typing import Dict, List, Any | |
| from transformers import AutoProcessor, AutoModel | |
| import torch | |
| import torch.nn.functional as F | |
| from PIL import Image | |
| import requests | |
| import base64 | |
| from io import BytesIO | |
| class EndpointHandler: | |
| def __init__(self, path=""): | |
| self.processor = AutoProcessor.from_pretrained(path, trust_remote_code=True, use_fast=True) | |
| self.model = AutoModel.from_pretrained( | |
| path, | |
| trust_remote_code=True, | |
| torch_dtype=torch.float16, | |
| device_map="auto" | |
| ) | |
| self.model.eval() | |
| self.query_text = "This image shows a component that is used in HVAC system" | |
| def __call__(self, data: Dict[str, Any]) -> List[Dict[str, Any]]: | |
| inputs_data = data.get("inputs", []) | |
| if not isinstance(inputs_data, list): | |
| inputs_data = [inputs_data] | |
| results = [] | |
| for item in inputs_data: | |
| if isinstance(item, dict): | |
| image = item.get("image") | |
| text = item.get("text", self.query_text) | |
| else: | |
| image = None | |
| text = item if isinstance(item, str) else self.query_text | |
| if image: | |
| if isinstance(image, str) and image.startswith("data:image"): | |
| img_data = image.split(",")[1] | |
| pil_img = Image.open(BytesIO(base64.b64decode(img_data))) | |
| elif isinstance(image, str): | |
| pil_img = Image.open(requests.get(image, stream=True).raw) | |
| else: | |
| pil_img = image | |
| messages = [{"role": "user", "content": [ | |
| {"type": "image", "image": pil_img}, | |
| {"type": "text", "text": text}, | |
| ]}] | |
| prompt = self.processor.apply_chat_template( | |
| messages, tokenize=False, add_generation_prompt=False | |
| ) | |
| inputs = self.processor( | |
| text=[prompt], images=[pil_img], | |
| return_tensors="pt", padding=True | |
| ).to(self.model.device) | |
| else: | |
| messages = [{"role": "user", "content": [{"type": "text", "text": text}]}] | |
| prompt = self.processor.apply_chat_template( | |
| messages, tokenize=False, add_generation_prompt=False | |
| ) | |
| inputs = self.processor( | |
| text=[prompt], return_tensors="pt", padding=True | |
| ).to(self.model.device) | |
| with torch.no_grad(): | |
| out = self.model(**inputs, output_hidden_states=True) | |
| hidden = out.last_hidden_state if hasattr(out, 'last_hidden_state') else out[0] | |
| emb = hidden.mean(dim=1) | |
| emb = F.normalize(emb, p=2, dim=-1) | |
| results.append({"embedding": emb[0].cpu().float().tolist()}) | |
| return results |