File size: 2,914 Bytes
a2c5870
7dd5a74
b66ea6d
7dd5a74
37c9551
 
7dd5a74
 
a2c5870
 
 
e046c4e
b66ea6d
a2c5870
 
b66ea6d
 
a2c5870
b66ea6d
7dd5a74
a2c5870
 
e046c4e
37c9551
 
 
7dd5a74
37c9551
7dd5a74
e046c4e
7dd5a74
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e046c4e
 
 
 
 
 
 
7dd5a74
 
e046c4e
 
 
 
 
 
7dd5a74
 
e046c4e
 
 
7dd5a74
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
from typing import Dict, List, Any
from transformers import AutoProcessor, AutoModel
import torch
import torch.nn.functional as F
from PIL import Image
import requests
import base64
from io import BytesIO

class EndpointHandler:
    def __init__(self, path=""):
        self.processor = AutoProcessor.from_pretrained(path, trust_remote_code=True, use_fast=True)
        self.model = AutoModel.from_pretrained(
            path,
            trust_remote_code=True,
            torch_dtype=torch.float16,
            device_map="auto"
        )
        self.model.eval()
        self.query_text = "This image shows a component that is used in HVAC system"

    def __call__(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
        inputs_data = data.get("inputs", [])
        if not isinstance(inputs_data, list):
            inputs_data = [inputs_data]

        results = []
        for item in inputs_data:
            if isinstance(item, dict):
                image = item.get("image")
                text = item.get("text", self.query_text)
            else:
                image = None
                text = item if isinstance(item, str) else self.query_text

            if image:
                if isinstance(image, str) and image.startswith("data:image"):
                    img_data = image.split(",")[1]
                    pil_img = Image.open(BytesIO(base64.b64decode(img_data)))
                elif isinstance(image, str):
                    pil_img = Image.open(requests.get(image, stream=True).raw)
                else:
                    pil_img = image

                messages = [{"role": "user", "content": [
                    {"type": "image", "image": pil_img},
                    {"type": "text", "text": text},
                ]}]
                prompt = self.processor.apply_chat_template(
                    messages, tokenize=False, add_generation_prompt=False
                )
                inputs = self.processor(
                    text=[prompt], images=[pil_img],
                    return_tensors="pt", padding=True
                ).to(self.model.device)
            else:
                messages = [{"role": "user", "content": [{"type": "text", "text": text}]}]
                prompt = self.processor.apply_chat_template(
                    messages, tokenize=False, add_generation_prompt=False
                )
                inputs = self.processor(
                    text=[prompt], return_tensors="pt", padding=True
                ).to(self.model.device)

            with torch.no_grad():
                out = self.model(**inputs, output_hidden_states=True)
                hidden = out.last_hidden_state if hasattr(out, 'last_hidden_state') else out[0]
                emb = hidden.mean(dim=1)
                emb = F.normalize(emb, p=2, dim=-1)

            results.append({"embedding": emb[0].cpu().float().tolist()})

        return results