developerses's picture
updated
0cd59dc verified
Raw History Blame Contribute Delete
2.85 kB
from fastapi import FastAPI
from pydantic import BaseModel
from typing import Any, List, Dict
from transformers import AutoProcessor, AutoModel
import torch
import torch.nn.functional as F
from PIL import Image
import requests
import base64
from io import BytesIO
import os
app = FastAPI(title="Vision Embedding API")
MODEL_ID = "saveenergysystems/vision-embedding-model-SES"
processor = AutoProcessor.from_pretrained(MODEL_ID, trust_remote_code=True)
model = AutoModel.from_pretrained(
MODEL_ID,
trust_remote_code=True,
torch_dtype=torch.bfloat16,
device_map="auto",
load_in_8bit=True,
attn_implementation="sdpa"
)
model.eval()
class Request(BaseModel):
inputs: Any
@app.get("/health")
def health():
return {"status": "ok"}
@app.post("/embed")
def embed(request: Request):
inputs_data = request.inputs
if not isinstance(inputs_data, list):
inputs_data = [inputs_data]
results = []
for item in inputs_data:
if isinstance(item, dict):
image = item.get("image")
text = item.get("text", "This image shows a component that is used in HVAC system")
else:
image = None
text = item if isinstance(item, str) else "This image shows a component that is used in HVAC system"
# Image processing
pil_img = None
if image:
if isinstance(image, str) and image.startswith("data:image"):
img_data = image.split(",", 1)[1]
pil_img = Image.open(BytesIO(base64.b64decode(img_data)))
elif isinstance(image, str):
pil_img = Image.open(requests.get(image, stream=True).raw)
if pil_img:
messages = [{"role": "user", "content": [
{"type": "image", "image": pil_img},
{"type": "text", "text": text}
]}]
prompt = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
inputs = processor(text=[prompt], images=[pil_img], return_tensors="pt", padding=True)
else:
messages = [{"role": "user", "content": [{"type": "text", "text": text}]}]
prompt = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
inputs = processor(text=[prompt], return_tensors="pt", padding=True)
inputs = inputs.to(model.device)
with torch.no_grad():
out = model(**inputs, output_hidden_states=True)
hidden = out.hidden_states[-1] if hasattr(out, 'hidden_states') else out.last_hidden_state
emb = hidden.mean(dim=1)
emb = F.normalize(emb, p=2, dim=-1)
results.append({"embedding": emb[0].cpu().float().tolist()})
return results