from fastapi import FastAPI from pydantic import BaseModel from typing import Any, List, Dict from transformers import AutoProcessor, AutoModel import torch import torch.nn.functional as F from PIL import Image import requests import base64 from io import BytesIO import os app = FastAPI(title="Vision Embedding API") MODEL_ID = "saveenergysystems/vision-embedding-model-SES" processor = AutoProcessor.from_pretrained(MODEL_ID, trust_remote_code=True) model = AutoModel.from_pretrained( MODEL_ID, trust_remote_code=True, torch_dtype=torch.bfloat16, device_map="auto", load_in_8bit=True, attn_implementation="sdpa" ) model.eval() class Request(BaseModel): inputs: Any @app.get("/health") def health(): return {"status": "ok"} @app.post("/embed") def embed(request: Request): inputs_data = request.inputs if not isinstance(inputs_data, list): inputs_data = [inputs_data] results = [] for item in inputs_data: if isinstance(item, dict): image = item.get("image") text = item.get("text", "This image shows a component that is used in HVAC system") else: image = None text = item if isinstance(item, str) else "This image shows a component that is used in HVAC system" # Image processing pil_img = None if image: if isinstance(image, str) and image.startswith("data:image"): img_data = image.split(",", 1)[1] pil_img = Image.open(BytesIO(base64.b64decode(img_data))) elif isinstance(image, str): pil_img = Image.open(requests.get(image, stream=True).raw) if pil_img: messages = [{"role": "user", "content": [ {"type": "image", "image": pil_img}, {"type": "text", "text": text} ]}] prompt = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=False) inputs = processor(text=[prompt], images=[pil_img], return_tensors="pt", padding=True) else: messages = [{"role": "user", "content": [{"type": "text", "text": text}]}] prompt = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=False) inputs = processor(text=[prompt], return_tensors="pt", padding=True) inputs = inputs.to(model.device) with torch.no_grad(): out = model(**inputs, output_hidden_states=True) hidden = out.hidden_states[-1] if hasattr(out, 'hidden_states') else out.last_hidden_state emb = hidden.mean(dim=1) emb = F.normalize(emb, p=2, dim=-1) results.append({"embedding": emb[0].cpu().float().tolist()}) return results