Update app/main.py
Browse files- app/main.py +139 -93
app/main.py
CHANGED
|
@@ -1,134 +1,180 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
from fastapi.middleware.cors import CORSMiddleware
|
| 4 |
import os
|
|
|
|
|
|
|
| 5 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
from sentence_transformers import SentenceTransformer
|
| 7 |
from transformers import AutoModel, AutoTokenizer
|
| 8 |
-
import torch
|
| 9 |
-
|
| 10 |
-
app = FastAPI()
|
| 11 |
|
| 12 |
-
app.add_middleware(
|
| 13 |
-
CORSMiddleware,
|
| 14 |
-
allow_origins=["*"],
|
| 15 |
-
allow_methods=["*"],
|
| 16 |
-
allow_headers=["*"],
|
| 17 |
-
)
|
| 18 |
|
| 19 |
-
MODEL_PATH = "/app/model"
|
| 20 |
DEVICE = os.getenv("DEVICE", "cpu")
|
| 21 |
-
|
| 22 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
hf_model = None
|
| 24 |
hf_tokenizer = None
|
|
|
|
| 25 |
|
| 26 |
|
| 27 |
-
def
|
| 28 |
-
if task == "query":
|
| 29 |
-
return f"query: {text}"
|
| 30 |
-
return f"passage: {text}"
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
@app.get("/")
|
| 34 |
-
async def root():
|
| 35 |
-
return {"status": "running", "msg": "embedding server is alive"}
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
@app.on_event("startup")
|
| 39 |
-
async def load_model():
|
| 40 |
global st_model, hf_model, hf_tokenizer
|
| 41 |
try:
|
| 42 |
-
print(f"Trying to load SentenceTransformer model from {MODEL_PATH} on {DEVICE}")
|
| 43 |
st_model = SentenceTransformer(MODEL_PATH, device=DEVICE)
|
| 44 |
-
|
| 45 |
-
except Exception
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
hf_tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)
|
| 50 |
-
hf_model = AutoModel.from_pretrained(MODEL_PATH).to(DEVICE)
|
| 51 |
-
hf_model.eval()
|
| 52 |
-
print("✅ Loaded as HuggingFace AutoModel")
|
| 53 |
-
except Exception as e2:
|
| 54 |
-
print(f"❌ Failed to load model in both modes: {e2}")
|
| 55 |
|
| 56 |
|
| 57 |
-
@
|
| 58 |
-
async def
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
|
|
|
|
|
|
|
|
|
| 62 |
|
| 63 |
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
|
| 68 |
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
if st_model:
|
| 74 |
-
embedding = st_model.encode(
|
| 75 |
-
text,
|
| 76 |
-
normalize_embeddings=True,
|
| 77 |
-
convert_to_numpy=True
|
| 78 |
-
).tolist()
|
| 79 |
|
| 80 |
-
elif hf_model and hf_tokenizer:
|
| 81 |
-
inputs = hf_tokenizer(text, return_tensors="pt").to(DEVICE)
|
| 82 |
|
| 83 |
-
|
| 84 |
-
|
|
|
|
| 85 |
|
| 86 |
-
# mean pooling
|
| 87 |
-
embedding = outputs.last_hidden_state.mean(dim=1).squeeze().cpu().tolist()
|
| 88 |
|
| 89 |
-
|
| 90 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
|
| 92 |
-
return {
|
| 93 |
-
"embedding": embedding,
|
| 94 |
-
"dim": len(embedding),
|
| 95 |
-
}
|
| 96 |
|
|
|
|
|
|
|
| 97 |
|
| 98 |
-
class BatchRequest(BaseModel):
|
| 99 |
-
inputs: list[str]
|
| 100 |
-
task: str = "query"
|
| 101 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
|
| 103 |
-
@app.post("/embed/batch")
|
| 104 |
-
async def embed_batch(request: BatchRequest):
|
| 105 |
-
texts = [add_prefix(t, request.task) for t in request.inputs]
|
| 106 |
|
| 107 |
-
|
| 108 |
-
|
|
|
|
| 109 |
texts,
|
|
|
|
| 110 |
normalize_embeddings=True,
|
| 111 |
-
convert_to_numpy=True
|
|
|
|
| 112 |
).tolist()
|
| 113 |
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 121 |
|
| 122 |
-
with torch.no_grad():
|
| 123 |
-
outputs = hf_model(**inputs)
|
| 124 |
|
| 125 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 126 |
|
| 127 |
-
else:
|
| 128 |
-
return {"error": "model not loaded"}
|
| 129 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 130 |
return {
|
| 131 |
"embeddings": vectors,
|
| 132 |
"count": len(vectors),
|
| 133 |
-
"dim": len(vectors[0])
|
| 134 |
-
}
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
import hmac
|
|
|
|
| 3 |
import os
|
| 4 |
+
from contextlib import asynccontextmanager
|
| 5 |
+
from typing import Literal
|
| 6 |
|
| 7 |
+
import torch
|
| 8 |
+
import torch.nn.functional as F
|
| 9 |
+
from fastapi import FastAPI, Header, HTTPException
|
| 10 |
+
from fastapi.middleware.cors import CORSMiddleware
|
| 11 |
+
from pydantic import BaseModel, Field
|
| 12 |
from sentence_transformers import SentenceTransformer
|
| 13 |
from transformers import AutoModel, AutoTokenizer
|
|
|
|
|
|
|
|
|
|
| 14 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
+
MODEL_PATH = os.getenv("MODEL_PATH", "/app/model")
|
| 17 |
DEVICE = os.getenv("DEVICE", "cpu")
|
| 18 |
+
MAX_BATCH_SIZE = int(os.getenv("MAX_BATCH_SIZE", "64"))
|
| 19 |
+
MAX_LENGTH = int(os.getenv("MAX_LENGTH", "512"))
|
| 20 |
+
EXPECTED_DIMENSION = 384
|
| 21 |
+
API_TOKEN = os.getenv("EMBEDDING_API_TOKEN")
|
| 22 |
+
ALLOWED_ORIGINS = [
|
| 23 |
+
origin.strip()
|
| 24 |
+
for origin in os.getenv("ALLOWED_ORIGINS", "").split(",")
|
| 25 |
+
if origin.strip()
|
| 26 |
+
]
|
| 27 |
+
|
| 28 |
+
st_model: SentenceTransformer | None = None
|
| 29 |
hf_model = None
|
| 30 |
hf_tokenizer = None
|
| 31 |
+
encode_lock: asyncio.Lock | None = None
|
| 32 |
|
| 33 |
|
| 34 |
+
def _load_model() -> None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
global st_model, hf_model, hf_tokenizer
|
| 36 |
try:
|
|
|
|
| 37 |
st_model = SentenceTransformer(MODEL_PATH, device=DEVICE)
|
| 38 |
+
st_model.max_seq_length = MAX_LENGTH
|
| 39 |
+
except Exception:
|
| 40 |
+
hf_tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)
|
| 41 |
+
hf_model = AutoModel.from_pretrained(MODEL_PATH).to(DEVICE)
|
| 42 |
+
hf_model.eval()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
|
| 44 |
|
| 45 |
+
@asynccontextmanager
|
| 46 |
+
async def lifespan(_: FastAPI):
|
| 47 |
+
global encode_lock
|
| 48 |
+
await asyncio.to_thread(_load_model)
|
| 49 |
+
if st_model is None and hf_model is None:
|
| 50 |
+
raise RuntimeError("E5 model failed to load")
|
| 51 |
+
encode_lock = asyncio.Lock()
|
| 52 |
+
yield
|
| 53 |
|
| 54 |
|
| 55 |
+
app = FastAPI(title="Studify E5 Embeddings", version="2.0.0", lifespan=lifespan)
|
| 56 |
+
if ALLOWED_ORIGINS:
|
| 57 |
+
app.add_middleware(
|
| 58 |
+
CORSMiddleware,
|
| 59 |
+
allow_origins=ALLOWED_ORIGINS,
|
| 60 |
+
allow_methods=["GET", "POST"],
|
| 61 |
+
allow_headers=["Content-Type", "Authorization"],
|
| 62 |
+
)
|
| 63 |
|
| 64 |
|
| 65 |
+
class EmbeddingRequest(BaseModel):
|
| 66 |
+
input: str = Field(min_length=1)
|
| 67 |
+
task: Literal["query", "passage"] = "passage"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
|
|
|
|
|
|
|
| 69 |
|
| 70 |
+
class BatchRequest(BaseModel):
|
| 71 |
+
inputs: list[str] = Field(min_length=1)
|
| 72 |
+
task: Literal["query", "passage"] = "passage"
|
| 73 |
|
|
|
|
|
|
|
| 74 |
|
| 75 |
+
def _authorize(authorization: str | None) -> None:
|
| 76 |
+
if not API_TOKEN:
|
| 77 |
+
raise HTTPException(503, "EMBEDDING_API_TOKEN is not configured")
|
| 78 |
+
supplied = (
|
| 79 |
+
authorization[7:]
|
| 80 |
+
if authorization and authorization.startswith("Bearer ")
|
| 81 |
+
else ""
|
| 82 |
+
)
|
| 83 |
+
if not hmac.compare_digest(supplied, API_TOKEN):
|
| 84 |
+
raise HTTPException(401, "unauthorized")
|
| 85 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
|
| 87 |
+
def _prefix(text: str, task: Literal["query", "passage"]) -> str:
|
| 88 |
+
return f"{task}: {text.strip()}"
|
| 89 |
|
|
|
|
|
|
|
|
|
|
| 90 |
|
| 91 |
+
def _mean_pool(last_hidden_state: torch.Tensor, attention_mask: torch.Tensor):
|
| 92 |
+
mask = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()
|
| 93 |
+
summed = torch.sum(last_hidden_state * mask, dim=1)
|
| 94 |
+
counts = torch.clamp(mask.sum(dim=1), min=1e-9)
|
| 95 |
+
return summed / counts
|
| 96 |
|
|
|
|
|
|
|
|
|
|
| 97 |
|
| 98 |
+
def _encode(texts: list[str]) -> list[list[float]]:
|
| 99 |
+
if st_model is not None:
|
| 100 |
+
return st_model.encode(
|
| 101 |
texts,
|
| 102 |
+
batch_size=min(32, len(texts)),
|
| 103 |
normalize_embeddings=True,
|
| 104 |
+
convert_to_numpy=True,
|
| 105 |
+
show_progress_bar=False,
|
| 106 |
).tolist()
|
| 107 |
|
| 108 |
+
if hf_model is None or hf_tokenizer is None:
|
| 109 |
+
raise RuntimeError("model is not loaded")
|
| 110 |
+
inputs = hf_tokenizer(
|
| 111 |
+
texts,
|
| 112 |
+
padding=True,
|
| 113 |
+
truncation=True,
|
| 114 |
+
max_length=MAX_LENGTH,
|
| 115 |
+
return_tensors="pt",
|
| 116 |
+
).to(DEVICE)
|
| 117 |
+
with torch.inference_mode():
|
| 118 |
+
outputs = hf_model(**inputs)
|
| 119 |
+
pooled = _mean_pool(outputs.last_hidden_state, inputs["attention_mask"])
|
| 120 |
+
normalized = F.normalize(pooled, p=2, dim=1)
|
| 121 |
+
return normalized.cpu().tolist()
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
async def _encode_safely(texts: list[str]) -> list[list[float]]:
|
| 125 |
+
if len(texts) > MAX_BATCH_SIZE:
|
| 126 |
+
raise HTTPException(413, f"batch exceeds MAX_BATCH_SIZE={MAX_BATCH_SIZE}")
|
| 127 |
+
if any(not text.strip() for text in texts):
|
| 128 |
+
raise HTTPException(422, "inputs cannot contain empty strings")
|
| 129 |
+
assert encode_lock is not None
|
| 130 |
+
async with encode_lock:
|
| 131 |
+
try:
|
| 132 |
+
vectors = await asyncio.to_thread(_encode, texts)
|
| 133 |
+
if any(len(vector) != EXPECTED_DIMENSION for vector in vectors):
|
| 134 |
+
raise RuntimeError(
|
| 135 |
+
f"model dimension mismatch; expected {EXPECTED_DIMENSION}"
|
| 136 |
+
)
|
| 137 |
+
return vectors
|
| 138 |
+
except HTTPException:
|
| 139 |
+
raise
|
| 140 |
+
except Exception as error:
|
| 141 |
+
raise HTTPException(500, f"embedding failed: {error}") from error
|
| 142 |
|
|
|
|
|
|
|
| 143 |
|
| 144 |
+
@app.get("/")
|
| 145 |
+
@app.get("/healthz")
|
| 146 |
+
async def health():
|
| 147 |
+
if st_model is None and hf_model is None:
|
| 148 |
+
raise HTTPException(503, "model not loaded")
|
| 149 |
+
return {
|
| 150 |
+
"status": "ok",
|
| 151 |
+
"device": DEVICE,
|
| 152 |
+
"model_path": MODEL_PATH,
|
| 153 |
+
"dimension": EXPECTED_DIMENSION,
|
| 154 |
+
}
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
@app.post("/embed")
|
| 158 |
+
async def embed(
|
| 159 |
+
request: EmbeddingRequest,
|
| 160 |
+
authorization: str | None = Header(default=None),
|
| 161 |
+
):
|
| 162 |
+
_authorize(authorization)
|
| 163 |
+
vectors = await _encode_safely([_prefix(request.input, request.task)])
|
| 164 |
+
return {"embedding": vectors[0], "dim": len(vectors[0])}
|
| 165 |
|
|
|
|
|
|
|
| 166 |
|
| 167 |
+
@app.post("/embed/batch")
|
| 168 |
+
async def embed_batch(
|
| 169 |
+
request: BatchRequest,
|
| 170 |
+
authorization: str | None = Header(default=None),
|
| 171 |
+
):
|
| 172 |
+
_authorize(authorization)
|
| 173 |
+
vectors = await _encode_safely(
|
| 174 |
+
[_prefix(text, request.task) for text in request.inputs]
|
| 175 |
+
)
|
| 176 |
return {
|
| 177 |
"embeddings": vectors,
|
| 178 |
"count": len(vectors),
|
| 179 |
+
"dim": len(vectors[0]),
|
| 180 |
+
}
|