edusocial commited on
Commit
071760e
·
verified ·
1 Parent(s): 937d2ca

Update app/main.py

Browse files
Files changed (1) hide show
  1. app/main.py +139 -93
app/main.py CHANGED
@@ -1,134 +1,180 @@
1
- from fastapi import FastAPI
2
- from pydantic import BaseModel
3
- from fastapi.middleware.cors import CORSMiddleware
4
  import os
 
 
5
 
 
 
 
 
 
6
  from sentence_transformers import SentenceTransformer
7
  from transformers import AutoModel, AutoTokenizer
8
- import torch
9
-
10
- app = FastAPI()
11
 
12
- app.add_middleware(
13
- CORSMiddleware,
14
- allow_origins=["*"],
15
- allow_methods=["*"],
16
- allow_headers=["*"],
17
- )
18
 
19
- MODEL_PATH = "/app/model"
20
  DEVICE = os.getenv("DEVICE", "cpu")
21
-
22
- st_model = None
 
 
 
 
 
 
 
 
 
23
  hf_model = None
24
  hf_tokenizer = None
 
25
 
26
 
27
- def add_prefix(text: str, task: str):
28
- if task == "query":
29
- return f"query: {text}"
30
- return f"passage: {text}"
31
-
32
-
33
- @app.get("/")
34
- async def root():
35
- return {"status": "running", "msg": "embedding server is alive"}
36
-
37
-
38
- @app.on_event("startup")
39
- async def load_model():
40
  global st_model, hf_model, hf_tokenizer
41
  try:
42
- print(f"Trying to load SentenceTransformer model from {MODEL_PATH} on {DEVICE}")
43
  st_model = SentenceTransformer(MODEL_PATH, device=DEVICE)
44
- print("✅ Loaded as SentenceTransformer")
45
- except Exception as e:
46
- print(f"⚠️ SentenceTransformer load failed: {e}")
47
- print("➡️ Falling back to HuggingFace AutoModel...")
48
- try:
49
- hf_tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)
50
- hf_model = AutoModel.from_pretrained(MODEL_PATH).to(DEVICE)
51
- hf_model.eval()
52
- print("✅ Loaded as HuggingFace AutoModel")
53
- except Exception as e2:
54
- print(f"❌ Failed to load model in both modes: {e2}")
55
 
56
 
57
- @app.get("/healthz")
58
- async def healthz():
59
- if st_model or hf_model:
60
- return {"status": "ok", "device": DEVICE}
61
- return {"status": "error", "msg": "model not loaded"}
 
 
 
62
 
63
 
64
- class EmbeddingRequest(BaseModel):
65
- input: str
66
- task: str = "query" # query 或 passage
 
 
 
 
 
67
 
68
 
69
- @app.post("/embed")
70
- async def embed(request: EmbeddingRequest):
71
- text = add_prefix(request.input, request.task)
72
-
73
- if st_model:
74
- embedding = st_model.encode(
75
- text,
76
- normalize_embeddings=True,
77
- convert_to_numpy=True
78
- ).tolist()
79
 
80
- elif hf_model and hf_tokenizer:
81
- inputs = hf_tokenizer(text, return_tensors="pt").to(DEVICE)
82
 
83
- with torch.no_grad():
84
- outputs = hf_model(**inputs)
 
85
 
86
- # mean pooling
87
- embedding = outputs.last_hidden_state.mean(dim=1).squeeze().cpu().tolist()
88
 
89
- else:
90
- return {"error": "model not loaded"}
 
 
 
 
 
 
 
 
91
 
92
- return {
93
- "embedding": embedding,
94
- "dim": len(embedding),
95
- }
96
 
 
 
97
 
98
- class BatchRequest(BaseModel):
99
- inputs: list[str]
100
- task: str = "query"
101
 
 
 
 
 
 
102
 
103
- @app.post("/embed/batch")
104
- async def embed_batch(request: BatchRequest):
105
- texts = [add_prefix(t, request.task) for t in request.inputs]
106
 
107
- if st_model:
108
- vectors = st_model.encode(
 
109
  texts,
 
110
  normalize_embeddings=True,
111
- convert_to_numpy=True
 
112
  ).tolist()
113
 
114
- elif hf_model and hf_tokenizer:
115
- inputs = hf_tokenizer(
116
- texts,
117
- padding=True,
118
- truncation=True,
119
- return_tensors="pt"
120
- ).to(DEVICE)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
121
 
122
- with torch.no_grad():
123
- outputs = hf_model(**inputs)
124
 
125
- vectors = outputs.last_hidden_state.mean(dim=1).cpu().tolist()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
126
 
127
- else:
128
- return {"error": "model not loaded"}
129
 
 
 
 
 
 
 
 
 
 
130
  return {
131
  "embeddings": vectors,
132
  "count": len(vectors),
133
- "dim": len(vectors[0]) if vectors else 0,
134
- }
 
1
+ import asyncio
2
+ import hmac
 
3
  import os
4
+ from contextlib import asynccontextmanager
5
+ from typing import Literal
6
 
7
+ import torch
8
+ import torch.nn.functional as F
9
+ from fastapi import FastAPI, Header, HTTPException
10
+ from fastapi.middleware.cors import CORSMiddleware
11
+ from pydantic import BaseModel, Field
12
  from sentence_transformers import SentenceTransformer
13
  from transformers import AutoModel, AutoTokenizer
 
 
 
14
 
 
 
 
 
 
 
15
 
16
+ MODEL_PATH = os.getenv("MODEL_PATH", "/app/model")
17
  DEVICE = os.getenv("DEVICE", "cpu")
18
+ MAX_BATCH_SIZE = int(os.getenv("MAX_BATCH_SIZE", "64"))
19
+ MAX_LENGTH = int(os.getenv("MAX_LENGTH", "512"))
20
+ EXPECTED_DIMENSION = 384
21
+ API_TOKEN = os.getenv("EMBEDDING_API_TOKEN")
22
+ ALLOWED_ORIGINS = [
23
+ origin.strip()
24
+ for origin in os.getenv("ALLOWED_ORIGINS", "").split(",")
25
+ if origin.strip()
26
+ ]
27
+
28
+ st_model: SentenceTransformer | None = None
29
  hf_model = None
30
  hf_tokenizer = None
31
+ encode_lock: asyncio.Lock | None = None
32
 
33
 
34
+ def _load_model() -> None:
 
 
 
 
 
 
 
 
 
 
 
 
35
  global st_model, hf_model, hf_tokenizer
36
  try:
 
37
  st_model = SentenceTransformer(MODEL_PATH, device=DEVICE)
38
+ st_model.max_seq_length = MAX_LENGTH
39
+ except Exception:
40
+ hf_tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)
41
+ hf_model = AutoModel.from_pretrained(MODEL_PATH).to(DEVICE)
42
+ hf_model.eval()
 
 
 
 
 
 
43
 
44
 
45
+ @asynccontextmanager
46
+ async def lifespan(_: FastAPI):
47
+ global encode_lock
48
+ await asyncio.to_thread(_load_model)
49
+ if st_model is None and hf_model is None:
50
+ raise RuntimeError("E5 model failed to load")
51
+ encode_lock = asyncio.Lock()
52
+ yield
53
 
54
 
55
+ app = FastAPI(title="Studify E5 Embeddings", version="2.0.0", lifespan=lifespan)
56
+ if ALLOWED_ORIGINS:
57
+ app.add_middleware(
58
+ CORSMiddleware,
59
+ allow_origins=ALLOWED_ORIGINS,
60
+ allow_methods=["GET", "POST"],
61
+ allow_headers=["Content-Type", "Authorization"],
62
+ )
63
 
64
 
65
+ class EmbeddingRequest(BaseModel):
66
+ input: str = Field(min_length=1)
67
+ task: Literal["query", "passage"] = "passage"
 
 
 
 
 
 
 
68
 
 
 
69
 
70
+ class BatchRequest(BaseModel):
71
+ inputs: list[str] = Field(min_length=1)
72
+ task: Literal["query", "passage"] = "passage"
73
 
 
 
74
 
75
+ def _authorize(authorization: str | None) -> None:
76
+ if not API_TOKEN:
77
+ raise HTTPException(503, "EMBEDDING_API_TOKEN is not configured")
78
+ supplied = (
79
+ authorization[7:]
80
+ if authorization and authorization.startswith("Bearer ")
81
+ else ""
82
+ )
83
+ if not hmac.compare_digest(supplied, API_TOKEN):
84
+ raise HTTPException(401, "unauthorized")
85
 
 
 
 
 
86
 
87
+ def _prefix(text: str, task: Literal["query", "passage"]) -> str:
88
+ return f"{task}: {text.strip()}"
89
 
 
 
 
90
 
91
+ def _mean_pool(last_hidden_state: torch.Tensor, attention_mask: torch.Tensor):
92
+ mask = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()
93
+ summed = torch.sum(last_hidden_state * mask, dim=1)
94
+ counts = torch.clamp(mask.sum(dim=1), min=1e-9)
95
+ return summed / counts
96
 
 
 
 
97
 
98
+ def _encode(texts: list[str]) -> list[list[float]]:
99
+ if st_model is not None:
100
+ return st_model.encode(
101
  texts,
102
+ batch_size=min(32, len(texts)),
103
  normalize_embeddings=True,
104
+ convert_to_numpy=True,
105
+ show_progress_bar=False,
106
  ).tolist()
107
 
108
+ if hf_model is None or hf_tokenizer is None:
109
+ raise RuntimeError("model is not loaded")
110
+ inputs = hf_tokenizer(
111
+ texts,
112
+ padding=True,
113
+ truncation=True,
114
+ max_length=MAX_LENGTH,
115
+ return_tensors="pt",
116
+ ).to(DEVICE)
117
+ with torch.inference_mode():
118
+ outputs = hf_model(**inputs)
119
+ pooled = _mean_pool(outputs.last_hidden_state, inputs["attention_mask"])
120
+ normalized = F.normalize(pooled, p=2, dim=1)
121
+ return normalized.cpu().tolist()
122
+
123
+
124
+ async def _encode_safely(texts: list[str]) -> list[list[float]]:
125
+ if len(texts) > MAX_BATCH_SIZE:
126
+ raise HTTPException(413, f"batch exceeds MAX_BATCH_SIZE={MAX_BATCH_SIZE}")
127
+ if any(not text.strip() for text in texts):
128
+ raise HTTPException(422, "inputs cannot contain empty strings")
129
+ assert encode_lock is not None
130
+ async with encode_lock:
131
+ try:
132
+ vectors = await asyncio.to_thread(_encode, texts)
133
+ if any(len(vector) != EXPECTED_DIMENSION for vector in vectors):
134
+ raise RuntimeError(
135
+ f"model dimension mismatch; expected {EXPECTED_DIMENSION}"
136
+ )
137
+ return vectors
138
+ except HTTPException:
139
+ raise
140
+ except Exception as error:
141
+ raise HTTPException(500, f"embedding failed: {error}") from error
142
 
 
 
143
 
144
+ @app.get("/")
145
+ @app.get("/healthz")
146
+ async def health():
147
+ if st_model is None and hf_model is None:
148
+ raise HTTPException(503, "model not loaded")
149
+ return {
150
+ "status": "ok",
151
+ "device": DEVICE,
152
+ "model_path": MODEL_PATH,
153
+ "dimension": EXPECTED_DIMENSION,
154
+ }
155
+
156
+
157
+ @app.post("/embed")
158
+ async def embed(
159
+ request: EmbeddingRequest,
160
+ authorization: str | None = Header(default=None),
161
+ ):
162
+ _authorize(authorization)
163
+ vectors = await _encode_safely([_prefix(request.input, request.task)])
164
+ return {"embedding": vectors[0], "dim": len(vectors[0])}
165
 
 
 
166
 
167
+ @app.post("/embed/batch")
168
+ async def embed_batch(
169
+ request: BatchRequest,
170
+ authorization: str | None = Header(default=None),
171
+ ):
172
+ _authorize(authorization)
173
+ vectors = await _encode_safely(
174
+ [_prefix(text, request.task) for text in request.inputs]
175
+ )
176
  return {
177
  "embeddings": vectors,
178
  "count": len(vectors),
179
+ "dim": len(vectors[0]),
180
+ }