Jupyter / app /bge_m3_api.py
SmartDataAlex's picture
Deploy vectorization service from GitHub f2352e8140c520d4480226b9332c05011709ed54
350ad7e verified
Raw History Blame Contribute Delete
3.27 kB
"""Authenticated BGE-M3 triple-vector API, separate from dense-only APIs."""
from __future__ import annotations
from typing import Literal
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, ConfigDict, Field, field_validator
from app.bge_m3 import BGE_M3_ID
from app.embedding import TooManyTokens
from app.models import ModelRegistry, ServiceBusy
from app.voyage_api import authenticate, resident_models
router = APIRouter(prefix="/bge-m3", tags=["BGE-M3 triple embeddings"])
class BgeM3Request(BaseModel):
model_config = ConfigDict(extra="forbid")
input: str | list[str] = Field(description="One to eight texts (without prefixes)")
input_type: Literal["query", "document"] = "document"
return_colbert: bool = Field(
default=False,
description="Compute and include ColBERT multivectors (large response). "
"Default false returns only dense and sparse.",
)
@field_validator("input")
@classmethod
def check_texts(cls, value):
texts = [value] if isinstance(value, str) else value
if not texts or len(texts) > 8:
raise ValueError("BGE-M3 requires between 1 and 8 texts")
if any(not text.strip() for text in texts):
raise ValueError("BGE-M3 inputs cannot be blank")
return value
@router.post("/encode", dependencies=[Depends(authenticate)])
def encode_bge_m3(payload: BgeM3Request,
registry: ModelRegistry = Depends(resident_models)):
if registry.bge_m3 is None:
raise HTTPException(status_code=503, detail="BGE-M3 model is disabled")
texts = [payload.input] if isinstance(payload.input, str) else payload.input
# BGE-M3 is symmetric: query and document use the same tokenizer,
# no artificial 'query:' or 'passage:' prefix is added.
try:
results = registry.bge_m3_encode(
texts, return_colbert=payload.return_colbert,
)
except TooManyTokens as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
except ServiceBusy as exc:
raise HTTPException(status_code=429, detail=str(exc),
headers={"Retry-After": "2"}) from exc
except Exception as exc:
import logging
logging.getLogger(__name__).exception("BGE-M3 inference failed")
raise HTTPException(status_code=500, detail="BGE-M3 inference failed") from exc
return {
"model": BGE_M3_ID,
"object": "list",
"dense_dimensions": registry.bge_m3.dimensions,
"colbert_dimensions": registry.bge_m3.colbert_dimensions,
"data": [
{
"index": index,
"dense": item.dense,
"sparse": {
"indices": item.sparse_indices,
"values": item.sparse_values,
},
**({"colbert": item.colbert} if payload.return_colbert else {}),
"tokens_count": item.tokens_count,
}
for index, item in enumerate(results)
],
"usage": {"total_tokens": sum(item.tokens_count for item in results)},
}