danieldunderfelt's picture
Add onnx model.
6e3bdfa
Raw History Blame Contribute Delete
8.78 kB
from fastapi import FastAPI, Request, Response
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from pydantic import BaseModel, Field
from sentence_transformers import SentenceTransformer, __version__ as st_version
import os
import numpy as np
import logging
from typing import List, Optional, Dict, Any, Union
import torch
import sys
import traceback
import platform
import time
from datetime import datetime
import uvicorn
# Configure logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
stream=sys.stdout
)
logger = logging.getLogger(__name__)
# Initialize FastAPI
app = FastAPI(
title="Model2Vec Embedding Service",
description="High-performance embedding service using the full Model2Vec model",
version="1.0.0"
)
# Enable CORS
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# Global variables
model = None
startup_time = datetime.utcnow().isoformat()
request_count = 0
total_processing_time = 0
# Pydantic models for request/response validation
class EmbeddingRequest(BaseModel):
texts: Union[str, List[str]]
batch_size: int = Field(default=32, gt=0)
normalize: bool = True
class EmbeddingResponse(BaseModel):
embeddings: List[List[float]]
metadata: Dict[str, Any]
def load_model(model_path: str = "/app/model") -> Optional[SentenceTransformer]:
"""Load the model with proper device placement and optimizations"""
try:
logger.info(f"Starting model loading from {model_path}")
logger.info(f"Current working directory: {os.getcwd()}")
required_files = [
"model.safetensors",
"config.json",
"tokenizer.json",
"modules.json"
]
missing_files = [f for f in required_files if not os.path.exists(os.path.join(model_path, f))]
if missing_files:
logger.error(f"Missing required model files: {missing_files}")
return None
# Configure device and optimize for CPU
device = "cpu"
torch.set_num_threads(int(os.cpu_count() or 4))
torch.set_grad_enabled(False) # Disable gradients for inference
logger.info(f"Loading model on {device} with {torch.get_num_threads()} threads")
model = SentenceTransformer(model_path, device=device, truncate_dim=384, cache_folder=model_path, local_files_only=True)
# Warmup
logger.info("Warming up model with test input...")
test_result = model.encode(
["test sentence"] * 4,
batch_size=4,
convert_to_numpy=True
)
logger.info(f"Warmup complete. Test shape: {test_result.shape}")
return model
except Exception as e:
logger.error(f"Error loading model: {str(e)}")
logger.error(f"Traceback: {traceback.format_exc()}")
return None
# Initialize model at startup
try:
model = load_model()
if model is None:
raise RuntimeError("Model failed to load")
except Exception as e:
logger.error(f"Critical error during model initialization: {str(e)}")
logger.error(f"Traceback: {traceback.format_exc()}")
@app.middleware("http")
async def add_process_time_header(request: Request, call_next):
"""Middleware to track request timing and add security headers"""
global request_count, total_processing_time
start_time = time.time()
response = await call_next(request)
process_time = time.time() - start_time
request_count += 1
total_processing_time += process_time
# Add security headers
response.headers['X-Process-Time'] = str(process_time)
response.headers['X-Content-Type-Options'] = 'nosniff'
response.headers['X-Frame-Options'] = 'DENY'
response.headers['Strict-Transport-Security'] = 'max-age=31536000; includeSubDomains'
logger.info(
f"Request completed: endpoint={request.url.path}, "
f"method={request.method}, status={response.status_code}, "
f"duration={process_time:.3f}s"
)
return response
@app.get("/")
async def root() -> Dict[str, Any]:
"""Root endpoint with service information"""
return {
"service": "Model2Vec Embedding Service",
"status": "healthy" if model is not None else "unhealthy",
"version": st_version,
"startup_time": startup_time,
"endpoints": {
"/": "Service information (GET)",
"/health": "Health check and metrics (GET)",
"/encode": "Generate embeddings (POST)"
}
}
@app.get("/health")
async def health_check() -> Dict[str, Any]:
"""Health check endpoint with metrics"""
try:
if model is None:
return JSONResponse(
status_code=500,
content={
"status": "unhealthy",
"error": "Model not loaded",
"startup_time": startup_time,
"python_version": platform.python_version(),
"torch_version": torch.__version__,
"sentence_transformers_version": st_version
}
)
# Quick model test
test_result = model.encode(["test sentence"], convert_to_numpy=True)
# Calculate average processing time
avg_time = total_processing_time / request_count if request_count > 0 else 0
return {
"status": "healthy",
"model": {
"loaded": True,
"embedding_dimension": model.get_sentence_embedding_dimension(),
"device": model.device.type,
"max_seq_length": model.max_seq_length,
"test_shape": test_result.shape
},
"metrics": {
"uptime": startup_time,
"total_requests": request_count,
"avg_processing_time": f"{avg_time:.3f}s"
},
"system": {
"python_version": platform.python_version(),
"torch_version": torch.__version__,
"sentence_transformers_version": st_version,
"cpu_threads": torch.get_num_threads()
}
}
except Exception as e:
logger.error(f"Health check error: {str(e)}")
logger.error(f"Traceback: {traceback.format_exc()}")
return JSONResponse(
status_code=500,
content={
"status": "error",
"error": str(e)
}
)
@app.post("/encode", response_model=EmbeddingResponse)
async def encode(request: EmbeddingRequest) -> Dict[str, Any]:
"""Generate embeddings for input texts"""
try:
if model is None:
return JSONResponse(
status_code=500,
content={"error": "Model not loaded"}
)
texts = request.texts
if isinstance(texts, str):
texts = [texts]
# Limit maximum number of texts
max_texts = 100
if len(texts) > max_texts:
return JSONResponse(
status_code=400,
content={"error": f"Too many texts. Maximum allowed: {max_texts}"}
)
# Generate embeddings with batching
start_time = time.time()
embeddings = model.encode(
texts,
batch_size=request.batch_size,
show_progress_bar=False,
convert_to_numpy=True,
normalize_embeddings=request.normalize
)
processing_time = time.time() - start_time
return {
"embeddings": embeddings.tolist(),
"metadata": {
"num_texts": len(texts),
"embedding_dim": embeddings.shape[1],
"normalized": request.normalize,
"processing_time": f"{processing_time:.3f}s"
}
}
except Exception as e:
logger.error(f"Encode error: {str(e)}")
logger.error(f"Traceback: {traceback.format_exc()}")
return JSONResponse(
status_code=500,
content={
"error": str(e),
"type": type(e).__name__
}
)
if __name__ == "__main__":
port = int(os.environ.get("PORT", 8080))
# Use multiple workers for better concurrency
uvicorn.run(
"app:app",
host="0.0.0.0",
port=port,
workers=int(os.cpu_count() or 2), # Use multiple workers
log_level="info"
)