from fastapi import FastAPI, Request, Response from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse from pydantic import BaseModel, Field from sentence_transformers import SentenceTransformer, __version__ as st_version import os import numpy as np import logging from typing import List, Optional, Dict, Any, Union import torch import sys import traceback import platform import time from datetime import datetime import uvicorn # Configure logging logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', stream=sys.stdout ) logger = logging.getLogger(__name__) # Initialize FastAPI app = FastAPI( title="Model2Vec Embedding Service", description="High-performance embedding service using the full Model2Vec model", version="1.0.0" ) # Enable CORS app.add_middleware( CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) # Global variables model = None startup_time = datetime.utcnow().isoformat() request_count = 0 total_processing_time = 0 # Pydantic models for request/response validation class EmbeddingRequest(BaseModel): texts: Union[str, List[str]] batch_size: int = Field(default=32, gt=0) normalize: bool = True class EmbeddingResponse(BaseModel): embeddings: List[List[float]] metadata: Dict[str, Any] def load_model(model_path: str = "/app/model") -> Optional[SentenceTransformer]: """Load the model with proper device placement and optimizations""" try: logger.info(f"Starting model loading from {model_path}") logger.info(f"Current working directory: {os.getcwd()}") required_files = [ "model.safetensors", "config.json", "tokenizer.json", "modules.json" ] missing_files = [f for f in required_files if not os.path.exists(os.path.join(model_path, f))] if missing_files: logger.error(f"Missing required model files: {missing_files}") return None # Configure device and optimize for CPU device = "cpu" torch.set_num_threads(int(os.cpu_count() or 4)) torch.set_grad_enabled(False) # Disable gradients for inference logger.info(f"Loading model on {device} with {torch.get_num_threads()} threads") model = SentenceTransformer(model_path, device=device, truncate_dim=384, cache_folder=model_path, local_files_only=True) # Warmup logger.info("Warming up model with test input...") test_result = model.encode( ["test sentence"] * 4, batch_size=4, convert_to_numpy=True ) logger.info(f"Warmup complete. Test shape: {test_result.shape}") return model except Exception as e: logger.error(f"Error loading model: {str(e)}") logger.error(f"Traceback: {traceback.format_exc()}") return None # Initialize model at startup try: model = load_model() if model is None: raise RuntimeError("Model failed to load") except Exception as e: logger.error(f"Critical error during model initialization: {str(e)}") logger.error(f"Traceback: {traceback.format_exc()}") @app.middleware("http") async def add_process_time_header(request: Request, call_next): """Middleware to track request timing and add security headers""" global request_count, total_processing_time start_time = time.time() response = await call_next(request) process_time = time.time() - start_time request_count += 1 total_processing_time += process_time # Add security headers response.headers['X-Process-Time'] = str(process_time) response.headers['X-Content-Type-Options'] = 'nosniff' response.headers['X-Frame-Options'] = 'DENY' response.headers['Strict-Transport-Security'] = 'max-age=31536000; includeSubDomains' logger.info( f"Request completed: endpoint={request.url.path}, " f"method={request.method}, status={response.status_code}, " f"duration={process_time:.3f}s" ) return response @app.get("/") async def root() -> Dict[str, Any]: """Root endpoint with service information""" return { "service": "Model2Vec Embedding Service", "status": "healthy" if model is not None else "unhealthy", "version": st_version, "startup_time": startup_time, "endpoints": { "/": "Service information (GET)", "/health": "Health check and metrics (GET)", "/encode": "Generate embeddings (POST)" } } @app.get("/health") async def health_check() -> Dict[str, Any]: """Health check endpoint with metrics""" try: if model is None: return JSONResponse( status_code=500, content={ "status": "unhealthy", "error": "Model not loaded", "startup_time": startup_time, "python_version": platform.python_version(), "torch_version": torch.__version__, "sentence_transformers_version": st_version } ) # Quick model test test_result = model.encode(["test sentence"], convert_to_numpy=True) # Calculate average processing time avg_time = total_processing_time / request_count if request_count > 0 else 0 return { "status": "healthy", "model": { "loaded": True, "embedding_dimension": model.get_sentence_embedding_dimension(), "device": model.device.type, "max_seq_length": model.max_seq_length, "test_shape": test_result.shape }, "metrics": { "uptime": startup_time, "total_requests": request_count, "avg_processing_time": f"{avg_time:.3f}s" }, "system": { "python_version": platform.python_version(), "torch_version": torch.__version__, "sentence_transformers_version": st_version, "cpu_threads": torch.get_num_threads() } } except Exception as e: logger.error(f"Health check error: {str(e)}") logger.error(f"Traceback: {traceback.format_exc()}") return JSONResponse( status_code=500, content={ "status": "error", "error": str(e) } ) @app.post("/encode", response_model=EmbeddingResponse) async def encode(request: EmbeddingRequest) -> Dict[str, Any]: """Generate embeddings for input texts""" try: if model is None: return JSONResponse( status_code=500, content={"error": "Model not loaded"} ) texts = request.texts if isinstance(texts, str): texts = [texts] # Limit maximum number of texts max_texts = 100 if len(texts) > max_texts: return JSONResponse( status_code=400, content={"error": f"Too many texts. Maximum allowed: {max_texts}"} ) # Generate embeddings with batching start_time = time.time() embeddings = model.encode( texts, batch_size=request.batch_size, show_progress_bar=False, convert_to_numpy=True, normalize_embeddings=request.normalize ) processing_time = time.time() - start_time return { "embeddings": embeddings.tolist(), "metadata": { "num_texts": len(texts), "embedding_dim": embeddings.shape[1], "normalized": request.normalize, "processing_time": f"{processing_time:.3f}s" } } except Exception as e: logger.error(f"Encode error: {str(e)}") logger.error(f"Traceback: {traceback.format_exc()}") return JSONResponse( status_code=500, content={ "error": str(e), "type": type(e).__name__ } ) if __name__ == "__main__": port = int(os.environ.get("PORT", 8080)) # Use multiple workers for better concurrency uvicorn.run( "app:app", host="0.0.0.0", port=port, workers=int(os.cpu_count() or 2), # Use multiple workers log_level="info" )