Spaces:
Sleeping
Sleeping
Download sdk/python/examples/production_setup.py from fmbishu/image-search-api: direct link, hf CLI and curl.
- Browser
- Download file 7.17 kB
-
https://huggingface.co/spaces/fmbishu/image-search-api/resolve/main/sdk/python/examples/production_setup.py
- Command line
-
hf download hf://spaces/fmbishu/image-search-api/sdk/python/examples/production_setup.py
-
curl -L -o production_setup.py https://huggingface.co/spaces/fmbishu/image-search-api/resolve/main/sdk/python/examples/production_setup.py
7.17 kB
| """ | |
| Production Setup Example - Enterprise-grade configuration. | |
| This example shows how to set up the SDK for production use with: | |
| - Multiple vector store options (Pinecone, Qdrant, Weaviate, pgVector) | |
| - Error handling and monitoring | |
| - Performance optimization | |
| - Auto-monitoring for new products | |
| """ | |
| import sys | |
| from pathlib import Path | |
| import os | |
| # Add SDK to path | |
| sdk_path = Path(__file__).parent.parent | |
| sys.path.insert(0, str(sdk_path)) | |
| from image_search_sdk import ImageSearchClient, SearchOptions | |
| def setup_with_pinecone(): | |
| """Production setup with Pinecone (Managed Vector Database)""" | |
| print("Setting up with Pinecone (Managed)") | |
| client = ImageSearchClient( | |
| data_dir="./production_data", | |
| model_name="clip-ViT-L-14", | |
| vector_store_type="pinecone", | |
| vector_store_config={ | |
| "api_key": os.getenv("PINECONE_API_KEY"), | |
| "environment": os.getenv("PINECONE_ENVIRONMENT", "us-west1-gcp"), | |
| "index_name": "product-search" | |
| } | |
| ) | |
| return client | |
| def setup_with_qdrant(): | |
| """Production setup with Qdrant (Self-hosted or Cloud)""" | |
| print("Setting up with Qdrant") | |
| client = ImageSearchClient( | |
| data_dir="./production_data", | |
| model_name="clip-ViT-L-14", | |
| vector_store_type="qdrant", | |
| vector_store_config={ | |
| "url": os.getenv("QDRANT_URL", "http://localhost:6333"), | |
| "api_key": os.getenv("QDRANT_API_KEY"), # Optional for cloud | |
| "collection_name": "products" | |
| } | |
| ) | |
| return client | |
| def setup_with_weaviate(): | |
| """Production setup with Weaviate""" | |
| print("Setting up with Weaviate") | |
| client = ImageSearchClient( | |
| data_dir="./production_data", | |
| model_name="clip-ViT-L-14", | |
| vector_store_type="weaviate", | |
| vector_store_config={ | |
| "url": os.getenv("WEAVIATE_URL", "http://localhost:8080"), | |
| "api_key": os.getenv("WEAVIATE_API_KEY"), # Optional | |
| } | |
| ) | |
| return client | |
| def setup_with_pgvector(): | |
| """Production setup with pgVector (PostgreSQL extension)""" | |
| print("Setting up with pgVector (PostgreSQL)") | |
| client = ImageSearchClient( | |
| data_dir="./production_data", | |
| model_name="clip-ViT-L-14", | |
| vector_store_type="pgvector", | |
| vector_store_config={ | |
| "host": os.getenv("PG_HOST", "localhost"), | |
| "port": int(os.getenv("PG_PORT", "5432")), | |
| "database": os.getenv("PG_DATABASE", "products"), | |
| "user": os.getenv("PG_USER", "postgres"), | |
| "password": os.getenv("PG_PASSWORD"), | |
| "table_name": "product_vectors" | |
| } | |
| ) | |
| return client | |
| def setup_with_chromadb(): | |
| """Development/Small-scale setup with ChromaDB""" | |
| print("Setting up with ChromaDB (Local/Development)") | |
| client = ImageSearchClient( | |
| data_dir="./production_data", | |
| model_name="clip-ViT-L-14", | |
| vector_store_type="chromadb", | |
| vector_store_config={ | |
| "mode": "persistent", | |
| "path": "./production_data/chroma_vectors" | |
| } | |
| ) | |
| return client | |
| def main(): | |
| print("Production Setup Example") | |
| print("=" * 70) | |
| # Choose your vector store (set via environment variable) | |
| vector_store = os.getenv("VECTOR_STORE", "chromadb").lower() | |
| print(f"\nUsing vector store: {vector_store}") | |
| print("=" * 70) | |
| # Initialize based on choice | |
| if vector_store == "pinecone": | |
| client = setup_with_pinecone() | |
| elif vector_store == "qdrant": | |
| client = setup_with_qdrant() | |
| elif vector_store == "weaviate": | |
| client = setup_with_weaviate() | |
| elif vector_store == "pgvector": | |
| client = setup_with_pgvector() | |
| else: | |
| client = setup_with_chromadb() | |
| print("\nβ SDK initialized") | |
| # Connect to your production database | |
| print("\nConnecting to production database...") | |
| try: | |
| client.connect_database( | |
| connection_string=os.getenv("DATABASE_URL"), | |
| table_name=os.getenv("PRODUCTS_TABLE", "products"), | |
| id_column="id", | |
| title_column="title", | |
| image_url_column="image_url", | |
| category_column="category", | |
| brand_column="brand", | |
| price_column="price", | |
| rating_column="rating", | |
| description_column="description" | |
| ) | |
| print("β Database connected") | |
| except Exception as e: | |
| print(f"β Database connection failed: {e}") | |
| return | |
| # Initialize embeddings (with error handling) | |
| print("\nInitializing embeddings...") | |
| try: | |
| stats = client.initialize(batch_size=100) | |
| print(f"β Initialization complete") | |
| print(f" - Indexed: {stats['indexed_products']:,} products") | |
| print(f" - Failed: {stats['failed_products']:,} products") | |
| print(f" - Rate: {stats['products_per_second']:.1f} products/sec") | |
| if stats['errors']: | |
| print(f"\nβ Errors encountered: {len(stats['errors'])}") | |
| # Log errors to file or monitoring system | |
| with open("indexing_errors.log", "w") as f: | |
| for error in stats['errors']: | |
| f.write(f"{error}\n") | |
| except Exception as e: | |
| print(f"β Initialization failed: {e}") | |
| return | |
| # Enable auto-monitoring for production | |
| print("\nEnabling auto-monitoring...") | |
| try: | |
| monitor_interval = int(os.getenv("MONITOR_INTERVAL", "60")) | |
| client.enable_auto_monitor( | |
| poll_interval=monitor_interval, | |
| enable=True | |
| ) | |
| print(f"β Auto-monitoring enabled (interval: {monitor_interval}s)") | |
| except Exception as e: | |
| print(f"β Auto-monitoring failed: {e}") | |
| # Test search functionality | |
| print("\nTesting search...") | |
| try: | |
| test_results = client.search_by_text( | |
| "test query", | |
| options=SearchOptions(top_k=1) | |
| ) | |
| print(f"β Search working ({len(test_results)} result)") | |
| except Exception as e: | |
| print(f"β Search test failed: {e}") | |
| # Display statistics | |
| print("\n" + "=" * 70) | |
| print("PRODUCTION STATISTICS") | |
| print("=" * 70) | |
| stats = client.get_stats() | |
| print(f"Vector Store: {stats.get('store_type', 'N/A')}") | |
| print(f"Total Vectors: {stats.get('total_vectors', 0):,}") | |
| print(f"Status: {'Operational' if stats.get('is_initialized') else 'Not Ready'}") | |
| print("\n" + "=" * 70) | |
| print("Production setup complete!") | |
| print("The system is now running and monitoring for new products.") | |
| print("=" * 70) | |
| # Keep running (for monitoring) | |
| print("\nPress Ctrl+C to shutdown...") | |
| try: | |
| import time | |
| while True: | |
| time.sleep(60) | |
| # Optionally log health metrics here | |
| current_stats = client.get_stats() | |
| print(f"[Health Check] Vectors: {current_stats.get('total_vectors', 0):,}") | |
| except KeyboardInterrupt: | |
| print("\n\nShutting down...") | |
| client.close() | |
| print("β Shutdown complete") | |
| if __name__ == "__main__": | |
| main() | |