image-search-api / sdk /python /examples /production_setup.py
mbishu's picture
Initialize SDK and fix vector store issues
a475b1f
Raw History Blame Contribute Delete
7.17 kB
"""
Production Setup Example - Enterprise-grade configuration.
This example shows how to set up the SDK for production use with:
- Multiple vector store options (Pinecone, Qdrant, Weaviate, pgVector)
- Error handling and monitoring
- Performance optimization
- Auto-monitoring for new products
"""
import sys
from pathlib import Path
import os
# Add SDK to path
sdk_path = Path(__file__).parent.parent
sys.path.insert(0, str(sdk_path))
from image_search_sdk import ImageSearchClient, SearchOptions
def setup_with_pinecone():
"""Production setup with Pinecone (Managed Vector Database)"""
print("Setting up with Pinecone (Managed)")
client = ImageSearchClient(
data_dir="./production_data",
model_name="clip-ViT-L-14",
vector_store_type="pinecone",
vector_store_config={
"api_key": os.getenv("PINECONE_API_KEY"),
"environment": os.getenv("PINECONE_ENVIRONMENT", "us-west1-gcp"),
"index_name": "product-search"
}
)
return client
def setup_with_qdrant():
"""Production setup with Qdrant (Self-hosted or Cloud)"""
print("Setting up with Qdrant")
client = ImageSearchClient(
data_dir="./production_data",
model_name="clip-ViT-L-14",
vector_store_type="qdrant",
vector_store_config={
"url": os.getenv("QDRANT_URL", "http://localhost:6333"),
"api_key": os.getenv("QDRANT_API_KEY"), # Optional for cloud
"collection_name": "products"
}
)
return client
def setup_with_weaviate():
"""Production setup with Weaviate"""
print("Setting up with Weaviate")
client = ImageSearchClient(
data_dir="./production_data",
model_name="clip-ViT-L-14",
vector_store_type="weaviate",
vector_store_config={
"url": os.getenv("WEAVIATE_URL", "http://localhost:8080"),
"api_key": os.getenv("WEAVIATE_API_KEY"), # Optional
}
)
return client
def setup_with_pgvector():
"""Production setup with pgVector (PostgreSQL extension)"""
print("Setting up with pgVector (PostgreSQL)")
client = ImageSearchClient(
data_dir="./production_data",
model_name="clip-ViT-L-14",
vector_store_type="pgvector",
vector_store_config={
"host": os.getenv("PG_HOST", "localhost"),
"port": int(os.getenv("PG_PORT", "5432")),
"database": os.getenv("PG_DATABASE", "products"),
"user": os.getenv("PG_USER", "postgres"),
"password": os.getenv("PG_PASSWORD"),
"table_name": "product_vectors"
}
)
return client
def setup_with_chromadb():
"""Development/Small-scale setup with ChromaDB"""
print("Setting up with ChromaDB (Local/Development)")
client = ImageSearchClient(
data_dir="./production_data",
model_name="clip-ViT-L-14",
vector_store_type="chromadb",
vector_store_config={
"mode": "persistent",
"path": "./production_data/chroma_vectors"
}
)
return client
def main():
print("Production Setup Example")
print("=" * 70)
# Choose your vector store (set via environment variable)
vector_store = os.getenv("VECTOR_STORE", "chromadb").lower()
print(f"\nUsing vector store: {vector_store}")
print("=" * 70)
# Initialize based on choice
if vector_store == "pinecone":
client = setup_with_pinecone()
elif vector_store == "qdrant":
client = setup_with_qdrant()
elif vector_store == "weaviate":
client = setup_with_weaviate()
elif vector_store == "pgvector":
client = setup_with_pgvector()
else:
client = setup_with_chromadb()
print("\nβœ“ SDK initialized")
# Connect to your production database
print("\nConnecting to production database...")
try:
client.connect_database(
connection_string=os.getenv("DATABASE_URL"),
table_name=os.getenv("PRODUCTS_TABLE", "products"),
id_column="id",
title_column="title",
image_url_column="image_url",
category_column="category",
brand_column="brand",
price_column="price",
rating_column="rating",
description_column="description"
)
print("βœ“ Database connected")
except Exception as e:
print(f"βœ— Database connection failed: {e}")
return
# Initialize embeddings (with error handling)
print("\nInitializing embeddings...")
try:
stats = client.initialize(batch_size=100)
print(f"βœ“ Initialization complete")
print(f" - Indexed: {stats['indexed_products']:,} products")
print(f" - Failed: {stats['failed_products']:,} products")
print(f" - Rate: {stats['products_per_second']:.1f} products/sec")
if stats['errors']:
print(f"\n⚠ Errors encountered: {len(stats['errors'])}")
# Log errors to file or monitoring system
with open("indexing_errors.log", "w") as f:
for error in stats['errors']:
f.write(f"{error}\n")
except Exception as e:
print(f"βœ— Initialization failed: {e}")
return
# Enable auto-monitoring for production
print("\nEnabling auto-monitoring...")
try:
monitor_interval = int(os.getenv("MONITOR_INTERVAL", "60"))
client.enable_auto_monitor(
poll_interval=monitor_interval,
enable=True
)
print(f"βœ“ Auto-monitoring enabled (interval: {monitor_interval}s)")
except Exception as e:
print(f"⚠ Auto-monitoring failed: {e}")
# Test search functionality
print("\nTesting search...")
try:
test_results = client.search_by_text(
"test query",
options=SearchOptions(top_k=1)
)
print(f"βœ“ Search working ({len(test_results)} result)")
except Exception as e:
print(f"βœ— Search test failed: {e}")
# Display statistics
print("\n" + "=" * 70)
print("PRODUCTION STATISTICS")
print("=" * 70)
stats = client.get_stats()
print(f"Vector Store: {stats.get('store_type', 'N/A')}")
print(f"Total Vectors: {stats.get('total_vectors', 0):,}")
print(f"Status: {'Operational' if stats.get('is_initialized') else 'Not Ready'}")
print("\n" + "=" * 70)
print("Production setup complete!")
print("The system is now running and monitoring for new products.")
print("=" * 70)
# Keep running (for monitoring)
print("\nPress Ctrl+C to shutdown...")
try:
import time
while True:
time.sleep(60)
# Optionally log health metrics here
current_stats = client.get_stats()
print(f"[Health Check] Vectors: {current_stats.get('total_vectors', 0):,}")
except KeyboardInterrupt:
print("\n\nShutting down...")
client.close()
print("βœ“ Shutdown complete")
if __name__ == "__main__":
main()