Spaces:
Sleeping
Sleeping
File size: 7,170 Bytes
a475b1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 | """
Production Setup Example - Enterprise-grade configuration.
This example shows how to set up the SDK for production use with:
- Multiple vector store options (Pinecone, Qdrant, Weaviate, pgVector)
- Error handling and monitoring
- Performance optimization
- Auto-monitoring for new products
"""
import sys
from pathlib import Path
import os
# Add SDK to path
sdk_path = Path(__file__).parent.parent
sys.path.insert(0, str(sdk_path))
from image_search_sdk import ImageSearchClient, SearchOptions
def setup_with_pinecone():
"""Production setup with Pinecone (Managed Vector Database)"""
print("Setting up with Pinecone (Managed)")
client = ImageSearchClient(
data_dir="./production_data",
model_name="clip-ViT-L-14",
vector_store_type="pinecone",
vector_store_config={
"api_key": os.getenv("PINECONE_API_KEY"),
"environment": os.getenv("PINECONE_ENVIRONMENT", "us-west1-gcp"),
"index_name": "product-search"
}
)
return client
def setup_with_qdrant():
"""Production setup with Qdrant (Self-hosted or Cloud)"""
print("Setting up with Qdrant")
client = ImageSearchClient(
data_dir="./production_data",
model_name="clip-ViT-L-14",
vector_store_type="qdrant",
vector_store_config={
"url": os.getenv("QDRANT_URL", "http://localhost:6333"),
"api_key": os.getenv("QDRANT_API_KEY"), # Optional for cloud
"collection_name": "products"
}
)
return client
def setup_with_weaviate():
"""Production setup with Weaviate"""
print("Setting up with Weaviate")
client = ImageSearchClient(
data_dir="./production_data",
model_name="clip-ViT-L-14",
vector_store_type="weaviate",
vector_store_config={
"url": os.getenv("WEAVIATE_URL", "http://localhost:8080"),
"api_key": os.getenv("WEAVIATE_API_KEY"), # Optional
}
)
return client
def setup_with_pgvector():
"""Production setup with pgVector (PostgreSQL extension)"""
print("Setting up with pgVector (PostgreSQL)")
client = ImageSearchClient(
data_dir="./production_data",
model_name="clip-ViT-L-14",
vector_store_type="pgvector",
vector_store_config={
"host": os.getenv("PG_HOST", "localhost"),
"port": int(os.getenv("PG_PORT", "5432")),
"database": os.getenv("PG_DATABASE", "products"),
"user": os.getenv("PG_USER", "postgres"),
"password": os.getenv("PG_PASSWORD"),
"table_name": "product_vectors"
}
)
return client
def setup_with_chromadb():
"""Development/Small-scale setup with ChromaDB"""
print("Setting up with ChromaDB (Local/Development)")
client = ImageSearchClient(
data_dir="./production_data",
model_name="clip-ViT-L-14",
vector_store_type="chromadb",
vector_store_config={
"mode": "persistent",
"path": "./production_data/chroma_vectors"
}
)
return client
def main():
print("Production Setup Example")
print("=" * 70)
# Choose your vector store (set via environment variable)
vector_store = os.getenv("VECTOR_STORE", "chromadb").lower()
print(f"\nUsing vector store: {vector_store}")
print("=" * 70)
# Initialize based on choice
if vector_store == "pinecone":
client = setup_with_pinecone()
elif vector_store == "qdrant":
client = setup_with_qdrant()
elif vector_store == "weaviate":
client = setup_with_weaviate()
elif vector_store == "pgvector":
client = setup_with_pgvector()
else:
client = setup_with_chromadb()
print("\nβ SDK initialized")
# Connect to your production database
print("\nConnecting to production database...")
try:
client.connect_database(
connection_string=os.getenv("DATABASE_URL"),
table_name=os.getenv("PRODUCTS_TABLE", "products"),
id_column="id",
title_column="title",
image_url_column="image_url",
category_column="category",
brand_column="brand",
price_column="price",
rating_column="rating",
description_column="description"
)
print("β Database connected")
except Exception as e:
print(f"β Database connection failed: {e}")
return
# Initialize embeddings (with error handling)
print("\nInitializing embeddings...")
try:
stats = client.initialize(batch_size=100)
print(f"β Initialization complete")
print(f" - Indexed: {stats['indexed_products']:,} products")
print(f" - Failed: {stats['failed_products']:,} products")
print(f" - Rate: {stats['products_per_second']:.1f} products/sec")
if stats['errors']:
print(f"\nβ Errors encountered: {len(stats['errors'])}")
# Log errors to file or monitoring system
with open("indexing_errors.log", "w") as f:
for error in stats['errors']:
f.write(f"{error}\n")
except Exception as e:
print(f"β Initialization failed: {e}")
return
# Enable auto-monitoring for production
print("\nEnabling auto-monitoring...")
try:
monitor_interval = int(os.getenv("MONITOR_INTERVAL", "60"))
client.enable_auto_monitor(
poll_interval=monitor_interval,
enable=True
)
print(f"β Auto-monitoring enabled (interval: {monitor_interval}s)")
except Exception as e:
print(f"β Auto-monitoring failed: {e}")
# Test search functionality
print("\nTesting search...")
try:
test_results = client.search_by_text(
"test query",
options=SearchOptions(top_k=1)
)
print(f"β Search working ({len(test_results)} result)")
except Exception as e:
print(f"β Search test failed: {e}")
# Display statistics
print("\n" + "=" * 70)
print("PRODUCTION STATISTICS")
print("=" * 70)
stats = client.get_stats()
print(f"Vector Store: {stats.get('store_type', 'N/A')}")
print(f"Total Vectors: {stats.get('total_vectors', 0):,}")
print(f"Status: {'Operational' if stats.get('is_initialized') else 'Not Ready'}")
print("\n" + "=" * 70)
print("Production setup complete!")
print("The system is now running and monitoring for new products.")
print("=" * 70)
# Keep running (for monitoring)
print("\nPress Ctrl+C to shutdown...")
try:
import time
while True:
time.sleep(60)
# Optionally log health metrics here
current_stats = client.get_stats()
print(f"[Health Check] Vectors: {current_stats.get('total_vectors', 0):,}")
except KeyboardInterrupt:
print("\n\nShutting down...")
client.close()
print("β Shutdown complete")
if __name__ == "__main__":
main()
|