""" Batch Indexing Example - Efficiently index large product catalogs. This example demonstrates how to index a large number of products efficiently using batch operations and progress tracking. """ import sys from pathlib import Path from datetime import datetime # Add SDK to path sdk_path = Path(__file__).parent.parent sys.path.insert(0, str(sdk_path)) from image_search_sdk import ImageSearchClient, Product def main(): print("Batch Indexing Example") print("=" * 70) # Initialize SDK print("\nInitializing SDK with production-ready configuration...") client = ImageSearchClient( data_dir="./batch_data", model_name="clip-ViT-L-14", vector_store_type="chromadb", # Can use pinecone, qdrant, etc. in production vector_store_config={ "mode": "persistent", "path": "./batch_data/vectors" } ) # Connect to your database print("Connecting to product database...") client.connect_database( connection_string="postgresql://user:pass@localhost/products", # Example table_name="products", id_column="product_id", title_column="product_name", image_url_column="main_image_url", description_column="description", category_column="category", brand_column="brand_name", price_column="sale_price" ) # Method 1: Automatic initialization (recommended) print("\n" + "=" * 70) print("METHOD 1: Automatic Batch Initialization") print("=" * 70) start_time = datetime.now() stats = client.initialize(batch_size=100) # Process 100 products at a time end_time = datetime.now() print(f"\nIndexing Complete!") print(f" Total Products: {stats['total_products']}") print(f" Successfully Indexed: {stats['indexed_products']}") print(f" Failed: {stats['failed_products']}") print(f" Duration: {stats['duration_seconds']:.1f}s") print(f" Rate: {stats['products_per_second']:.1f} products/second") if stats['errors']: print(f"\nErrors encountered:") for error in stats['errors'][:5]: # Show first 5 errors print(f" - {error}") # Method 2: Manual batch addition (for more control) print("\n" + "=" * 70) print("METHOD 2: Manual Batch Addition") print("=" * 70) # Example: Adding products manually new_products = [ Product( id="new_001", title="Premium Wireless Headphones", image_url="https://example.com/headphones.jpg", category="Electronics", brand="AudioTech", price=299.99, description="High-quality noise-canceling headphones" ), Product( id="new_002", title="Smart Watch Pro", image_url="https://example.com/smartwatch.jpg", category="Electronics", brand="TechWear", price=399.99, description="Advanced fitness tracking smartwatch" ), # Add more products... ] print(f"\nAdding {len(new_products)} new products manually...") for product in new_products: try: success = client.add_product(product) if success: print(f" ✓ Added: {product.title}") else: print(f" ✗ Failed: {product.title}") except Exception as e: print(f" ✗ Error adding {product.title}: {e}") # Get final statistics print("\n" + "=" * 70) print("FINAL STATISTICS") print("=" * 70) final_stats = client.get_stats() print(f"Vector Store Type: {final_stats.get('store_type', 'N/A')}") print(f"Total Vectors Stored: {final_stats.get('total_vectors', 0):,}") print(f"Initialization Status: {'Complete' if final_stats.get('is_initialized') else 'Pending'}") # Cleanup print("\nClosing client...") client.close() print("Done!") if __name__ == "__main__": main()