Spaces:
Sleeping
Sleeping
File size: 4,038 Bytes
a475b1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 | """
Batch Indexing Example - Efficiently index large product catalogs.
This example demonstrates how to index a large number of products
efficiently using batch operations and progress tracking.
"""
import sys
from pathlib import Path
from datetime import datetime
# Add SDK to path
sdk_path = Path(__file__).parent.parent
sys.path.insert(0, str(sdk_path))
from image_search_sdk import ImageSearchClient, Product
def main():
print("Batch Indexing Example")
print("=" * 70)
# Initialize SDK
print("\nInitializing SDK with production-ready configuration...")
client = ImageSearchClient(
data_dir="./batch_data",
model_name="clip-ViT-L-14",
vector_store_type="chromadb", # Can use pinecone, qdrant, etc. in production
vector_store_config={
"mode": "persistent",
"path": "./batch_data/vectors"
}
)
# Connect to your database
print("Connecting to product database...")
client.connect_database(
connection_string="postgresql://user:pass@localhost/products", # Example
table_name="products",
id_column="product_id",
title_column="product_name",
image_url_column="main_image_url",
description_column="description",
category_column="category",
brand_column="brand_name",
price_column="sale_price"
)
# Method 1: Automatic initialization (recommended)
print("\n" + "=" * 70)
print("METHOD 1: Automatic Batch Initialization")
print("=" * 70)
start_time = datetime.now()
stats = client.initialize(batch_size=100) # Process 100 products at a time
end_time = datetime.now()
print(f"\nIndexing Complete!")
print(f" Total Products: {stats['total_products']}")
print(f" Successfully Indexed: {stats['indexed_products']}")
print(f" Failed: {stats['failed_products']}")
print(f" Duration: {stats['duration_seconds']:.1f}s")
print(f" Rate: {stats['products_per_second']:.1f} products/second")
if stats['errors']:
print(f"\nErrors encountered:")
for error in stats['errors'][:5]: # Show first 5 errors
print(f" - {error}")
# Method 2: Manual batch addition (for more control)
print("\n" + "=" * 70)
print("METHOD 2: Manual Batch Addition")
print("=" * 70)
# Example: Adding products manually
new_products = [
Product(
id="new_001",
title="Premium Wireless Headphones",
image_url="https://example.com/headphones.jpg",
category="Electronics",
brand="AudioTech",
price=299.99,
description="High-quality noise-canceling headphones"
),
Product(
id="new_002",
title="Smart Watch Pro",
image_url="https://example.com/smartwatch.jpg",
category="Electronics",
brand="TechWear",
price=399.99,
description="Advanced fitness tracking smartwatch"
),
# Add more products...
]
print(f"\nAdding {len(new_products)} new products manually...")
for product in new_products:
try:
success = client.add_product(product)
if success:
print(f" ✓ Added: {product.title}")
else:
print(f" ✗ Failed: {product.title}")
except Exception as e:
print(f" ✗ Error adding {product.title}: {e}")
# Get final statistics
print("\n" + "=" * 70)
print("FINAL STATISTICS")
print("=" * 70)
final_stats = client.get_stats()
print(f"Vector Store Type: {final_stats.get('store_type', 'N/A')}")
print(f"Total Vectors Stored: {final_stats.get('total_vectors', 0):,}")
print(f"Initialization Status: {'Complete' if final_stats.get('is_initialized') else 'Pending'}")
# Cleanup
print("\nClosing client...")
client.close()
print("Done!")
if __name__ == "__main__":
main()
|