Spaces:
Sleeping
Sleeping
Download sdk/python/examples/batch_indexing.py from fmbishu/image-search-api: direct link, hf CLI and curl.
- Browser
- Download file 4.04 kB
-
https://huggingface.co/spaces/fmbishu/image-search-api/resolve/main/sdk/python/examples/batch_indexing.py
- Command line
-
hf download hf://spaces/fmbishu/image-search-api/sdk/python/examples/batch_indexing.py
-
curl -L -o batch_indexing.py https://huggingface.co/spaces/fmbishu/image-search-api/resolve/main/sdk/python/examples/batch_indexing.py
4.04 kB
| """ | |
| Batch Indexing Example - Efficiently index large product catalogs. | |
| This example demonstrates how to index a large number of products | |
| efficiently using batch operations and progress tracking. | |
| """ | |
| import sys | |
| from pathlib import Path | |
| from datetime import datetime | |
| # Add SDK to path | |
| sdk_path = Path(__file__).parent.parent | |
| sys.path.insert(0, str(sdk_path)) | |
| from image_search_sdk import ImageSearchClient, Product | |
| def main(): | |
| print("Batch Indexing Example") | |
| print("=" * 70) | |
| # Initialize SDK | |
| print("\nInitializing SDK with production-ready configuration...") | |
| client = ImageSearchClient( | |
| data_dir="./batch_data", | |
| model_name="clip-ViT-L-14", | |
| vector_store_type="chromadb", # Can use pinecone, qdrant, etc. in production | |
| vector_store_config={ | |
| "mode": "persistent", | |
| "path": "./batch_data/vectors" | |
| } | |
| ) | |
| # Connect to your database | |
| print("Connecting to product database...") | |
| client.connect_database( | |
| connection_string="postgresql://user:pass@localhost/products", # Example | |
| table_name="products", | |
| id_column="product_id", | |
| title_column="product_name", | |
| image_url_column="main_image_url", | |
| description_column="description", | |
| category_column="category", | |
| brand_column="brand_name", | |
| price_column="sale_price" | |
| ) | |
| # Method 1: Automatic initialization (recommended) | |
| print("\n" + "=" * 70) | |
| print("METHOD 1: Automatic Batch Initialization") | |
| print("=" * 70) | |
| start_time = datetime.now() | |
| stats = client.initialize(batch_size=100) # Process 100 products at a time | |
| end_time = datetime.now() | |
| print(f"\nIndexing Complete!") | |
| print(f" Total Products: {stats['total_products']}") | |
| print(f" Successfully Indexed: {stats['indexed_products']}") | |
| print(f" Failed: {stats['failed_products']}") | |
| print(f" Duration: {stats['duration_seconds']:.1f}s") | |
| print(f" Rate: {stats['products_per_second']:.1f} products/second") | |
| if stats['errors']: | |
| print(f"\nErrors encountered:") | |
| for error in stats['errors'][:5]: # Show first 5 errors | |
| print(f" - {error}") | |
| # Method 2: Manual batch addition (for more control) | |
| print("\n" + "=" * 70) | |
| print("METHOD 2: Manual Batch Addition") | |
| print("=" * 70) | |
| # Example: Adding products manually | |
| new_products = [ | |
| Product( | |
| id="new_001", | |
| title="Premium Wireless Headphones", | |
| image_url="https://example.com/headphones.jpg", | |
| category="Electronics", | |
| brand="AudioTech", | |
| price=299.99, | |
| description="High-quality noise-canceling headphones" | |
| ), | |
| Product( | |
| id="new_002", | |
| title="Smart Watch Pro", | |
| image_url="https://example.com/smartwatch.jpg", | |
| category="Electronics", | |
| brand="TechWear", | |
| price=399.99, | |
| description="Advanced fitness tracking smartwatch" | |
| ), | |
| # Add more products... | |
| ] | |
| print(f"\nAdding {len(new_products)} new products manually...") | |
| for product in new_products: | |
| try: | |
| success = client.add_product(product) | |
| if success: | |
| print(f" ✓ Added: {product.title}") | |
| else: | |
| print(f" ✗ Failed: {product.title}") | |
| except Exception as e: | |
| print(f" ✗ Error adding {product.title}: {e}") | |
| # Get final statistics | |
| print("\n" + "=" * 70) | |
| print("FINAL STATISTICS") | |
| print("=" * 70) | |
| final_stats = client.get_stats() | |
| print(f"Vector Store Type: {final_stats.get('store_type', 'N/A')}") | |
| print(f"Total Vectors Stored: {final_stats.get('total_vectors', 0):,}") | |
| print(f"Initialization Status: {'Complete' if final_stats.get('is_initialized') else 'Pending'}") | |
| # Cleanup | |
| print("\nClosing client...") | |
| client.close() | |
| print("Done!") | |
| if __name__ == "__main__": | |
| main() | |