""" Script to clean invalid images from SQLite database. This script: 1. Checks all products in the SQLite database 2. Validates each product's image URL is accessible and valid 3. Deletes products with invalid images Usage: python scripts/clean_invalid_images.py \ --database-path app/data/products.db # Dry run (only report, don't delete) python scripts/clean_invalid_images.py \ --database-path app/data/products.db \ --dry-run Options: --database-path: Path to SQLite database file (required) --dry-run: Only report invalid images without deleting them --timeout: Request timeout in seconds (default: 30) --max-retries: Maximum retry attempts (default: 2) """ import argparse import sys import sqlite3 import time from pathlib import Path from typing import Tuple from urllib.parse import urlparse try: import requests from PIL import Image import io except ImportError as e: print(f"ERROR: Required packages not installed: {e}") print("Install with: pip install requests pillow") sys.exit(1) def is_valid_image_url(url: str, timeout: int = 30, max_retries: int = 2) -> Tuple[bool, str]: """ Check if an image URL is valid and accessible. Returns: (is_valid, error_message) """ if not url or not url.strip(): return False, "Empty URL" # Basic URL validation try: parsed = urlparse(url) if not parsed.scheme or not parsed.netloc: return False, "Invalid URL format" except Exception as e: return False, f"URL parse error: {e}" # Try to fetch and validate image retry_delay = 1 last_error = None for attempt in range(max_retries): try: headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8', } response = requests.get(url, timeout=timeout, headers=headers, allow_redirects=True) response.raise_for_status() # Check content type content_type = response.headers.get('content-type', '').lower() if not any(img_type in content_type for img_type in ['image/', 'application/octet-stream']): return False, f"Invalid content type: {content_type}" # Validate image can be opened try: image = Image.open(io.BytesIO(response.content)) # Check dimensions if image.size[0] < 10 or image.size[1] < 10: return False, f"Image too small: {image.size}" # Try to convert to RGB (validates image format) if image.mode != 'RGB': image = image.convert('RGB') return True, "" except Exception as e: return False, f"Invalid image format: {e}" except requests.exceptions.Timeout: last_error = "Request timeout" if attempt < max_retries - 1: time.sleep(retry_delay) retry_delay *= 2 continue except requests.exceptions.RequestException as e: last_error = f"Request failed: {e}" if attempt < max_retries - 1: time.sleep(retry_delay) retry_delay *= 2 continue except Exception as e: return False, f"Unexpected error: {e}" return False, last_error or "Failed after retries" def clean_sqlite(database_path: str, dry_run: bool = False, timeout: int = 30, max_retries: int = 2) -> Tuple[int, int]: """Clean invalid images from SQLite database.""" conn = sqlite3.connect(database_path) conn.row_factory = sqlite3.Row try: # Get all products cursor = conn.execute("SELECT id, title, image_url FROM products") products = cursor.fetchall() if not products: print("No products found in database") return 0, 0 print(f"Found {len(products)} products to check...") invalid_count = 0 checked_count = 0 for product in products: product_id = product['id'] title = product['title'] image_url = product['image_url'] checked_count += 1 print(f"[{checked_count}/{len(products)}] Checking {product_id}: {title[:50]}...", end=" ") is_valid, error = is_valid_image_url(image_url, timeout, max_retries) if is_valid: print("[OK] Valid") else: print(f"[INVALID] {error}") invalid_count += 1 if not dry_run: # Delete product conn.execute("DELETE FROM products WHERE id = ?", (product_id,)) conn.commit() print(f" -> Deleted product {product_id}") return invalid_count, checked_count finally: conn.close() def main(): parser = argparse.ArgumentParser(description="Clean invalid images from SQLite database") parser.add_argument( "--database-path", type=str, required=True, help="Path to SQLite database file (e.g., app/data/products.db)" ) parser.add_argument( "--dry-run", action="store_true", help="Only report invalid images without deleting them" ) parser.add_argument( "--timeout", type=int, default=30, help="Request timeout in seconds (default: 30)" ) parser.add_argument( "--max-retries", type=int, default=2, help="Maximum retry attempts (default: 2)" ) args = parser.parse_args() print("=" * 60) print("Image Cleanup Script (SQLite)") print("=" * 60) if args.dry_run: print("DRY RUN MODE - No products will be deleted") print() try: # Check if database exists if not Path(args.database_path).exists(): print(f"ERROR: Database file not found: {args.database_path}") sys.exit(1) print(f"Database: {args.database_path}") invalid_count, checked_count = clean_sqlite( args.database_path, dry_run=args.dry_run, timeout=args.timeout, max_retries=args.max_retries ) print() print("=" * 60) print(f"Summary:") print(f" Total checked: {checked_count}") print(f" Invalid images: {invalid_count}") print(f" Valid images: {checked_count - invalid_count}") if args.dry_run: print(f"\nDRY RUN - No products were deleted") print(f"Run without --dry-run to delete {invalid_count} products with invalid images") else: print(f"\nDeleted {invalid_count} products with invalid images") print("=" * 60) except KeyboardInterrupt: print("\n\nInterrupted by user") sys.exit(1) except Exception as e: print(f"\nERROR: {e}") import traceback traceback.print_exc() sys.exit(1) if __name__ == "__main__": main()