image-search-api / scripts /clean_invalid_images.py
mbishu's picture
Skip existing embeddings on startup and add invalid image cleanup script
6b0e776
Raw History Blame Contribute Delete
7.55 kB
"""
Script to clean invalid images from SQLite database.
This script:
1. Checks all products in the SQLite database
2. Validates each product's image URL is accessible and valid
3. Deletes products with invalid images
Usage:
python scripts/clean_invalid_images.py \
--database-path app/data/products.db
# Dry run (only report, don't delete)
python scripts/clean_invalid_images.py \
--database-path app/data/products.db \
--dry-run
Options:
--database-path: Path to SQLite database file (required)
--dry-run: Only report invalid images without deleting them
--timeout: Request timeout in seconds (default: 30)
--max-retries: Maximum retry attempts (default: 2)
"""
import argparse
import sys
import sqlite3
import time
from pathlib import Path
from typing import Tuple
from urllib.parse import urlparse
try:
import requests
from PIL import Image
import io
except ImportError as e:
print(f"ERROR: Required packages not installed: {e}")
print("Install with: pip install requests pillow")
sys.exit(1)
def is_valid_image_url(url: str, timeout: int = 30, max_retries: int = 2) -> Tuple[bool, str]:
"""
Check if an image URL is valid and accessible.
Returns:
(is_valid, error_message)
"""
if not url or not url.strip():
return False, "Empty URL"
# Basic URL validation
try:
parsed = urlparse(url)
if not parsed.scheme or not parsed.netloc:
return False, "Invalid URL format"
except Exception as e:
return False, f"URL parse error: {e}"
# Try to fetch and validate image
retry_delay = 1
last_error = None
for attempt in range(max_retries):
try:
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
}
response = requests.get(url, timeout=timeout, headers=headers, allow_redirects=True)
response.raise_for_status()
# Check content type
content_type = response.headers.get('content-type', '').lower()
if not any(img_type in content_type for img_type in ['image/', 'application/octet-stream']):
return False, f"Invalid content type: {content_type}"
# Validate image can be opened
try:
image = Image.open(io.BytesIO(response.content))
# Check dimensions
if image.size[0] < 10 or image.size[1] < 10:
return False, f"Image too small: {image.size}"
# Try to convert to RGB (validates image format)
if image.mode != 'RGB':
image = image.convert('RGB')
return True, ""
except Exception as e:
return False, f"Invalid image format: {e}"
except requests.exceptions.Timeout:
last_error = "Request timeout"
if attempt < max_retries - 1:
time.sleep(retry_delay)
retry_delay *= 2
continue
except requests.exceptions.RequestException as e:
last_error = f"Request failed: {e}"
if attempt < max_retries - 1:
time.sleep(retry_delay)
retry_delay *= 2
continue
except Exception as e:
return False, f"Unexpected error: {e}"
return False, last_error or "Failed after retries"
def clean_sqlite(database_path: str, dry_run: bool = False, timeout: int = 30, max_retries: int = 2) -> Tuple[int, int]:
"""Clean invalid images from SQLite database."""
conn = sqlite3.connect(database_path)
conn.row_factory = sqlite3.Row
try:
# Get all products
cursor = conn.execute("SELECT id, title, image_url FROM products")
products = cursor.fetchall()
if not products:
print("No products found in database")
return 0, 0
print(f"Found {len(products)} products to check...")
invalid_count = 0
checked_count = 0
for product in products:
product_id = product['id']
title = product['title']
image_url = product['image_url']
checked_count += 1
print(f"[{checked_count}/{len(products)}] Checking {product_id}: {title[:50]}...", end=" ")
is_valid, error = is_valid_image_url(image_url, timeout, max_retries)
if is_valid:
print("[OK] Valid")
else:
print(f"[INVALID] {error}")
invalid_count += 1
if not dry_run:
# Delete product
conn.execute("DELETE FROM products WHERE id = ?", (product_id,))
conn.commit()
print(f" -> Deleted product {product_id}")
return invalid_count, checked_count
finally:
conn.close()
def main():
parser = argparse.ArgumentParser(description="Clean invalid images from SQLite database")
parser.add_argument(
"--database-path",
type=str,
required=True,
help="Path to SQLite database file (e.g., app/data/products.db)"
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Only report invalid images without deleting them"
)
parser.add_argument(
"--timeout",
type=int,
default=30,
help="Request timeout in seconds (default: 30)"
)
parser.add_argument(
"--max-retries",
type=int,
default=2,
help="Maximum retry attempts (default: 2)"
)
args = parser.parse_args()
print("=" * 60)
print("Image Cleanup Script (SQLite)")
print("=" * 60)
if args.dry_run:
print("DRY RUN MODE - No products will be deleted")
print()
try:
# Check if database exists
if not Path(args.database_path).exists():
print(f"ERROR: Database file not found: {args.database_path}")
sys.exit(1)
print(f"Database: {args.database_path}")
invalid_count, checked_count = clean_sqlite(
args.database_path,
dry_run=args.dry_run,
timeout=args.timeout,
max_retries=args.max_retries
)
print()
print("=" * 60)
print(f"Summary:")
print(f" Total checked: {checked_count}")
print(f" Invalid images: {invalid_count}")
print(f" Valid images: {checked_count - invalid_count}")
if args.dry_run:
print(f"\nDRY RUN - No products were deleted")
print(f"Run without --dry-run to delete {invalid_count} products with invalid images")
else:
print(f"\nDeleted {invalid_count} products with invalid images")
print("=" * 60)
except KeyboardInterrupt:
print("\n\nInterrupted by user")
sys.exit(1)
except Exception as e:
print(f"\nERROR: {e}")
import traceback
traceback.print_exc()
sys.exit(1)
if __name__ == "__main__":
main()