Spaces:
Sleeping
Sleeping
Download scripts/clean_invalid_images.py from fmbishu/image-search-api: direct link, hf CLI and curl.
- Browser
- Download file 7.55 kB
-
https://huggingface.co/spaces/fmbishu/image-search-api/resolve/main/scripts/clean_invalid_images.py
- Command line
-
hf download hf://spaces/fmbishu/image-search-api/scripts/clean_invalid_images.py
-
curl -L -o clean_invalid_images.py https://huggingface.co/spaces/fmbishu/image-search-api/resolve/main/scripts/clean_invalid_images.py
7.55 kB
| """ | |
| Script to clean invalid images from SQLite database. | |
| This script: | |
| 1. Checks all products in the SQLite database | |
| 2. Validates each product's image URL is accessible and valid | |
| 3. Deletes products with invalid images | |
| Usage: | |
| python scripts/clean_invalid_images.py \ | |
| --database-path app/data/products.db | |
| # Dry run (only report, don't delete) | |
| python scripts/clean_invalid_images.py \ | |
| --database-path app/data/products.db \ | |
| --dry-run | |
| Options: | |
| --database-path: Path to SQLite database file (required) | |
| --dry-run: Only report invalid images without deleting them | |
| --timeout: Request timeout in seconds (default: 30) | |
| --max-retries: Maximum retry attempts (default: 2) | |
| """ | |
| import argparse | |
| import sys | |
| import sqlite3 | |
| import time | |
| from pathlib import Path | |
| from typing import Tuple | |
| from urllib.parse import urlparse | |
| try: | |
| import requests | |
| from PIL import Image | |
| import io | |
| except ImportError as e: | |
| print(f"ERROR: Required packages not installed: {e}") | |
| print("Install with: pip install requests pillow") | |
| sys.exit(1) | |
| def is_valid_image_url(url: str, timeout: int = 30, max_retries: int = 2) -> Tuple[bool, str]: | |
| """ | |
| Check if an image URL is valid and accessible. | |
| Returns: | |
| (is_valid, error_message) | |
| """ | |
| if not url or not url.strip(): | |
| return False, "Empty URL" | |
| # Basic URL validation | |
| try: | |
| parsed = urlparse(url) | |
| if not parsed.scheme or not parsed.netloc: | |
| return False, "Invalid URL format" | |
| except Exception as e: | |
| return False, f"URL parse error: {e}" | |
| # Try to fetch and validate image | |
| retry_delay = 1 | |
| last_error = None | |
| for attempt in range(max_retries): | |
| try: | |
| headers = { | |
| 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', | |
| 'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8', | |
| } | |
| response = requests.get(url, timeout=timeout, headers=headers, allow_redirects=True) | |
| response.raise_for_status() | |
| # Check content type | |
| content_type = response.headers.get('content-type', '').lower() | |
| if not any(img_type in content_type for img_type in ['image/', 'application/octet-stream']): | |
| return False, f"Invalid content type: {content_type}" | |
| # Validate image can be opened | |
| try: | |
| image = Image.open(io.BytesIO(response.content)) | |
| # Check dimensions | |
| if image.size[0] < 10 or image.size[1] < 10: | |
| return False, f"Image too small: {image.size}" | |
| # Try to convert to RGB (validates image format) | |
| if image.mode != 'RGB': | |
| image = image.convert('RGB') | |
| return True, "" | |
| except Exception as e: | |
| return False, f"Invalid image format: {e}" | |
| except requests.exceptions.Timeout: | |
| last_error = "Request timeout" | |
| if attempt < max_retries - 1: | |
| time.sleep(retry_delay) | |
| retry_delay *= 2 | |
| continue | |
| except requests.exceptions.RequestException as e: | |
| last_error = f"Request failed: {e}" | |
| if attempt < max_retries - 1: | |
| time.sleep(retry_delay) | |
| retry_delay *= 2 | |
| continue | |
| except Exception as e: | |
| return False, f"Unexpected error: {e}" | |
| return False, last_error or "Failed after retries" | |
| def clean_sqlite(database_path: str, dry_run: bool = False, timeout: int = 30, max_retries: int = 2) -> Tuple[int, int]: | |
| """Clean invalid images from SQLite database.""" | |
| conn = sqlite3.connect(database_path) | |
| conn.row_factory = sqlite3.Row | |
| try: | |
| # Get all products | |
| cursor = conn.execute("SELECT id, title, image_url FROM products") | |
| products = cursor.fetchall() | |
| if not products: | |
| print("No products found in database") | |
| return 0, 0 | |
| print(f"Found {len(products)} products to check...") | |
| invalid_count = 0 | |
| checked_count = 0 | |
| for product in products: | |
| product_id = product['id'] | |
| title = product['title'] | |
| image_url = product['image_url'] | |
| checked_count += 1 | |
| print(f"[{checked_count}/{len(products)}] Checking {product_id}: {title[:50]}...", end=" ") | |
| is_valid, error = is_valid_image_url(image_url, timeout, max_retries) | |
| if is_valid: | |
| print("[OK] Valid") | |
| else: | |
| print(f"[INVALID] {error}") | |
| invalid_count += 1 | |
| if not dry_run: | |
| # Delete product | |
| conn.execute("DELETE FROM products WHERE id = ?", (product_id,)) | |
| conn.commit() | |
| print(f" -> Deleted product {product_id}") | |
| return invalid_count, checked_count | |
| finally: | |
| conn.close() | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Clean invalid images from SQLite database") | |
| parser.add_argument( | |
| "--database-path", | |
| type=str, | |
| required=True, | |
| help="Path to SQLite database file (e.g., app/data/products.db)" | |
| ) | |
| parser.add_argument( | |
| "--dry-run", | |
| action="store_true", | |
| help="Only report invalid images without deleting them" | |
| ) | |
| parser.add_argument( | |
| "--timeout", | |
| type=int, | |
| default=30, | |
| help="Request timeout in seconds (default: 30)" | |
| ) | |
| parser.add_argument( | |
| "--max-retries", | |
| type=int, | |
| default=2, | |
| help="Maximum retry attempts (default: 2)" | |
| ) | |
| args = parser.parse_args() | |
| print("=" * 60) | |
| print("Image Cleanup Script (SQLite)") | |
| print("=" * 60) | |
| if args.dry_run: | |
| print("DRY RUN MODE - No products will be deleted") | |
| print() | |
| try: | |
| # Check if database exists | |
| if not Path(args.database_path).exists(): | |
| print(f"ERROR: Database file not found: {args.database_path}") | |
| sys.exit(1) | |
| print(f"Database: {args.database_path}") | |
| invalid_count, checked_count = clean_sqlite( | |
| args.database_path, | |
| dry_run=args.dry_run, | |
| timeout=args.timeout, | |
| max_retries=args.max_retries | |
| ) | |
| print() | |
| print("=" * 60) | |
| print(f"Summary:") | |
| print(f" Total checked: {checked_count}") | |
| print(f" Invalid images: {invalid_count}") | |
| print(f" Valid images: {checked_count - invalid_count}") | |
| if args.dry_run: | |
| print(f"\nDRY RUN - No products were deleted") | |
| print(f"Run without --dry-run to delete {invalid_count} products with invalid images") | |
| else: | |
| print(f"\nDeleted {invalid_count} products with invalid images") | |
| print("=" * 60) | |
| except KeyboardInterrupt: | |
| print("\n\nInterrupted by user") | |
| sys.exit(1) | |
| except Exception as e: | |
| print(f"\nERROR: {e}") | |
| import traceback | |
| traceback.print_exc() | |
| sys.exit(1) | |
| if __name__ == "__main__": | |
| main() | |