Update backend with high-quality catalog and fixes

app/api/product_quality_checker_lib.py

@@ -0,0 +1,496 @@
+#!/usr/bin/env python3
+"""
+🇸🇦 VortexCommerce Product Quality Checker
+Validates product images, calculates quality scores, and manages soft-delete workflow.
+"""
+import os
+import sys
+import json
+import logging
+import hashlib
+import requests
+from datetime import datetime, timezone, timedelta
+from typing import Dict, List, Any, Optional, Tuple
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import urllib3
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+if __name__ == "__main__":
+    import os
+    import sys
+    sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+try:
+    from app.db.base import engine, SessionLocal, Base
+    from app.models.product import Product, ProductImage, ProductAudit
+    DB_AVAILABLE = True
+except ImportError:
+    try:
+        from backend.app.db.base import engine, SessionLocal, Base
+        from backend.app.models.product import Product, ProductImage, ProductAudit
+        DB_AVAILABLE = True
+    except ImportError:
+        # Standalone mode - we'll define mock models if needed or just skip DB parts
+        engine = None
+        SessionLocal = None
+        Base = None
+        Product = None
+        DB_AVAILABLE = False
+
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.FileHandler(f'product_quality_{datetime.now().strftime("%Y%m%d_%H%M%S")}.log'),
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__)
+
+
+class QualityReport:
+    """Quality check report container."""
+    
+    def __init__(self):
+        self.timestamp = datetime.now(timezone.utc)
+        self.total_checked = 0
+        self.total_valid = 0
+        self.total_invalid = 0
+        self.deleted_products: List[Dict[str, Any]] = []
+        self.updated_scores: List[Dict[str, Any]] = []
+        self.errors: List[str] = []
+        
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "timestamp": self.timestamp.isoformat(),
+            "total_checked": self.total_checked,
+            "total_valid": self.total_valid,
+            "total_invalid": self.total_invalid,
+            "deleted_products": self.deleted_products,
+            "updated_scores": self.updated_scores,
+            "errors": self.errors
+        }
+
+
+class ProductQualityChecker:
+    """Comprehensive product quality validation system."""
+    
+    def __init__(self, db_session=None):
+        self.db = db_session
+        self.report = QualityReport()
+        self.session = requests.Session()
+        self.session.headers.update({
+            'User-Agent': 'VortexCommerce-QualityChecker/1.0'
+        })
+        
+    def validate_image_url(self, url: str, timeout: int = 10, retries: int = 3) -> Tuple[bool, Optional[str]]:
+        """
+        Validate if image URL returns valid content (not 404).
+        Returns (is_valid, error_message).
+        """
+        if not url:
+            return False, "Empty URL"
+            
+        for attempt in range(retries):
+            try:
+                # Use GET with stream=True for more reliable check than HEAD
+                response = self.session.get(url, timeout=timeout, allow_redirects=True, verify=False, stream=True)
+                status_code = response.status_code
+                content_type = response.headers.get('Content-Type', '')
+                response.close()
+                
+                if status_code == 200 and content_type.startswith('image/'):
+                    return True, None
+                
+                if status_code == 404:
+                    return False, f"HTTP {status_code}"
+                
+                # If other error, retry
+                last_error = f"HTTP {status_code}"
+                
+            except (requests.exceptions.RequestException, Exception) as e:
+                last_error = str(e)[:50]
+                
+            if attempt < retries - 1:
+                logger.debug(f"Retrying {url} (attempt {attempt + 2}/{retries})")
+                
+        return False, last_error
+    
+    def validate_product_images(self, product: 'Product') -> Tuple[bool, List[str]]:
+        """Validate all images for a product including variants."""
+        errors = []
+        
+        # Check primary images
+        if not product.images:
+            errors.append("No primary images attached to product")
+        else:
+            for img in product.images:
+                is_valid, error = self.validate_image_url(img.image_url)
+                if not is_valid:
+                    errors.append(f"Primary Image {img.id}: {error}")
+        
+        # Check variant images if present in specs
+        if product.specs and isinstance(product.specs, dict) and 'variants' in product.specs:
+            variants = product.specs['variants']
+            for i, var in enumerate(variants):
+                img_url = var.get('image_url') or var.get('image')
+                if img_url:
+                    is_valid, error = self.validate_image_url(img_url)
+                    if not is_valid:
+                        errors.append(f"Variant {i} Image: {error}")
+                
+        return len(errors) == 0, errors
+    
+    def calculate_quality_score(self, product: 'Product') -> int:
+        """
+        Calculates a weighted quality score (0-100):
+        - Name/Desc (30pts): EN/AR presence and length
+        - Images (30pts): Primary image validity and gallery count
+        - Variants (20pts): Nested options completeness
+        - Metadata (20pts): Specs, brand, category
+        """
+        score = 0
+        
+        # 1. Names & Descriptions (30 pts)
+        if product.name_en and len(product.name_en) > 10: score += 7
+        if product.name_ar and len(product.name_ar) > 10: score += 8
+        if product.description_en and len(product.description_en) > 50: score += 7
+        if product.description_ar and len(product.description_ar) > 50: score += 8
+        
+        # 2. Images (30 pts)
+        if product.images and len(product.images) > 0:
+            score += 15 # Has at least one image
+            if len(product.images) >= 3: score += 15 # Good gallery
+            elif len(product.images) == 2: score += 10
+        
+        # 3. Variants/Options (20 pts)
+        has_options = False
+        if product.specs and isinstance(product.specs, dict):
+            options = product.specs.get('options')
+            if options and len(options) > 0:
+                has_options = True
+                score += 20
+        
+        # 4. Metadata (20 pts)
+        if product.category_id: score += 5
+        if product.price > 0: score += 5
+        if product.stock > 0: score += 5
+        if product.specs and len(product.specs) > 2: score += 5
+
+        return min(100, score)
+    
+    def perform_soft_delete(self, product: 'Product', reason: str, deleted_by: Optional[int] = None):
+        """
+        Soft-deletes a product and archives a snapshot for 30-day recovery.
+        """
+        try:
+            # Prepare recovery snapshot
+            snapshot = {
+                "name_en": product.name_en,
+                "name_ar": product.name_ar,
+                "price": product.price,
+                "stock": product.stock,
+                "specs": product.specs,
+                "images": [img.image_url for img in product.images],
+                "soft_deleted_at": datetime.now(timezone.utc).isoformat()
+            }
+            
+            product.deleted_at = datetime.now(timezone.utc)
+            product.is_active = False
+            product.deletion_reason = reason
+            product.deleted_by = deleted_by
+            
+            # Log audit trail
+            audit = ProductAudit(
+                product_id=product.id,
+                action="IMAGE_FAILURE_AUTO_DELETE",
+                reason=reason,
+                snapshot=snapshot,
+                performed_by=deleted_by
+            )
+            self.db.add(audit)
+            
+            self.report.deleted_products.append({
+                "id": product.id,
+                "reason": reason
+            })
+            self.report.total_invalid += 1
+            logger.warning(f"🚨 AUTO-DELETED product {product.id} due to {reason}")
+            return True
+        except Exception as e:
+            logger.error(f"Failed to soft-delete {product.id}: {e}")
+            return False
+    
+    def process_product(self, product: 'Product') -> bool:
+        """Process quality lifecycle for a single product."""
+        self.report.total_checked += 1
+        
+        # 1. Critical Image Integrity Check (Phase 3 Requirement)
+        if not product.images:
+            return self.perform_soft_delete(product, "Missing all images")
+            
+        for img in product.images:
+            is_valid, error = self.validate_image_url(img.image_url)
+            if not is_valid:
+                return self.perform_soft_delete(product, f"Image 404/Error: {error}")
+        
+        # 2. Score Calculation
+        old_score = product.quality_score
+        new_score = self.calculate_quality_score(product)
+        
+        if old_score != new_score:
+            product.quality_score = new_score
+            self.report.updated_scores.append({
+                "id": product.id,
+                "old": old_score,
+                "new": new_score
+            })
+            
+        self.report.total_valid += 1
+        return True
+    
+    def run_quality_check(self, batch_size: int = 50) -> QualityReport:
+        """Run full quality check on all active products."""
+        if not self.db:
+            logger.error("No database session available")
+            return self.report
+            
+        logger.info("=" * 60)
+        logger.info("Starting Product Quality Check")
+        logger.info("=" * 60)
+        
+        try:
+            products = self.db.query(Product).filter(
+                Product.is_active == True,
+                Product.deleted_at == None
+            ).all()
+            
+            logger.info(f"Found {len(products)} active products to check")
+            
+            for idx, product in enumerate(products, 1):
+                self.process_product(product)
+                
+                if idx % batch_size == 0:
+                    self.db.commit()
+                    logger.info(f"Processed {idx}/{len(products)} products...")
+                    
+            self.db.commit()
+            
+        except Exception as e:
+            logger.error(f"Quality check failed: {str(e)}")
+            self.db.rollback()
+            self.report.errors.append(f"Quality check failed: {str(e)}")
+            
+        logger.info("=" * 60)
+        logger.info(f"Quality Check Complete")
+        logger.info(f"  Total Checked: {self.report.total_checked}")
+        logger.info(f"  Valid Products: {self.report.total_valid}")
+        logger.info(f"  Invalid (Deleted): {self.report.total_invalid}")
+        logger.info(f"  Score Updates: {len(self.report.updated_scores)}")
+        logger.info("=" * 60)
+        
+        return self.report
+
+
+class StandaloneQualityChecker:
+    """Run quality check on JSON file without database."""
+    
+    def __init__(self, json_path: str):
+        self.json_path = json_path
+        self.report = QualityReport()
+        self.session = requests.Session()
+        
+    def validate_image_url(self, url: str, timeout: int = 10) -> Tuple[bool, Optional[str]]:
+        """Validate single image URL."""
+        if not url:
+            return False, "Empty URL"
+            
+        try:
+            response = self.session.head(url, timeout=timeout, allow_redirects=True, verify=False)
+            
+            if response.status_code >= 400:
+                response = self.session.get(url, stream=True, timeout=timeout, verify=False)
+                response.close()
+                
+            if response.status_code >= 400:
+                return False, f"HTTP {response.status_code}"
+                
+            content_type = response.headers.get('Content-Type', '')
+            if not content_type.startswith('image/'):
+                return False, f"Invalid content type: {content_type}"
+                
+            return True, None
+            
+        except Exception as e:
+            return False, str(e)[:50]
+    
+    def check_json_products(self) -> Dict[str, Any]:
+        """Check products from JSON file."""
+        logger.info(f"Loading products from {self.json_path}")
+        
+        with open(self.json_path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+            
+        products = data.get('products', [])
+        logger.info(f"Found {len(products)} products to validate")
+        
+        results = {
+            "valid_products": [],
+            "invalid_products": [],
+            "summary": {
+                "total": len(products),
+                "valid": 0,
+                "invalid": 0
+            }
+        }
+        
+        for idx, product in enumerate(products, 1):
+            product_id = product.get('id', f'unknown_{idx}')
+            name_en = product.get('name_en', 'Unknown')
+            images = product.get('images', [])
+            
+            invalid_images = []
+            valid_images = []
+            
+            for img_url in images:
+                is_valid, error = self.validate_image_url(img_url)
+                if is_valid:
+                    valid_images.append(img_url)
+                else:
+                    invalid_images.append({"url": img_url, "error": error})
+            
+            product_result = {
+                "id": product_id,
+                "name_en": name_en,
+                "images": {
+                    "total": len(images),
+                    "valid": len(valid_images),
+                    "invalid": len(invalid_images),
+                    "invalid_details": invalid_images[:2]
+                }
+            }
+            
+            if invalid_images:
+                results["invalid_products"].append(product_result)
+                results["summary"]["invalid"] += 1
+                logger.warning(f"Product {product_id}: {len(invalid_images)} invalid images")
+            else:
+                results["valid_products"].append(product_result)
+                results["summary"]["valid"] += 1
+                
+            if idx % 50 == 0:
+                logger.info(f"Processed {idx}/{len(products)} products...")
+        
+        logger.info(f"Validation complete: {results['summary']['valid']} valid, {results['summary']['invalid']} invalid")
+        
+        output_path = self.json_path.replace('.json', '_validation.json')
+        with open(output_path, 'w', encoding='utf-8') as f:
+            json.dump(results, f, ensure_ascii=False, indent=2)
+            
+        logger.info(f"Results saved to {output_path}")
+        
+        return results
+
+
+def create_backup(db_session) -> Optional[str]:
+    """Create a backup of current products before quality operations."""
+    try:
+        products = db_session.query(Product).all()
+        
+        backup_data = {
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "total_products": len(products),
+            "products": []
+        }
+        
+        for p in products:
+            product_dict = {
+                "id": p.id,
+                "name_en": p.name_en,
+                "name_ar": p.name_ar,
+                "price": p.price,
+                "stock": p.stock,
+                "is_active": p.is_active,
+                "deleted_at": p.deleted_at.isoformat() if p.deleted_at else None,
+                "quality_score": p.quality_score,
+                "images": [{"id": img.id, "image_url": img.image_url} for img in p.images]
+            }
+            backup_data["products"].append(product_dict)
+        
+        backup_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "backups")
+        os.makedirs(backup_dir, exist_ok=True)
+        
+        backup_filename = f"products_backup_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
+        backup_path = os.path.join(backup_dir, backup_filename)
+        
+        with open(backup_path, 'w', encoding='utf-8') as f:
+            json.dump(backup_data, f, ensure_ascii=False, indent=2)
+            
+        logger.info(f"Backup created: {backup_path}")
+        return backup_path
+        
+    except Exception as e:
+        logger.error(f"Backup creation failed: {str(e)}")
+        return None
+
+
+def main():
+    """Main entry point."""
+    import argparse
+    
+    parser = argparse.ArgumentParser(description='VortexCommerce Product Quality Checker')
+    parser.add_argument('--mode', choices=['db', 'json'], default='db',
+                        help='Run against database or JSON file')
+    parser.add_argument('--json-path', 
+                        default=os.path.join(os.path.dirname(os.path.dirname(__file__)), 
+                                            'backend', 'real_products_1000.json'),
+                        help='Path to JSON file for standalone mode')
+    parser.add_argument('--backup', action='store_true',
+                        help='Create backup before quality operations')
+    parser.add_argument('--batch-size', type=int, default=50,
+                        help='Batch size for database commits')
+    
+    args = parser.parse_args()
+    
+    logger.info("=" * 60)
+    logger.info("VortexCommerce Product Quality Checker v2.0")
+    logger.info("=" * 60)
+    
+    if args.mode == 'json':
+        logger.info(f"Running in standalone JSON mode: {args.json_path}")
+        checker = StandaloneQualityChecker(args.json_path)
+        results = checker.check_json_products()
+        logger.info(f"Summary: {results['summary']}")
+        return 0
+    
+    if not DB_AVAILABLE:
+        logger.error("Database modules not available. Use --mode json for standalone mode.")
+        return 1
+    
+    db = SessionLocal()
+    
+    try:
+        if args.backup:
+            backup_path = create_backup(db)
+            if not backup_path:
+                logger.warning("Continuing without backup...")
+        
+        checker = ProductQualityChecker(db)
+        report = checker.run_quality_check(args.batch_size)
+        
+        report_path = os.path.join(os.path.dirname(__file__), f"quality_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json")
+        with open(report_path, 'w', encoding='utf-8') as f:
+            json.dump(report.to_dict(), f, ensure_ascii=False, indent=2)
+            
+        logger.info(f"Quality report saved: {report_path}")
+        
+    finally:
+        db.close()
+    
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

app/api/seed.py

@@ -117,14 +117,26 @@ async def seed_database(clear: bool = False, db: Session = Depends(get_db)):
         db.add(admin_user)
         db.commit()
 
-    json_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "backend", "real_products_1000.json")
-    if not os.path.exists(json_path):
-        json_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "real_products_1000.json")
-        
-    if not os.path.exists(json_path):
+    # Search for real_products_1000.json in multiple likely locations
+    possible_paths = [
+        os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "backend", "real_products_1000.json"),
+        os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "real_products_1000.json"),
+        os.path.abspath("real_products_1000.json"),
+        os.path.abspath("backend/real_products_1000.json"),
+        "/app/real_products_1000.json",
+        "/app/backend/real_products_1000.json"
+    ]
+    
+    json_path = None
+    for p in possible_paths:
+        if os.path.exists(p):
+            json_path = p
+            break
+            
+    if not json_path:
         return AuthResponse(
             isSuccess=False,
-            value={"message": f"Source file {json_path} not found."},
+            value={"message": f"Source file real_products_1000.json not found in attempted paths."},
             statusCode=404,
             error="File not found"
         )

app/main.py

@@ -66,6 +66,27 @@ def create_app() -> FastAPI:
             "statusCode": 200,
         }
 
+    @app.on_event("startup")
+    async def startup_event():
+        from app.db.base import SessionLocal
+        from app.models.product import Product
+        from app.api.seed import seed_database
+        
+        db = SessionLocal()
+        try:
+            # Check if we have any products
+            count = db.query(Product).count()
+            if count == 0:
+                print(">>> [BOOTSTRAP] Database is empty. Auto-seeding from real_products_1000.json...")
+                await seed_database(clear=False, db=db)
+                print(">>> [BOOTSTRAP] Seeding complete.")
+            else:
+                print(f">>> [BOOTSTRAP] Database already has {count} products. Skipping seed.")
+        except Exception as e:
+            print(f">>> [BOOTSTRAP] Auto-seeding error: {e}")
+        finally:
+            db.close()
+
     return app