Spaces:
Sleeping
Sleeping
| # ================================================================= | |
| # 🛒 Extra.com → VortexCommerce | Full Product Scraper | |
| # ================================================================= | |
| import asyncio, re, os, uuid, sqlite3, logging, zipfile, random, json, requests, time | |
| from datetime import datetime, timezone | |
| from typing import List, Optional, Dict, Any | |
| # Kaggle Notebook - ملف واحد شامل: | |
| # ✅ جلب المنتجات من extra.com (EN + AR) | |
| # ✅ زيارة صفحة كل منتج للحصول على: | |
| # - عدة صور حقيقية (Image Gallery) | |
| # - المواصفات التفصيلية (Specs) | |
| # - الاسم بالعربية والإنجليزية | |
| # - التقييم وعدد المراجعات | |
| # ✅ تصدير JSON بهيكلة VortexCommerce | |
| # ================================================================= | |
| # ================= CONFIGURATION ================= | |
| CATEGORIES_TO_SCRAPE = { | |
| "smartphones": "/en-sa/mobiles-tablets/mobiles/smartphone/c/2-212-3/facet/?q=%3Arelevance%3AinStock%3Atrue%3Atype%3APRODUCT&text=&pg={page}&pageSize=24&sort=relevance", | |
| "laptops": "/en-sa/computer/laptops/c/3-303/facet/?q=%3Arelevance%3AinStock%3Atrue%3Atype%3APRODUCT&text=&pg={page}&pageSize=24&sort=relevance", | |
| "tablets": "/en-sa/mobiles-tablets/tablets/c/2-217/facet/?q=%3Arelevance%3AinStock%3Atrue%3Atype%3APRODUCT&text=&pg={page}&pageSize=24&sort=relevance", | |
| "tvs": "/en-sa/electronics/television/c/1-109/facet/?q=%3Arelevance%3AinStock%3Atrue%3Atype%3APRODUCT&text=&pg={page}&pageSize=24&sort=relevance", | |
| "headphones": "/en-sa/mobiles-tablets/portable-audio/c/2-623/facet/?q=%3Arelevance%3AinStock%3Atrue%3Atype%3APRODUCT&text=&pg={page}&pageSize=24&sort=relevance", | |
| "smartwatches": "/en-sa/smart-watches/c/8126/facet/?q=%3Arelevance%3AinStock%3Atrue%3Atype%3APRODUCT&text=&pg={page}&pageSize=24&sort=relevance", | |
| "gaming": "/en-sa/electronic-games/c/13/facet/?q=%3Arelevance%3AinStock%3Atrue%3Atype%3APRODUCT&text=&pg={page}&pageSize=24&sort=relevance", | |
| "small_appliances": "/en-sa/small-appliances/c/5/facet/?q=%3Arelevance%3AinStock%3Atrue%3Atype%3APRODUCT&text=&pg={page}&pageSize=24&sort=relevance", | |
| "large_appliances": "/en-sa/white-goods/c/4/facet/?q=%3Arelevance%3AinStock%3Atrue%3Atype%3APRODUCT&text=&pg={page}&pageSize=24&sort=relevance", | |
| "cameras": "/en-sa/computer/cameras/c/3-603/facet/?q=%3Arelevance%3AinStock%3Atrue%3Atype%3APRODUCT&text=&pg={page}&pageSize=24&sort=relevance", | |
| } | |
| TARGET_PRODUCTS_PER_CATEGORY = 100 # Default target per category | |
| DATABASE_NAME = "extra_catalog.db" | |
| # للمزامنة المباشرة مع PostgreSQL (Supabase): | |
| # SUPABASE_DATABASE_URL = "postgresql://user:pass@host:port/dbname" | |
| DATABASE_URL = os.environ.get("DATABASE_URL", None) | |
| # ================================================= | |
| # ----------------------------------------------- | |
| # 1. تثبيت Playwright | |
| # ----------------------------------------------- | |
| try: | |
| import playwright | |
| except ImportError: | |
| print("📦 Installing Playwright & Dependencies...") | |
| !pip install playwright | |
| !playwright install chromium | |
| !playwright install-deps | |
| from playwright.async_api import async_playwright | |
| logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') | |
| logger = logging.getLogger("ExtraScraper") | |
| # ----------------------------------------------- | |
| # 2. الثوابت والهيكلية الاحترافية (Professional Hierarchy) | |
| # ----------------------------------------------- | |
| # الفئات الأساسية والفرعية مطابقة لتوزيع اكسترا وتصميم VortexCommerce | |
| CATEGORIES_HIERARCHY = { | |
| "الأجهزة الإلكترونية": { | |
| "en": "Electronics", | |
| "icon": "📱", | |
| "subs": { | |
| "smartphones": {"en": "Smartphones", "ar": "جوالات"}, | |
| "laptops": {"en": "Laptops", "ar": "لابتوب"}, | |
| "tablets": {"en": "Tablets", "ar": "أجهزة لوحية"}, | |
| "tvs": {"en": "TVs", "ar": "تلفزيونات"}, | |
| "cameras": {"en": "Cameras", "ar": "كاميرات"}, | |
| "printers": {"en": "Printers", "ar": "طابعات"} | |
| } | |
| }, | |
| "ملحقات واكسسوارات": { | |
| "en": "Accessories", | |
| "icon": "🔌", | |
| "subs": { | |
| "headphones": {"en": "Audio & Headphones", "ar": "أجهزة الصوت والسماعات"}, | |
| "smartwatches": {"en": "Smartwatches", "ar": "ساعات ذكية"} | |
| } | |
| }, | |
| "ألعاب جيمنج": { | |
| "en": "Gaming", | |
| "icon": "🎮", | |
| "subs": { | |
| "gaming": {"en": "Gaming Consoles", "ar": "أجهزة ألعاب"} | |
| } | |
| }, | |
| "الأجهزة المنزلية": { | |
| "en": "Home Appliances", | |
| "icon": "🏠", | |
| "subs": { | |
| "refrigerators": {"en": "Refrigerators", "ar": "ثلاجات"}, | |
| "washing-machines": {"en": "Washing Machines", "ar": "غسالات ومجففات"}, | |
| "air-conditioners": {"en": "Air Conditioners", "ar": "مكيفات"}, | |
| "small_appliances": {"en": "Small Appliances", "ar": "أجهزة منزلية صغيرة"}, | |
| "large_appliances": {"en": "Large Appliances", "ar": "أجهزة منزلية كبيرة"} | |
| } | |
| } | |
| } | |
| # Reverse map for quick parent lookup based on scraped category key | |
| EXTRA_CAT_TO_PARENT = {} | |
| for p_ar, info in CATEGORIES_HIERARCHY.items(): | |
| for sub_key in info["subs"].keys(): | |
| EXTRA_CAT_TO_PARENT[sub_key] = p_ar | |
| KNOWN_BRANDS = [ | |
| 'Apple', 'Samsung', 'Huawei', 'Xiaomi', 'Honor', 'Sony', 'HP', 'Dell', | |
| 'Lenovo', 'Asus', 'Acer', 'Microsoft', 'Nikon', 'Canon', 'Bose', | |
| 'Logitech', 'LG', 'Panasonic', 'Philips', 'Dyson', 'Bosch', 'Hisense', | |
| 'TCL', 'Class Pro', 'JBL', 'Anker', 'Oppo', 'Motorola', 'Tecno', | |
| 'Nokia', 'Google', 'Nothing', 'Realme', 'MSI', 'Razer', 'SteelSeries', | |
| 'HyperX', 'Corsair', 'PlayStation', 'Xbox', 'Nintendo', | |
| 'GoPro', 'DJI', 'Fujifilm', 'Garmin', 'Fitbit', 'Amazfit', 'Beats', | |
| 'Sennheiser', 'Jabra', 'Marshall', 'Harman Kardon', | |
| 'BISSELL', 'Gree', 'Haier', 'Midea', 'Toshiba', 'Whirlpool', | |
| 'Electrolux', 'Braun', 'Tefal', 'Kenwood', 'Moulinex', 'Black+Decker', | |
| 'Infinix', 'REDMI', 'POCO', 'OnePlus', 'EPSON', 'Brother', | |
| 'Skyworth', 'ASUS ROG', 'Bang & Olufsen', | |
| ] | |
| COLOR_MAP = { | |
| 'Black': 'أسود', 'White': 'أبيض', 'Silver': 'فضي', 'Gold': 'ذهبي', | |
| 'Blue': 'أزرق', 'Red': 'أحمر', 'Green': 'أخضر', 'Grey': 'رمادي', | |
| 'Gray': 'رمادي', 'Titanium': 'تيتانيوم', 'Purple': 'أرجواني', | |
| 'Pink': 'وردي', 'Orange': 'برتقالي', 'Yellow': 'أصفر', | |
| 'Rose Gold': 'ذهبي وردي', 'Space Gray': 'رمادي فلكي', | |
| 'Midnight': 'منتصف الليل', 'Starlight': 'ستارلايت', | |
| 'Cream': 'كريمي', 'Lavender': 'لافندر', 'Mint': 'نعناعي', | |
| 'Graphite': 'رصاصي غامق', 'Burgundy': 'عنابي', 'Navy': 'كحلي', | |
| 'Cobalt': 'كوبالت', 'Coral': 'مرجاني', 'Icyblue': 'أزرق ثلجي', | |
| 'Glacier Silver': 'فضي جليدي', 'Sky Blue': 'أزرق سماوي', | |
| } | |
| BRAND_AR = { | |
| 'Apple': 'آبل', 'Samsung': 'سامسونج', 'Huawei': 'هواوي', | |
| 'Xiaomi': 'شاومي', 'Honor': 'هونر', 'Sony': 'سوني', | |
| 'HP': 'إتش بي', 'Dell': 'ديل', 'Lenovo': 'لينوفو', | |
| 'Asus': 'أسوس', 'Acer': 'أيسر', 'Microsoft': 'مايكروسوفت', | |
| 'LG': 'إل جي', 'Panasonic': 'باناسونيك', 'Philips': 'فيلبس', | |
| 'Dyson': 'دايسون', 'Bosch': 'بوش', 'Canon': 'كانون', | |
| 'Nikon': 'نيكون', 'JBL': 'جي بي إل', 'Bose': 'بوز', | |
| 'Logitech': 'لوجيتيك', 'Oppo': 'أوبو', 'Motorola': 'موتورولا', | |
| 'Nokia': 'نوكيا', 'Google': 'جوجل', 'Beats': 'بيتس', | |
| 'Garmin': 'جارمين', 'Hisense': 'هايسنس', 'TCL': 'تي سي إل', | |
| 'PlayStation': 'بلايستيشن', 'Xbox': 'إكس بوكس', 'Nintendo': 'نينتندو', | |
| 'Tecno': 'تكنو', 'Infinix': 'إنفينيكس', 'Realme': 'ريلمي', | |
| 'Class Pro': 'كلاس برو', 'GoPro': 'جو برو', 'DJI': 'دي جي آي', | |
| 'MSI': 'إم إس آي', 'Razer': 'ريزر', 'Nothing': 'ناثينق', | |
| } | |
| # ----------------------------------------------- | |
| # 3. قاعدة البيانات | |
| # ----------------------------------------------- | |
| class Database: | |
| def __init__(self, path_or_url): | |
| self.is_pg = "postgresql" in path_or_url if path_or_url else False | |
| self.path_or_url = path_or_url or DATABASE_NAME | |
| if self.is_pg: | |
| try: | |
| import psycopg2 | |
| self.conn = psycopg2.connect(self.path_or_url) | |
| self.placeholder = "%s" | |
| except ImportError: | |
| print("📦 Installing psycopg2...") | |
| import subprocess | |
| subprocess.check_call(["pip", "install", "psycopg2-binary"]) | |
| import psycopg2 | |
| self.conn = psycopg2.connect(self.path_or_url) | |
| self.placeholder = "%s" | |
| else: | |
| self.conn = sqlite3.connect(self.path_or_url) | |
| self.placeholder = "?" | |
| self._ensure_tables() | |
| def _ensure_tables(self): | |
| cursor = self.conn.cursor() | |
| create_sql = f'''CREATE TABLE IF NOT EXISTS products ( | |
| id TEXT PRIMARY KEY, pid TEXT UNIQUE, | |
| title_en TEXT NOT NULL, title_ar TEXT, | |
| description_en TEXT, description_ar TEXT, | |
| price REAL NOT NULL, old_price REAL, | |
| currency TEXT DEFAULT 'SAR', | |
| images TEXT, product_url TEXT, | |
| category TEXT NOT NULL, brand TEXT, | |
| specs TEXT, rating REAL, rating_count INTEGER, | |
| created_at TEXT)''' | |
| cursor.execute(create_sql) | |
| self.conn.commit() | |
| def save_products(self, products): | |
| cursor = self.conn.cursor() | |
| saved = 0 | |
| for p in products: | |
| try: | |
| now = datetime.now(timezone.utc).isoformat() | |
| pid = p.get('pid') | |
| images_json = json.dumps(p.get('images', []), ensure_ascii=False) | |
| specs_json = json.dumps(p.get('specs', {}), ensure_ascii=False) | |
| cursor.execute(f"SELECT id FROM products WHERE pid = {self.placeholder}", (pid,)) | |
| existing = cursor.fetchone() | |
| if existing: | |
| cursor.execute(f'''UPDATE products SET | |
| title_en={self.placeholder}, title_ar={self.placeholder}, | |
| description_en={self.placeholder}, description_ar={self.placeholder}, | |
| price={self.placeholder}, old_price={self.placeholder}, | |
| images={self.placeholder}, product_url={self.placeholder}, | |
| brand={self.placeholder}, specs={self.placeholder}, | |
| rating={self.placeholder}, rating_count={self.placeholder} | |
| WHERE id={self.placeholder}''', | |
| (p['title_en'], p.get('title_ar',''), p.get('description_en',''), p.get('description_ar',''), | |
| p['price'], p.get('old_price'), images_json, p['product_url'], | |
| p.get('brand'), specs_json, p.get('rating'), p.get('rating_count'), existing[0])) | |
| else: | |
| cursor.execute(f'''INSERT INTO products ( | |
| id, pid, title_en, title_ar, description_en, description_ar, | |
| price, old_price, currency, images, product_url, | |
| category, brand, specs, rating, rating_count, created_at) | |
| VALUES ({self.placeholder},{self.placeholder},{self.placeholder},{self.placeholder}, | |
| {self.placeholder},{self.placeholder},{self.placeholder},{self.placeholder}, | |
| {self.placeholder},{self.placeholder},{self.placeholder},{self.placeholder}, | |
| {self.placeholder},{self.placeholder},{self.placeholder},{self.placeholder},{self.placeholder})''', | |
| (str(uuid.uuid4()), pid, p['title_en'], p.get('title_ar',''), | |
| p.get('description_en',''), p.get('description_ar',''), | |
| p['price'], p.get('old_price'), 'SAR', | |
| images_json, p['product_url'], | |
| p['category'], p.get('brand'), | |
| specs_json, p.get('rating'), p.get('rating_count'), now)) | |
| saved += 1 | |
| except Exception as e: | |
| logger.error(f"Error saving {p.get('title_en','?')}: {e}") | |
| self.conn.commit() | |
| return saved | |
| def close(self): | |
| self.conn.close() | |
| # ----------------------------------------------- | |
| # 4. السكرابر الرئيسي | |
| # ----------------------------------------------- | |
| # الاستراتيجية: | |
| # خطوة 1: جلب قائمة المنتجات من صفحة الفئة (PID + URL + السعر) | |
| # خطوة 2: زيارة كل صفحة منتج EN لجلب (العنوان EN + الصور + المواصفات) | |
| # خطوة 3: زيارة نفس المنتج AR لجلب (العنوان AR + الوصف AR + المواصفات AR) | |
| # ----------------------------------------------- | |
| # === JavaScript لاستخراج قائمة المنتجات من صفحة الفئة === | |
| LISTING_JS = r'''() => { | |
| const products = []; | |
| const seenPids = new Set(); | |
| // المحدد الأساسي المؤكد: a.product-tile-content-wrapper[data-product-id] | |
| const cards = document.querySelectorAll('a.product-tile-content-wrapper, [data-product-id]'); | |
| cards.forEach(card => { | |
| try { | |
| const pid = card.getAttribute('data-product-id') || card.getAttribute('data-sku') || ''; | |
| if (!pid || seenPids.has(pid)) return; | |
| seenPids.add(pid); | |
| let url = ''; | |
| if (card.tagName === 'A') url = card.href || ''; | |
| else { | |
| const a = card.querySelector('a[href*="/p/"]'); | |
| url = a ? a.href : ''; | |
| } | |
| if (!url || !url.includes('/p/')) return; | |
| // السعر: .price-container strong | |
| let price = 0, oldPrice = 0; | |
| const priceContainers = card.querySelectorAll('.price-container, [class*="price"]'); | |
| priceContainers.forEach(pc => { | |
| pc.querySelectorAll('strong').forEach(s => { | |
| const val = parseFloat((s.textContent || '').replace(/[^0-9.]/g, '')); | |
| if (val > 0 && price === 0) price = val; | |
| }); | |
| }); | |
| const striked = card.querySelector('.striked-off, [class*="striked"], del, s'); | |
| if (striked) { | |
| const val = parseFloat((striked.textContent || '').replace(/[^0-9.]/g, '')); | |
| if (val > 0) oldPrice = val; | |
| } | |
| if (price === 0) { | |
| const allText = card.textContent || ''; | |
| const m = allText.match(/(\d[\d,]*\.?\d*)\s*(?:SAR|ر\.س)/); | |
| if (m) price = parseFloat(m[1].replace(/,/g, '')); | |
| } | |
| if (price <= 0) return; | |
| if (oldPrice > 0 && oldPrice < price) [price, oldPrice] = [oldPrice, price]; | |
| // العلامة التجارية | |
| let brand = ''; | |
| const brandEl = card.querySelector('[class*="brand"]'); | |
| if (brandEl) brand = brandEl.textContent.trim(); | |
| if (!brand) { | |
| const bm = (card.textContent || '').match(/^[\s\n]*([A-Z][A-Z\s]+)\s*•/); | |
| if (bm) brand = bm[1].trim(); | |
| } | |
| products.push({ pid, product_url: url, price, old_price: oldPrice || null, brand }); | |
| } catch(e) {} | |
| }); | |
| // Fallback: بحث في الروابط | |
| if (products.length === 0) { | |
| document.querySelectorAll('a[href*="/p/"]').forEach(link => { | |
| const pidM = link.href.match(/\/p\/([A-Z0-9]+)/); | |
| if (!pidM || seenPids.has(pidM[1])) return; | |
| seenPids.add(pidM[1]); | |
| const parent = link.closest('li, [class*="product"], [class*="tile"]') || link.parentElement; | |
| let price = 0; | |
| const t = (parent?.textContent || ''); | |
| const pm = t.match(/(\d{3,6})/); | |
| if (pm) price = parseInt(pm[1]); | |
| if (price > 0) { | |
| products.push({ pid: pidM[1], product_url: link.href, price, old_price: null, brand: '' }); | |
| } | |
| }); | |
| } | |
| return products; | |
| }''' | |
| # === JavaScript لاستخراج التفاصيل من صفحة المنتج الإنجليزية === | |
| DETAIL_EN_JS = r'''() => { | |
| const result = { title: '', description: '', images: [], specs: {}, rating: 0, rating_count: 0, brand: '', variants: [], categories: [] }; | |
| // 0. استخراج الأقسام من Breadcrumbs أو URL | |
| const paths = window.location.pathname.split('/').filter(p => p && p !== 'en-sa' && p !== 'p'); | |
| result.categories = paths.slice(0, 3); | |
| // 1. JSON-LD - المصدر الأغنى بالبيانات (العنوان، الوصف، البراند، المواصفات، التقييم) | |
| document.querySelectorAll('script[type="application/ld+json"]').forEach(script => { | |
| try { | |
| const data = JSON.parse(script.textContent); | |
| if (data['@type'] === 'Product') { | |
| result.title = data.name || ''; | |
| result.description = data.description || ''; | |
| result.brand = data.brand?.name || ''; | |
| if (data.additionalProperty) { | |
| data.additionalProperty.forEach(prop => { | |
| if (prop.name && prop.value) { | |
| result.specs[prop.name] = prop.value; | |
| } | |
| }); | |
| } | |
| if (data.aggregateRating) { | |
| result.rating = parseFloat(data.aggregateRating.ratingValue) || 0; | |
| result.rating_count = parseInt(data.aggregateRating.reviewCount || data.aggregateRating.ratingCount) || 0; | |
| } | |
| } | |
| } catch(e) {} | |
| }); | |
| // 2. Fallback: H1 للعنوان | |
| if (!result.title) { | |
| const h1 = document.querySelector('h1'); | |
| if (h1) result.title = h1.textContent.trim(); | |
| } | |
| // 3. الصور: نعتمد على Python-side probing بدلاً من DOM scraping | |
| // لأن الصور lazy-loaded وغير موجودة كلها في DOM | |
| // سنترك images فارغة وسيتم ملؤها من Python عبر PID pattern | |
| result.images = []; | |
| // 4. استخراج الخيارات (Variants) | |
| const variantContainers = document.querySelectorAll('.same-sku-container, [class*="variant"], [class*="sku-container"]'); | |
| variantContainers.forEach(container => { | |
| const label = (container.querySelector('[class*="title"], [class*="label"]')?.textContent || '').trim(); | |
| const options = container.querySelectorAll('.tag, [class*="option"], [class*="item"]'); | |
| options.forEach(opt => { | |
| const name = (opt.querySelector('.text, [class*="value"]')?.textContent || opt.textContent || '').trim(); | |
| if (name) { | |
| result.variants.push({ | |
| type: label || 'Option', | |
| name: name, | |
| selected: opt.classList.contains('selected') || opt.getAttribute('aria-selected') === 'true' | |
| }); | |
| } | |
| }); | |
| }); | |
| if (result.variants.length === 0) { | |
| const storageSpec = result.specs['Memory (internal)'] || result.specs['Storage']; | |
| if (storageSpec) { | |
| result.variants.push({ type: 'Storage', name: storageSpec, selected: true }); | |
| } | |
| } | |
| return result; | |
| }''' | |
| # === JavaScript لاستخراج العنوان والوصف والمواصفات العربية === | |
| DETAIL_AR_JS = r'''() => { | |
| const result = { title_ar: '', description_ar: '', specs_ar: {}, variants_ar: [] }; | |
| // JSON-LD بالعربي | |
| document.querySelectorAll('script[type="application/ld+json"]').forEach(script => { | |
| try { | |
| const data = JSON.parse(script.textContent); | |
| if (data['@type'] === 'Product') { | |
| result.title_ar = data.name || ''; | |
| result.description_ar = data.description || ''; | |
| if (data.additionalProperty) { | |
| data.additionalProperty.forEach(prop => { | |
| if (prop.name && prop.value) { | |
| result.specs_ar[prop.name] = prop.value; | |
| } | |
| }); | |
| } | |
| } | |
| } catch(e) {} | |
| }); | |
| // استخراج الخيارات بالعربي | |
| const variantContainers = document.querySelectorAll('.same-sku-container, [class*="variant"]'); | |
| variantContainers.forEach(container => { | |
| const label = (container.querySelector('[class*="title"], [class*="label"]')?.textContent || '').trim(); | |
| const options = container.querySelectorAll('.tag, [class*="option"]'); | |
| options.forEach(opt => { | |
| const name = (opt.querySelector('.text, [class*="value"]')?.textContent || opt.textContent || '').trim(); | |
| if (name) { | |
| result.variants_ar.push({ | |
| type: label || 'خيار', | |
| name: name, | |
| selected: opt.classList.contains('selected') | |
| }); | |
| } | |
| }); | |
| }); | |
| // Fallback: H1 | |
| if (!result.title_ar) { | |
| const h1 = document.querySelector('h1'); | |
| if (h1) result.title_ar = h1.textContent.trim(); | |
| } | |
| return result; | |
| }''' | |
| class ExtraScraper: | |
| async def _go_safe(self, page, url, retries=3): | |
| """تحميل صفحة مع إعادة المحاولة""" | |
| for attempt in range(retries): | |
| try: | |
| await page.goto(url, wait_until="domcontentloaded", timeout=45000) | |
| return True | |
| except Exception as e: | |
| logger.warning(f"⚠️ Attempt {attempt+1} for {url[:80]}...: {e}") | |
| await asyncio.sleep(2) | |
| return False | |
| async def scrape_listing(self, page, category_key, url_template, target_count, current_total=0): | |
| """الخطوة 1: جلب PIDs والأسعار من صفحات القوائم بشكل ذكي""" | |
| listing = [] | |
| p_num = 0 | |
| while len(listing) < target_count: | |
| target = f"https://www.extra.com{url_template.format(page=p_num)}" | |
| logger.info(f"📃 [{category_key}] Listing page {p_num + 1} (Found: {len(listing)}/{target_count})") | |
| if not await self._go_safe(page, target): | |
| break | |
| await page.wait_for_timeout(3000) | |
| for y in [800, 2000, 4000]: | |
| await page.evaluate(f"window.scrollTo(0, {y})") | |
| await page.wait_for_timeout(200) | |
| items = await page.evaluate(LISTING_JS) | |
| if not items: | |
| logger.warning(f"⚠️ No products on page {p_num+1}, stopping category") | |
| break | |
| # إضافة المنتجات الجديدة التي لم نتجاوز بها الحد المطلوب | |
| for it in items: | |
| if len(listing) < target_count: | |
| it['category'] = category_key | |
| listing.append(it) | |
| else: | |
| break | |
| logger.info(f" ✅ {len(items)} products on this page | Total for category: {len(listing)}") | |
| p_num += 1 | |
| await asyncio.sleep(random.uniform(0.5, 1.5)) | |
| # إذا لم نحصل على أي منتجات جديدة أو عدد قليل جداً قد يعني نهاية الأقسام | |
| if len(items) < 5: | |
| break | |
| return listing | |
| # أنماط URL المتعددة التي يستخدمها media.extra.com | |
| # استخدام fmt=png لضمان التوافق مع Meta Catalog و Snapchat Catalog | |
| IMAGE_URL_PATTERNS = [ | |
| "https://media.extra.com/i/aurora/{pid}_100_{nn}?fmt=png&w=800", # النمط الأساسي (الأكثر شيوعاً) | |
| "https://media.extra.com/i/aurora/{pid}_{nn}?fmt=png&w=800", # نمط بديل بدون _100_ | |
| "https://media.extra.com/i/aurora/{pid}-{nn}?fmt=png&w=800", # نمط بديل بشرطة | |
| ] | |
| # حجم placeholder المعروف من extra.com (يُرجع 200 OK مع صورة بديلة "الصورة غير متوفرة") | |
| # حجم الـ placeholder المكتشف بدقة هو 12.6 kB (حوالي 12600 إلى 12750 بايت) | |
| PLACEHOLDER_SIZE_WINDOW = (12500, 12850) | |
| def _check_image_real(self, url, headers, session): | |
| """فحص صورة واحدة: هل هي حقيقية أم placeholder؟ | |
| يستخدم GET+stream بدل HEAD لضمان الحصول على Content-Length. | |
| يقرأ أول chunk لتحديد الحجم الحقيقي إذا لم يوجد Content-Length. | |
| """ | |
| try: | |
| response = session.get(url, headers=headers, timeout=8, allow_redirects=True, stream=True) | |
| if response.status_code != 200: | |
| response.close() | |
| return None, 0 | |
| # محاولة الحصول على الحجم من الهيدر | |
| size = int(response.headers.get('Content-Length', 0)) | |
| if size == 0: | |
| # Content-Length غائب → نقرأ أول 32KB لتحديد الحجم | |
| chunk = response.content[:32768] if not response.headers.get('Transfer-Encoding') else b'' | |
| if not chunk: | |
| # لو Transfer-Encoding: chunked، نقرأ الكل | |
| chunk = b'' | |
| for part in response.iter_content(chunk_size=8192): | |
| chunk += part | |
| if len(chunk) > 32768: | |
| break | |
| size = len(chunk) | |
| response.close() | |
| else: | |
| response.close() | |
| return url, size | |
| except Exception: | |
| return None, 0 | |
| def _discover_images_by_pid(self, pid): | |
| """اكتشاف صور المنتج عبر أنماط URL متعددة مع حماية من Rate Limiting | |
| التحسينات: | |
| 1. GET+stream بدل HEAD → يضمن Content-Length أو يقرأ الحقيقي | |
| 2. أنماط URL متعددة كـ fallback | |
| 3. delay بين الطلبات لتجنب Rate Limiting على Kaggle | |
| 4. حد فشل أعلى (5 بدل 3) | |
| 5. كشف placeholder عبر الحجم الثابت المعروف | |
| """ | |
| headers = { | |
| "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", | |
| "Accept": "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", | |
| "Referer": "https://www.extra.com/" | |
| } | |
| max_probe = 25 | |
| max_images = 20 | |
| max_consecutive_fail = 5 # رفع الحد من 3 إلى 5 | |
| # إنشاء session لإعادة استخدام الاتصال (أسرع + أقل rate limiting) | |
| session = requests.Session() | |
| session.headers.update(headers) | |
| # تجربة كل نمط URL حتى نجد صوراً | |
| for pattern in self.IMAGE_URL_PATTERNS: | |
| valid_images = [] | |
| consecutive_failures = 0 | |
| last_sizes = [] | |
| placeholder_size = None # حجم الـ placeholder لهذا المنتج (يُكتشف ديناميكياً) | |
| for i in range(1, max_probe + 1): | |
| nn = f"{i:02d}" | |
| img_url = pattern.format(pid=pid, nn=nn) | |
| url, size = self._check_image_real(img_url, headers, session) | |
| if url and size > 0: | |
| # كشف placeholder: | |
| # 1. القيمة المكتشفة من قبل المستخدم (12.6 kB) | |
| is_placeholder = (self.PLACEHOLDER_SIZE_WINDOW[0] <= size <= self.PLACEHOLDER_SIZE_WINDOW[1]) | |
| if is_placeholder: | |
| # هذا placeholder مؤكد -> هذا يعني انتهاء صور المنتج الحقيقية | |
| # نتوقف فوراً عن محاولة جلب الصور التالية لهذا المنتج | |
| logger.info(f" 🛑 Placeholder detected ({size} bytes) for {pid}. Stopping probe.") | |
| break | |
| if size < 20000: # صور المنتج الحقيقية غالباً أكبر من 20KB، لكننا نقبل الأصغر إذا لم تكن مكررة | |
| if placeholder_size is not None and abs(size - placeholder_size) < 100: | |
| consecutive_failures += 1 | |
| continue | |
| last_sizes.append(size) | |
| if len(last_sizes) >= 3: | |
| recent = last_sizes[-3:] | |
| if recent[0] == recent[1] == recent[2]: | |
| placeholder_size = recent[0] | |
| valid_images = valid_images[:-2] # إزالة الصور التي اتضح أنها placeholder | |
| break | |
| valid_images.append(img_url) | |
| consecutive_failures = 0 | |
| else: | |
| consecutive_failures += 1 | |
| if consecutive_failures >= max_consecutive_fail: | |
| break | |
| if len(valid_images) >= max_images: | |
| break | |
| # تأخير بسيط بين الطلبات لتجنب Rate Limiting | |
| time.sleep(random.uniform(0.05, 0.15)) | |
| # إذا وجدنا صوراً بهذا النمط → نرجعها | |
| if valid_images: | |
| session.close() | |
| return valid_images | |
| # Fallback أخير: محاولة الرابط المباشر بدون ترقيم (png) | |
| fallback_urls = [ | |
| f"https://media.extra.com/i/aurora/{pid}_100_01?fmt=png&w=800", | |
| f"https://media.extra.com/i/aurora/{pid}?fmt=png&w=800" | |
| ] | |
| for f_url in fallback_urls: | |
| url, size = self._check_image_real(f_url, headers, session) | |
| if url and size > 0: | |
| is_placeholder = (self.PLACEHOLDER_SIZE_WINDOW[0] <= size <= self.PLACEHOLDER_SIZE_WINDOW[1]) | |
| if not is_placeholder: | |
| session.close() | |
| return [f_url] | |
| session.close() | |
| return [] | |
| async def scrape_detail_en(self, page, product_url): | |
| """الخطوة 2: جلب تفاصيل المنتج بالإنجليزية (صور + مواصفات + عنوان)""" | |
| if not await self._go_safe(page, product_url): | |
| return None | |
| await page.wait_for_timeout(2000) | |
| # تمرير لتحميل المحتوى | |
| for y in [500, 1000]: | |
| await page.evaluate(f"window.scrollTo(0, {y})") | |
| await page.wait_for_timeout(200) | |
| # استخراج البيانات النصية من الصفحة (العنوان، المواصفات، التقييم) | |
| detail = await page.evaluate(DETAIL_EN_JS) | |
| if not detail: | |
| return None | |
| # --- اكتشاف الصور الحقيقية عبر PID Pattern (Python-side) --- | |
| pid_match = re.search(r'/p/([A-Z0-9]+)', product_url) | |
| if pid_match: | |
| pid = pid_match.group(1) | |
| logger.info(f"🖼️ Discovering images for PID={pid} via URL pattern probing...") | |
| discovered_images = self._discover_images_by_pid(pid) | |
| detail['images'] = discovered_images | |
| logger.info(f" ✅ Found {len(discovered_images)} real product images for PID={pid}") | |
| else: | |
| logger.warning(f" ⚠️ Could not extract PID from URL: {product_url}") | |
| detail['images'] = [] | |
| return detail | |
| async def scrape_detail_ar(self, page, product_url): | |
| """الخطوة 3: جلب العنوان والوصف والمواصفات بالعربية""" | |
| ar_url = product_url.replace('/en-sa/', '/ar-sa/') | |
| if not await self._go_safe(page, ar_url): | |
| return None | |
| await page.wait_for_timeout(1000) | |
| detail = await page.evaluate(DETAIL_AR_JS) | |
| return detail | |
| async def scrape(self, category_key, url_template, max_products=None, global_seen_pids=None): | |
| """السكرابر الكامل: قائمة → تفاصيل EN → تفاصيل AR""" | |
| results = [] | |
| if global_seen_pids is None: | |
| global_seen_pids = set() | |
| async with async_playwright() as pw: | |
| browser = await pw.chromium.launch(headless=True) | |
| context = await browser.new_context( | |
| user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", | |
| locale="en-SA", | |
| extra_http_headers={"Accept-Language": "en-US,en;q=0.9,ar;q=0.8"} | |
| ) | |
| page = await context.new_page() | |
| # === خطوة 1: جلب القائمة === | |
| listing = await self.scrape_listing(page, category_key, url_template, max_products, current_total=len(results)) | |
| logger.info(f"📋 [{category_key}] Total listings found: {len(listing)}") | |
| # === خطوة 2 + 3: زيارة كل منتج === | |
| for i, item in enumerate(listing): | |
| pid = item['pid'] | |
| url = item['product_url'] | |
| # --- الفلترة المسبقة الفورية (Early Deduplication) --- | |
| if pid in global_seen_pids: | |
| logger.info(f" ⏩ Skipping PID={pid} (Already processed previously)") | |
| continue | |
| if max_products and len(results) >= max_products: | |
| logger.info(f" 🎯 Reached quota for category: {category_key}") | |
| break | |
| logger.info(f" 🔎 [{category_key}] Product {len(results)+1}/{max_products or '?'}: PID={pid}") | |
| # --- تفاصيل إنجليزية --- | |
| en_detail = await self.scrape_detail_en(page, url) | |
| if not en_detail or not en_detail.get('title'): | |
| logger.warning(f" ⚠️ Skipping PID={pid} (no EN detail)") | |
| continue | |
| # --- تفاصيل عربية --- | |
| ar_detail = await self.scrape_detail_ar(page, url) | |
| # --- دمج البيانات --- | |
| product = { | |
| 'pid': pid, | |
| 'category': item['category'], | |
| 'product_url': url, | |
| 'price': item['price'], | |
| 'old_price': item.get('old_price'), | |
| 'brand': en_detail.get('brand') or item.get('brand', ''), | |
| 'title_en': en_detail['title'], | |
| 'title_ar': ar_detail.get('title_ar', '') if ar_detail else '', | |
| 'description_en': en_detail.get('description', ''), | |
| 'description_ar': ar_detail.get('description_ar', '') if ar_detail else '', | |
| 'images': en_detail.get('images', []), | |
| 'specs_en': en_detail.get('specs', {}), | |
| 'specs_ar': ar_detail.get('specs_ar', {}) if ar_detail else {}, | |
| 'variants_en': en_detail.get('variants', []), | |
| 'variants_ar': ar_detail.get('variants_ar', []) if ar_detail else [], | |
| 'rating_count': en_detail.get('rating_count', 0), | |
| 'raw_categories': en_detail.get('categories', []), | |
| } | |
| results.append(product) | |
| global_seen_pids.add(pid) # إضافة المنتج لقائمة المعالجين | |
| logger.info(f" ✅ {product['title_en'][:50]}... | {len(product['images'])} imgs | {len(product['specs_en'])} specs") | |
| await asyncio.sleep(random.uniform(0.1, 0.4)) | |
| await browser.close() | |
| return results | |
| # ----------------------------------------------- | |
| # 5. دوال التحويل إلى VortexCommerce | |
| # ----------------------------------------------- | |
| def extract_brand(title, raw_brand=''): | |
| if raw_brand: | |
| for b in KNOWN_BRANDS: | |
| if b.upper() == raw_brand.upper() or b.lower() == raw_brand.lower(): | |
| return b | |
| for brand in KNOWN_BRANDS: | |
| if brand.lower() in title.lower(): | |
| return brand | |
| if 'rog' in title.lower(): | |
| return 'ASUS ROG' | |
| return raw_brand.strip().title() if raw_brand else "" | |
| def extract_details(title): | |
| details = {"storage": "", "color": "", "color_ar": ""} | |
| m = re.search(r'(\d+)\s*(GB|TB|جيجا|تيرا)', title, re.IGNORECASE) | |
| if m: | |
| details["storage"] = m.group(0) | |
| for en, ar in COLOR_MAP.items(): | |
| if en.lower() in title.lower() or ar in title: | |
| details["color"] = en | |
| details["color_ar"] = ar | |
| break | |
| return details | |
| def make_options(raw_product, images): | |
| """إنشاء خيارات/متغيرات احترافية بناءً على البيانات المسحوبة""" | |
| opts = [] | |
| base_price = raw_product['price'] | |
| variants_en = raw_product.get('variants_en', []) | |
| variants_ar = raw_product.get('variants_ar', []) | |
| img1 = images[0] if images else '' | |
| img2 = images[1] if len(images) > 1 else img1 | |
| # 1. إذا وجدنا خيارات حقيقية مسحوبة من الصفحة | |
| if variants_en: | |
| # نقوم بتجميع الخيارات حسب النوع (مثلاً تجميع السعات معاً) | |
| # للتسجيل الحالي، سنأخذ أول مجموعة خيارات نجدها (غالباً الحجم) | |
| unique_variants = {} | |
| for i, ven in enumerate(variants_en): | |
| name_en = ven['name'] | |
| name_ar = variants_ar[i]['name'] if i < len(variants_ar) else name_en | |
| if name_en not in unique_variants: | |
| unique_variants[name_en] = { | |
| "name_en": name_en, | |
| "name_ar": name_ar, | |
| "selected": ven.get('selected', False) | |
| } | |
| def to_gb(t): | |
| m = re.search(r'(\d+)\s*(GB|TB|جيجا|تيرا)', t, re.IGNORECASE) | |
| if not m: return 0 | |
| v = int(m.group(1)) | |
| u = m.group(2).upper() | |
| return v * 1024 if ('TB' in u or 'تيرا' in u) else v | |
| for i, (name, vdata) in enumerate(unique_variants.items()): | |
| # Realistic Price Adjustment for Storage (SAR Deltas) | |
| # Instead of simple ratios, we use standard industry jumps for premium devices | |
| brand = (raw_product.get('brand') or "").upper() | |
| is_premium = brand in ["APPLE", "SAMSUNG", "GOOGLE"] or base_price > 3000 | |
| price_delta = 0 | |
| if "GB" in name or "TB" in name: | |
| val = to_gb(name) | |
| current_val = to_gb(raw_product['title_en']) | |
| if is_premium: | |
| # Premium tier offsets (Approximate market standard) | |
| # 128->256 (+500), 256->512 (+1000), 512->1TB (+1000), 1TB->2TB (+2000) | |
| tiers = {128: 0, 256: 500, 512: 1500, 1024: 2500, 2048: 4500} | |
| target_offset = 0 | |
| current_offset = 0 | |
| # Find closest tiers | |
| for t_size, t_off in sorted(tiers.items()): | |
| if val >= t_size: target_offset = t_off | |
| if current_val >= t_size: current_offset = t_off | |
| price_delta = target_offset - current_offset | |
| else: | |
| # Generic devices use simpler ratio | |
| ratio = val / current_val if current_val > 0 else 1 | |
| if ratio > 1: price_delta = base_price * (min(ratio - 1, 1.0) * 0.2) | |
| elif ratio < 1: price_delta = base_price * (max(ratio - 1, -0.4)) | |
| opts.append({ | |
| "sku": f"EXT-{uuid.uuid4().hex[:8]}", | |
| "name_en": vdata["name_en"], | |
| "name_ar": vdata["name_ar"], | |
| "price": round(base_price + price_delta, 2) if not vdata["selected"] else base_price, | |
| "stock": random.randint(5, 50), | |
| "image": img1 if vdata["selected"] else img2, | |
| }) | |
| # 2. إذا لم نجد خيارات، نضع خياراً افتراضياً واحداً | |
| if not opts: | |
| opts.append({ | |
| "sku": f"EXT-{uuid.uuid4().hex[:8]}", | |
| "name_en": "Standard Edition", | |
| "name_ar": "الإصدار القياسي", | |
| "price": base_price, | |
| "stock": random.randint(10, 100), | |
| "image": img1, | |
| }) | |
| return opts | |
| def transform_to_vortex(raw_products): | |
| """التحويل النهائي إلى هيكلة VortexCommerce (مطابقة لـ real_products_200.json)""" | |
| out = [] | |
| seen = set() | |
| idx = 0 | |
| for raw in raw_products: | |
| title_en = raw['title_en'].strip() | |
| key = title_en.lower() | |
| if key in seen: | |
| continue | |
| seen.add(key) | |
| idx += 1 | |
| cat_key = raw['category'] | |
| # تحديد الفئة الأب والفرعية بناءً على الهيكلية الجديدة | |
| parent_ar = EXTRA_CAT_TO_PARENT.get(cat_key, "الأجهزة الإلكترونية") | |
| parent_info = CATEGORIES_HIERARCHY[parent_ar] | |
| sub_info = parent_info["subs"].get(cat_key, {"en": cat_key.title(), "ar": cat_key}) | |
| display_cat_ar = sub_info["ar"] | |
| display_cat_en = sub_info["en"] | |
| parent_en = parent_info["en"] | |
| icon = parent_info["icon"] | |
| brand = extract_brand(title_en, raw.get('brand', '')) | |
| details = extract_details(title_en) | |
| # الأسماء | |
| name_ar = raw.get('title_ar', '') | |
| if not name_ar: | |
| ar_brand = BRAND_AR.get(brand, brand) | |
| if brand and title_en.lower().startswith(brand.lower()): | |
| name_ar = f"{ar_brand} {title_en[len(brand):].strip()}" | |
| else: | |
| name_ar = f"{ar_brand} - {title_en}" | |
| # الأوصاف | |
| desc_en = raw.get('description_en', '') | |
| desc_ar = raw.get('description_ar', '') | |
| if not desc_en: | |
| desc_en = f"The {title_en} is a premium product from {brand or 'a trusted brand'}. Available at eXtra stores in Saudi Arabia with official warranty." | |
| if not desc_ar: | |
| desc_ar = f"نقدم لكم {name_ar}، منتج عالي الجودة من {brand or 'علامة تجارية موثوقة'}. متوفر في معارض اكسترا في المملكة العربية السعودية مع ضمان رسمي." | |
| price = raw['price'] | |
| old_price = raw.get('old_price') | |
| compare = old_price if old_price and old_price > price else round(price * random.uniform(1.05, 1.18), 2) | |
| images = raw.get('images', []) | |
| rating = raw.get('rating') or round(random.uniform(3.5, 4.9), 2) | |
| rating_count = raw.get('rating_count') or random.randint(10, 500) | |
| # الخيارات الاحترافية (واقعية) | |
| options = make_options(raw, images) | |
| # المواصفات: دمج EN + AR مع حقول إضافية | |
| specs_en = raw.get('specs_en', {}) | |
| specs_ar = raw.get('specs_ar', {}) | |
| specs = { | |
| "origin": "Saudi Arabia", | |
| "condition": "New Genuine", | |
| "rating": rating, | |
| "warranty": "1 Year Official KSA Warranty", | |
| "sku_base": f"EXT-SKU-{raw.get('pid', idx)}", | |
| "source": "extra.com", | |
| "brand_name": brand, # إضافة البراند للمواصفات لسهولة الفلترة | |
| } | |
| # إضافة المواصفات التفصيلية من الموقع | |
| if specs_en: | |
| specs["details_en"] = specs_en | |
| if specs_ar: | |
| specs["details_ar"] = specs_ar | |
| if details['storage']: | |
| specs["storage"] = details['storage'] | |
| if details['color']: | |
| specs["color_en"] = details['color'] | |
| specs["color_ar"] = details['color_ar'] | |
| out.append({ | |
| "category": display_cat_ar, # الفئة الفرعية بالعربي (مثل: جوالات) | |
| "category_en": display_cat_en, # الفئة الفرعية بالإنجليزي | |
| "main_category": parent_ar, # الفئة الأساسية بالعربي (مثل: الأجهزة الإلكترونية) | |
| "main_category_en": parent_en, # الفئة الأساسية بالإنجليزي | |
| "category_icon": icon, # الأيقونة | |
| "brand": brand or "Generic", | |
| "name_en": title_en, | |
| "name_ar": name_ar, | |
| "description_en": desc_en, | |
| "description_ar": desc_ar, | |
| "price": round(price, 2), | |
| "compare_price": round(compare, 2), | |
| "images": images, | |
| "options": options, | |
| "specs": specs, | |
| "seller_info": {"name": "Vortex Authorized Seller", "rating": 4.9, "location": "Riyadh, KSA"}, | |
| "quality_score": 100 if len(images) > 0 else 80, | |
| "legacy_ref": f"EXT-API-{raw.get('pid', idx)}", | |
| }) | |
| return out | |
| # ----------------------------------------------- | |
| # 6. التشغيل الرئيسي | |
| # ----------------------------------------------- | |
| async def main(): | |
| db = Database(DATABASE_URL or DATABASE_NAME) | |
| scraper = ExtraScraper() | |
| all_raw = [] | |
| seen_pids = set() | |
| print("=" * 60) | |
| print("🛒 Extra.com → VortexCommerce Full Scraper (V2: Per-Category Quota)") | |
| print(f"📋 Independent Target: {TARGET_PRODUCTS_PER_CATEGORY} products per category") | |
| print(f"🌐 Languages: English + Arabic") | |
| print(f"🖼️ Mode: Multiple images + Full specs per product") | |
| print("=" * 60) | |
| for cat_key, url_tpl in CATEGORIES_TO_SCRAPE.items(): | |
| print(f"\n{'─'*50}") | |
| print(f"📂 Processing Category: {cat_key} (Target: {TARGET_PRODUCTS_PER_CATEGORY})") | |
| print(f"{'─'*50}") | |
| # Step 1: Get enough listings and scrape details with early deduplication | |
| # We pass global_seen_pids to skip already visited products across categories | |
| prods = await scraper.scrape(cat_key, url_tpl, max_products=TARGET_PRODUCTS_PER_CATEGORY, global_seen_pids=seen_pids) | |
| if prods: | |
| saved = db.save_products(prods) | |
| all_raw.extend(prods) | |
| img_counts = [len(p.get('images', [])) for p in prods] | |
| avg_imgs = sum(img_counts) / len(img_counts) if img_counts else 0 | |
| ar_count = sum(1 for p in prods if p.get('title_ar')) | |
| print(f" ✅ {len(prods)} products saved and processed for this category") | |
| print(f" 🖼️ Avg images: {avg_imgs:.1f} | 🌐 Arabic: {ar_count}/{len(prods)}") | |
| else: | |
| print(f" ⚠️ No new products found in this category") | |
| print(f"\n{'═'*60}") | |
| print(f"📦 FINAL SCRAPE COMPLETE: Total {len(all_raw)} products across all categories") | |
| print(f"{'═'*60}") | |
| # === التحويل === | |
| print("\n🔄 Converting to VortexCommerce format...") | |
| vortex_products = transform_to_vortex(all_raw) | |
| output = { | |
| "metadata": { | |
| "total_count": len(vortex_products), | |
| "generated_at": datetime.now(timezone.utc).isoformat(), | |
| "version": "Extra-Scraper-V2-Full", | |
| "quality_tier": "Extra-Validated-Full", | |
| "source": "https://www.extra.com/en-sa/", | |
| "features": ["multi-image", "bilingual", "full-specs"] | |
| }, | |
| "products": vortex_products | |
| } | |
| # === حفظ الملفات === | |
| vortex_file = "extra_products_vortex.json" | |
| raw_file = "extra_products_raw.json" | |
| with open(vortex_file, 'w', encoding='utf-8') as f: | |
| json.dump(output, f, ensure_ascii=False, indent=2) | |
| with open(raw_file, 'w', encoding='utf-8') as f: | |
| json.dump(all_raw, f, ensure_ascii=False, indent=2) | |
| with zipfile.ZipFile('extra_data.zip', 'w') as z: | |
| z.write(DATABASE_NAME) | |
| z.write(vortex_file) | |
| z.write(raw_file) | |
| # === الإحصائيات النهائية === | |
| print(f"\n{'═'*60}") | |
| print(f"📊 FINAL REPORT") | |
| print(f"{'═'*60}") | |
| cats = {} | |
| brands = {} | |
| total_imgs = 0 | |
| total_specs = 0 | |
| with_ar = 0 | |
| for p in vortex_products: | |
| cats[p['category']] = cats.get(p['category'], 0) + 1 | |
| brands[p['brand']] = brands.get(p['brand'], 0) + 1 | |
| total_imgs += len(p.get('images', [])) | |
| total_specs += len(p.get('specs', {}).get('details_en', {})) | |
| if p.get('name_ar'): | |
| with_ar += 1 | |
| n = len(vortex_products) | |
| print(f"\n📁 Categories:") | |
| for cat, count in sorted(cats.items()): | |
| print(f" {cat}: {count}") | |
| print(f"\n🏷️ Top Brands:") | |
| for brand, count in sorted(brands.items(), key=lambda x: -x[1])[:15]: | |
| print(f" {brand}: {count}") | |
| print(f"\n📈 Quality Metrics:") | |
| print(f" Total products: {n}") | |
| print(f" 🖼️ Total images: {total_imgs} (avg {total_imgs/n:.1f}/product)") | |
| print(f" 📋 Products with specs: {sum(1 for p in vortex_products if p['specs'].get('details_en'))}/{n}") | |
| print(f" 🌐 Arabic names: {with_ar}/{n}") | |
| print(f" 💰 Price range: {min(p['price'] for p in vortex_products):.0f} - {max(p['price'] for p in vortex_products):.0f} SAR") | |
| print(f"\n📄 Files generated:") | |
| print(f" ✅ {vortex_file} ({n} products)") | |
| print(f" ✅ {raw_file}") | |
| print(f" ✅ {DATABASE_NAME}") | |
| print(f" 📦 extra_data.zip (download from sidebar)") | |
| print(f"\n✅ Copy {vortex_file} → backend/kaggle/extra_products_vortex.json") | |
| db.close() | |
| await main() | |