Blame for scraper.py

import os
import asyncio
import hashlib
import random
import httpx
from playwright.async_api import async_playwright
import logging

# Setup logging
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
logger = logging.getLogger(__name__)

# --- UNIFIED INGESTION CONFIGURATION (OPTION A) ---
HF_SPACE_URL = os.environ.get("HF_SPACE_URL", "https://harshrawat18-govbridge-api.hf.space")
ADMIN_SECRET = os.environ.get("ADMIN_SECRET", "") 

# --- STEALTH SCRIPT COLLECTION ---
STEALTH_SCRIPTS = [
    """
    Object.defineProperty(navigator, 'webdriver', {
        get: () => undefined,
        configurable: true
    });
    """,
    """
    window.chrome = {
        app: {
            isInstalled: false,
            InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' },
            RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' }
        },
        runtime: {
            OnInstalledReason: { CHROME_UPDATE: 'chrome_update', INSTALL: 'install', SHARED_MODULE_UPDATE: 'shared_module_update', UPDATE: 'update' },
            OnRestartRequiredReason: { APP_UPDATE: 'app_update', GC_PAUSED: 'gc_paused', OS_UPDATE: 'os_update' },
            PlatformArch: { ARM: 'arm', ARM64: 'arm64', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' },
            PlatformNaclArch: { ARM: 'arm', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' },
            PlatformOs: { ANDROID: 'android', CROS: 'cros', LINUX: 'linux', MAC: 'mac', OPENBSD: 'openbsd', WIN: 'win' },
            RequestUpdateCheckStatus: { NO_UPDATE: 'no_update', THROTTLED: 'throttled', UPDATE_AVAILABLE: 'update_available' }
        }
    };
    """,
    """
    const makePlugin = (name, filename, description, mimeTypes) => {
        const plugin = Object.create(Plugin.prototype);
        Object.defineProperties(plugin, {
            name: { value: name, writable: false },
            filename: { value: filename, writable: false },
            description: { value: description, writable: false },
            length: { value: mimeTypes.length }
        });
        mimeTypes.forEach((mt, i) => {
            const mimeType = Object.create(MimeType.prototype);
            Object.defineProperties(mimeType, {
                type: { value: mt.type },
                suffixes: { value: mt.suffixes },
                description: { value: mt.description },
                enabledPlugin: { value: plugin }
            });
            plugin[i] = mimeType;
        });
        return plugin;
    };

    const pluginData = [
        makePlugin('PDF Viewer', 'internal-pdf-viewer', 'Portable Document Format',
            [{ type: 'application/pdf', suffixes: 'pdf', description: '' },
             { type: 'text/pdf', suffixes: 'pdf', description: '' }]),
        makePlugin('Chrome PDF Viewer', 'internal-pdf-viewer', 'Portable Document Format',
            [{ type: 'application/pdf', suffixes: 'pdf', description: '' }])
    ];

    const pluginArray = Object.create(PluginArray.prototype);
    Object.defineProperty(pluginArray, 'length', { value: pluginData.length });
    pluginData.forEach((p, i) => { pluginArray[i] = p; });
    pluginArray.item = (i) => pluginData[i] ?? null;
    pluginArray.namedItem = (name) => pluginData.find(p => p.name === name) ?? null;
    pluginArray.refresh = () => {};

    Object.defineProperty(navigator, 'plugins', {
        get: () => pluginArray,
        configurable: true
    });
    Object.defineProperty(navigator, 'mimeTypes', {
        get: () => {
            const arr = Object.create(MimeTypeArray.prototype);
            const allMimes = pluginData.flatMap(p => Array.from({length: p.length}, (_, i) => p[i]));
            Object.defineProperty(arr, 'length', { value: allMimes.length });
            allMimes.forEach((m, i) => { arr[i] = m; });
            return arr;
        }
    });
    """,
    """
    const getParameter = WebGLRenderingContext.prototype.getParameter;
    WebGLRenderingContext.prototype.getParameter = function(parameter) {
        if (parameter === 37446) return 'Intel Inc.';
        if (parameter === 37445) return 'Intel Iris OpenGL Engine';
        return getParameter.call(this, parameter);
    };

    const getParameter2 = WebGL2RenderingContext.prototype.getParameter;
    WebGL2RenderingContext.prototype.getParameter = function(parameter) {
        if (parameter === 37446) return 'Intel Inc.';
        if (parameter === 37445) return 'Intel Iris OpenGL Engine';
        return getParameter2.call(this, parameter);
    };
    """,
    """
    const originalQuery = window.navigator.permissions.query;
    window.navigator.permissions.query = (parameters) => (
        parameters.name === 'notifications' ?
            Promise.resolve({ state: Notification.permission }) :
            originalQuery(parameters)
    );
    """,
    """
    Object.defineProperty(navigator, 'languages', {
        get: () => ['en-IN', 'en-GB', 'en'],
        configurable: true
    });
    """,
    """
    const iframeProto = HTMLIFrameElement.prototype;
    const originalGetter = Object.getOwnPropertyDescriptor(iframeProto, 'contentWindow');
    Object.defineProperty(iframeProto, 'contentWindow', {
        get: function() {
            const win = originalGetter.get.call(this);
            if (!win) return win;
            try {
                Object.defineProperty(win.navigator, 'webdriver', {
                    get: () => undefined,
                    configurable: true
                });
            } catch(_) {}
            return win;
        }
    });
    """
]

# --- MULTI-SITE CONFIGURATION ---
GOVERNMENT_SITES = [
    {
        "name": "MyScheme.gov.in",
        "url": "https://www.myscheme.gov.in/search",
        "source": "MyScheme Portal",
        "selector": "h2",
        "extract_type": "myscheme",
        "doc_type": "scheme"
    },
    {
        "name": "PM-KISAN Scheme",
        "url": "https://pmkisan.gov.in/",
        "source": "Ministry of Agriculture",
        "selector": "h2, h3, div.content",
        "extract_type": "generic",
        "doc_type": "scheme"
    },
]

async def push_to_hf_api(title: str, text: str, source_url: str, ministry: str, doc_type: str):
    """Pushes scraped data directly to the Hugging Face Ingestion Engine"""
    payload = {
        "title": title,
        "text": text,
        "ministry": ministry,
        "state": "National", 
        "source_url": source_url,
        "doc_type": doc_type
    }
    
    async with httpx.AsyncClient() as client:
        try:
            resp = await client.post(
                f"{HF_SPACE_URL}/api/admin/ingest",
                json=payload,
                params={"admin_key": ADMIN_SECRET},
                headers={"X-Admin-Secret": ADMIN_SECRET},
                timeout=30.0
            )
            if resp.status_code == 200:
                logger.info(f"✅ Ingested to Vector DB: {title[:40]}...")
                return True
            else:
                logger.error(f"❌ HF API Error: {resp.status_code} - {resp.text}")
                return False
        except Exception as e:
            logger.error(f"⚠️ Network Error pushing {title[:30]}: {e}")
            return False

async def scrape_myscheme(page, site_config):
    """Specialized scraper for MyScheme portal"""
    try:
        logger.info(f"🎯 Scraping {site_config['name']}...")
        await asyncio.sleep(random.uniform(1.5, 3.5)) 
        await page.goto(site_config["url"], timeout=60000, wait_until="domcontentloaded")
        
        await page.evaluate("window.scrollTo(0, document.body.scrollHeight / 3)")
        await asyncio.sleep(random.uniform(0.8, 1.8))
        
        titles = await page.locator("h2").all_inner_texts()
        schemes = [t.strip() for t in titles if len(t.strip()) > 5 and "Filter By" not in t]
        
        results = []
        if len(schemes) >= 2:
            for i in range(0, len(schemes) - 1, 2):
                try:
                    title = schemes[i]
                    department = schemes[i + 1] if i + 1 < len(schemes) else "Unknown"
                    
                    # Push directly to HF Engine
                    success = await push_to_hf_api(
                        title=title,
                        text=f"Details regarding {title} provided by {department}.",
                        source_url=site_config["url"],
                        ministry=department,
                        doc_type=site_config["doc_type"]
                    )
                    results.append(success)
                except Exception as e:
                    logger.error(f"❌ Failed to upload {schemes[i]}: {str(e)}")
                    results.append(False)
            return results
        return []
    except Exception as e:
        logger.error(f"❌ Error scraping {site_config['name']}: {str(e)}")
        return []

async def scrape_generic(page, site_config):
    """Generic scraper for government websites"""
    try:
        logger.info(f"🎯 Scraping {site_config['name']}...")
        await asyncio.sleep(random.uniform(1.5, 3.5))
        await page.goto(site_config["url"], timeout=60000, wait_until="domcontentloaded")
        
        await page.evaluate("window.scrollTo(0, document.body.scrollHeight / 3)")
        await asyncio.sleep(random.uniform(0.8, 1.8))
        
        selectors = site_config["selector"].split(", ")
        all_text = []
        for selector in selectors:
            try:
                elements = await page.locator(selector).all_inner_texts()
                all_text.extend([t.strip() for t in elements if len(t.strip()) > 5])
            except:
                pass
        
        results = []
        if all_text:
            for i, text in enumerate(all_text[:10]):
                try:
                    # Push directly to HF Engine
                    success = await push_to_hf_api(
                        title=text,
                        text=text,
                        source_url=site_config["url"],
                        ministry=site_config["source"],
                        doc_type=site_config["doc_type"]
                    )
                    results.append(success)
                except Exception as e:
                    logger.warning(f"⚠️ Error: {str(e)[:50]}")
                    results.append(False)
            return results
        return []
    except Exception as e:
        logger.error(f"❌ Error scraping {site_config['name']}: {str(e)}")
        return []

async def scrape_and_upload():
    print("🥷 Booting up GovBridge UNIFIED Scraper (Sending to HF API)...\n")
    
    async with async_playwright() as p:
        # Launch hardened browser
        browser = await p.chromium.launch(
            headless=True,
            args=[
                "--no-sandbox",
                "--disable-setuid-sandbox",
                "--disable-blink-features=AutomationControlled",
                "--disable-features=IsolateOrigins,site-per-process",
                "--disable-dev-shm-usage",
                "--no-first-run",
                "--no-default-browser-check",
                "--disable-infobars",
            ]
        )
        
        context = await browser.new_context(
            viewport={"width": 1366, "height": 768},
            user_agent=(
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                "AppleWebKit/537.36 (KHTML, like Gecko) "
                "Chrome/124.0.0.0 Safari/537.36"
            ),
            locale="en-IN",
            timezone_id="Asia/Kolkata",
            color_scheme="light",
            extra_http_headers={
                "Accept-Language": "en-IN,en-GB;q=0.9,en;q=0.8",
                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
                "sec-ch-ua": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
                "sec-ch-ua-mobile": "?0",
                "sec-ch-ua-platform": '"Windows"',
            }
        )
        
        # Inject all stealth patches
        for script in STEALTH_SCRIPTS:
            await context.add_init_script(script=script)

        page = await context.new_page()
        total_uploaded = 0
        
        for site_config in GOVERNMENT_SITES:
            try:
                if site_config["extract_type"] == "myscheme":
                    results = await scrape_myscheme(page, site_config)
                else:
                    results = await scrape_generic(page, site_config)
                
                total_uploaded += sum(1 for r in results if r)
            except Exception as e:
                logger.error(f"❌ Critical error with {site_config['name']}: {str(e)}")
            
            await asyncio.sleep(random.uniform(2.0, 4.0)) # Delay between sites
        
        print(f"\n--- 📊 FINAL REPORT ---")
        print(f"✅ Total items pushed to Vector Engine: {total_uploaded}")
        print("\nShutting down bot.")
        await browser.close()

if __name__ == "__main__":
    asyncio.run(scrape_and_upload())