adding more sites in bot

scraper.py

@@ -2,6 +2,11 @@ import os
 import asyncio
 from playwright.async_api import async_playwright
 from supabase import create_client, Client
+import logging
+
+# Setup logging
+logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
+logger = logging.getLogger(__name__)
 
 # --- 1. SETUP YOUR SUPABASE VAULT HERE ---
 SUPABASE_URL = os.environ.get("SUPABASE_URL")
@@ -10,8 +15,119 @@ SUPABASE_KEY = os.environ.get("SUPABASE_KEY")
 # Connect to the database
 supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
 
+# --- MULTI-SITE CONFIGURATION ---
+GOVERNMENT_SITES = [
+    {
+        "name": "MyScheme.gov.in",
+        "url": "https://www.myscheme.gov.in/search",
+        "source": "MyScheme Portal",
+        "selector": "h2",
+        "extract_type": "myscheme"
+    },
+    {
+        "name": "Ministry of Education - Schemes",
+        "url": "https://www.education.gov.in/",
+        "source": "Ministry of Education",
+        "selector": "h3, h2",
+        "extract_type": "generic"
+    },
+    {
+        "name": "AICTE Schemes",
+        "url": "https://portal.aicte-india.org/",
+        "source": "AICTE",
+        "selector": "h2, .scheme-title",
+        "extract_type": "generic"
+    },
+    {
+        "name": "Ministry of Rural Development",
+        "url": "https://pmay-urb.gov.in/",
+        "source": "Ministry of Rural Development",
+        "selector": "h2, h3",
+        "extract_type": "generic"
+    }
+]
+
+async def scrape_myscheme(page, site_config):
+    """Specialized scraper for MyScheme portal"""
+    try:
+        logger.info(f"🎯 Scraping {site_config['name']}...")
+        await page.goto(site_config["url"], timeout=15000)
+        await page.wait_for_timeout(3000)
+        
+        titles = await page.locator("h2").all_inner_texts()
+        schemes = [t.strip() for t in titles if len(t.strip()) > 5 and "Filter By" not in t]
+        
+        results = []
+        if len(schemes) >= 2:
+            for i in range(0, len(schemes) - 1, 2):
+                try:
+                    data_to_save = {
+                        "title": schemes[i],
+                        "department": schemes[i + 1] if i + 1 < len(schemes) else "Unknown",
+                        "document_type": "Government Scheme",
+                        "file_url": site_config["url"],
+                        "authentic_source_url": site_config["source"],
+                        "scrape_source": site_config["name"]
+                    }
+                    supabase.table("official_documents").insert(data_to_save).execute()
+                    logger.info(f"✅ Uploaded: {schemes[i]}")
+                    results.append(True)
+                except Exception as e:
+                    logger.error(f"❌ Failed to upload {schemes[i]}: {str(e)}")
+                    results.append(False)
+            return results
+        else:
+            logger.warning(f"⚠️ Not enough data found on {site_config['name']}")
+            return []
+    except Exception as e:
+        logger.error(f"❌ Error scraping {site_config['name']}: {str(e)}")
+        return []
+
+async def scrape_generic(page, site_config):
+    """Generic scraper for government websites"""
+    try:
+        logger.info(f"🎯 Scraping {site_config['name']}...")
+        await page.goto(site_config["url"], timeout=15000)
+        await page.wait_for_timeout(3000)
+        
+        selectors = site_config["selector"].split(", ")
+        all_text = []
+        
+        for selector in selectors:
+            try:
+                elements = await page.locator(selector).all_inner_texts()
+                all_text.extend([t.strip() for t in elements if len(t.strip()) > 5])
+            except:
+                pass
+        
+        results = []
+        if all_text:
+            for i, text in enumerate(all_text[:10]):  # Limit to first 10 items
+                try:
+                    data_to_save = {
+                        "title": text,
+                        "department": site_config["source"],
+                        "document_type": "Government Content",
+                        "file_url": site_config["url"],
+                        "authentic_source_url": site_config["source"],
+                        "scrape_source": site_config["name"]
+                    }
+                    supabase.table("official_documents").insert(data_to_save).execute()
+                    logger.info(f"✅ Uploaded from {site_config['name']}: {text[:50]}...")
+                    results.append(True)
+                except Exception as e:
+                    logger.warning(f"⚠️ Skipped duplicate or error: {str(e)[:50]}")
+                    results.append(False)
+            return results
+        else:
+            logger.warning(f"⚠️ No data found on {site_config['name']}")
+            return []
+    except Exception as e:
+        logger.error(f"❌ Error scraping {site_config['name']}: {str(e)}")
+        return []
+
 async def scrape_and_upload():
-    print("🥷 Booting up GovBridge Bot (Upload Mode)...")
+    print("🥷 Booting up GovBridge Multi-Site Bot (Upload Mode)...\n")
     
     async with async_playwright() as p:
         browser = await p.chromium.launch(
@@ -26,42 +142,28 @@ async def scrape_and_upload():
         # Apply the stealth mask to hide from the firewall
         await context.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
         page = await context.new_page()
+        page.set_default_timeout(15000)
         
-        print("🌐 Navigating to myscheme.gov.in...")
-        await page.goto("https://www.myscheme.gov.in/search")
-        await page.wait_for_timeout(5000) 
-        
-        print("🎯 Target acquired! Ripping the text...")
-        titles = await page.locator("h2").all_inner_texts()
-        schemes = [t.strip() for t in titles if len(t.strip()) > 5 and "Filter By" not in t]
+        total_uploaded = 0
         
-        if len(schemes) >= 2:
-            # We noticed the website groups the Title first, and the Ministry second!
-            first_scheme_title = schemes[0]
-            first_scheme_ministry = schemes[1]
-            
-            print(f"\n--- INITIATING DATABASE UPLOAD ---")
-            print(f"📤 Uploading: {first_scheme_title}...")
-            
-            # Package the data to match our official_documents table perfectly
-            data_to_save = {
-                "title": first_scheme_title,
-                "department": first_scheme_ministry,
-                "document_type": "Auto-Scraped Scheme",
-                "file_url": "https://www.myscheme.gov.in/search",
-                "authentic_source_url": "https://www.myscheme.gov.in"
-            }
-            
+        # Scrape all configured sites
+        for site_config in GOVERNMENT_SITES:
             try:
-                # Fire the data directly into your Supabase vault!
-                supabase.table("official_documents").insert(data_to_save).execute()
-                print("✅ SUCCESS! The data bypassed the terminal and is now live in your database!")
-            except Exception as e:
-                print(f"❌ Upload failed: {e}")
-                
-        else:
-            print("❌ Didn't find enough data on the page to upload.")
+                if site_config["extract_type"] == "myscheme":
+                    results = await scrape_myscheme(page, site_config)
+                else:
+                    results = await scrape_generic(page, site_config)
                 
+                total_uploaded += sum(1 for r in results if r)
+            except Exception as e:
+                logger.error(f"❌ Critical error with {site_config['name']}: {str(e)}")
+            
+            # Respectful delay between requests
+            await page.wait_for_timeout(2000)
+        
+        print(f"\n--- 📊 FINAL REPORT ---")
+        print(f"✅ Total items uploaded: {total_uploaded}")
+        print(f"🌐 Sites processed: {len(GOVERNMENT_SITES)}")
         print("\nShutting down bot.")
         await browser.close()