"""Main entry point for the balanced scraper.""" import argparse import sys import os from src.core.config import ScraperConfig from src.core.balanced_scraper import BalancedScraper def main(): """Main function to run the balanced scraper.""" parser = argparse.ArgumentParser( description='Dynamic Product Catalog Scraper - BALANCED MODE', formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" BALANCED MODE combines lenient and strict approaches for optimal results. Strictness Levels: lenient - High recall, catches all products (some false positives) balanced - Good precision + recall (RECOMMENDED) strict - High precision, clean results (may miss some products) Examples: # Balanced mode (recommended) python main.py --url https://example.com/products # Lenient mode (catch everything) python main.py --url https://example.com/products --strictness lenient # Strict mode (very clean results) python main.py --url https://example.com/products --strictness strict # With export formats python main.py --url https://example.com/products --export all # Adjust crawl settings python main.py --url https://example.com/products --max-pages 100 --max-depth 4 Environment Variables (optional): BASE_URL - Default URL to scrape GEMINI_API_KEY - Gemini API key for AI features GOOGLE_CREDENTIALS_FILE - Google service account credentials GOOGLE_SPREADSHEET_ID - Target Google Sheets ID """ ) # Required: Target URL parser.add_argument( '--url', type=str, help='Target website URL to scrape (required unless set in .env)' ) # NEW: Strictness level parser.add_argument( '--strictness', type=str, default='balanced', choices=['lenient', 'balanced', 'strict'], help='Classification strictness (default: balanced)' ) # Export options parser.add_argument( '--export', type=str, default='json', help='Export formats: json, csv, csv_prices, quotation, google_sheets, all (comma-separated)' ) # Crawling options parser.add_argument( '--max-pages', type=int, default=50, help='Maximum pages to crawl (default: 50)' ) parser.add_argument( '--max-depth', type=int, default=3, help='Maximum crawl depth (default: 3)' ) parser.add_argument( '--delay', type=float, default=0.5, help='Delay between requests in seconds (default: 0.5)' ) # Output options parser.add_argument( '--output', type=str, default='product_catalog.json', help='Output filename (default: product_catalog.json)' ) args = parser.parse_args() # Get URL from args or env url = args.url or os.getenv('BASE_URL') if not url: print("\n❌ Error: No URL provided") print(" Use --url flag or set BASE_URL in .env file\n") sys.exit(1) # Parse export formats if args.export.lower() == 'all': export_formats = ['json', 'csv', 'csv_prices', 'quotation', 'google_sheets'] else: export_formats = [fmt.strip() for fmt in args.export.split(',')] # Create configuration config = ScraperConfig( base_url=url, max_pages=args.max_pages, max_depth=args.max_depth, output_filename=args.output, crawl_delay=args.delay ) # Validate configuration if not config.validate(): print("\n❌ Configuration validation failed. Please fix the errors above.\n") sys.exit(1) # Show configuration print("\n" + "="*80) print("SCRAPER CONFIGURATION - BALANCED MODE") print("="*80) print(f"Target URL: {config.base_url}") print(f"Strictness: {args.strictness.upper()}") print(f"Max Pages: {config.max_pages}") print(f"Max Depth: {config.max_depth}") print(f"Crawl Delay: {config.crawl_delay}s") print(f"Export Formats: {', '.join(export_formats)}") print(f"Output File: {config.full_output_path}") print("="*80 + "\n") # Explain strictness level strictness_info = { "lenient": "High recall - catches all products, some false positives", "balanced": "Good precision + recall - recommended for most sites", "strict": "High precision - very clean results, may miss some products" } print(f"ℹ️ Strictness: {strictness_info[args.strictness]}\n") # Confirm before starting try: user_input = input("Press Enter to start scraping (or Ctrl+C to cancel)... ") except KeyboardInterrupt: print("\n\n⛔ Scraping cancelled by user.\n") sys.exit(0) # Initialize scraper with strictness level scraper = BalancedScraper(config, strictness=args.strictness) # Scrape all products catalog = scraper.scrape_all_products() if not catalog: print("\n⚠️ No products found. Possible solutions:") print(" • Try --strictness lenient for higher recall") print(" • Increase --max-pages or --max-depth") print(" • The website structure might not match expected patterns") print(" • Verify the base URL is correct\n") sys.exit(1) # Save to specified formats print(f"\n{'='*80}") print("EXPORTING CATALOG") print(f"{'='*80}\n") scraper.save_catalog(catalog, export_formats=export_formats) # Print summary scraper.print_summary(catalog) print(f"\n✅ Done! Exported to {len(export_formats)} format(s).\n") # Show file locations print("📁 Output files:") if 'json' in export_formats: print(f" • JSON: {config.full_output_path}") if 'csv' in export_formats: print(f" • CSV: {config.full_output_path.replace('.json', '.csv')}") if 'csv_prices' in export_formats: print(f" • CSV (with prices): {config.full_output_path.replace('.json', '_with_prices.csv')}") if 'quotation' in export_formats: print(f" • Quotation Template: {config.full_output_path.replace('.json', '_quotation_template.json')}") if 'google_sheets' in export_formats: print(f" • Google Sheets: Check console output for URL") print() if __name__ == "__main__": main()