| from src.crawler import Crawler | |
| import logging | |
| # Configure logging to see everything | |
| logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') | |
| def test_crawl(url, limit=5): | |
| print(f"Testing crawl for: {url} (limit={limit})") | |
| crawler = Crawler() | |
| site_data, total_discovered, blocked_reason = crawler.crawl_domain(url, max_pages=limit) | |
| print("\n--- RESULTS ---") | |
| print(f"Pages Crawled: {len(site_data)}") | |
| print(f"Total Discovered: {total_discovered}") | |
| print(f"Blocked Reason: {blocked_reason}") | |
| if site_data: | |
| for p, imgs in site_data.items(): | |
| print(f" - {p}: {len(imgs)} images") | |
| else: | |
| print(" [!] No data found.") | |
| if __name__ == "__main__": | |
| import sys | |
| target = sys.argv[1] if len(sys.argv) > 1 else "https://www.techtile.ai/" | |
| test_crawl(target) | |