File size: 866 Bytes
37260b4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 | from src.crawler import Crawler
import logging
# Configure logging to see everything
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
def test_crawl(url, limit=5):
print(f"Testing crawl for: {url} (limit={limit})")
crawler = Crawler()
site_data, total_discovered, blocked_reason = crawler.crawl_domain(url, max_pages=limit)
print("\n--- RESULTS ---")
print(f"Pages Crawled: {len(site_data)}")
print(f"Total Discovered: {total_discovered}")
print(f"Blocked Reason: {blocked_reason}")
if site_data:
for p, imgs in site_data.items():
print(f" - {p}: {len(imgs)} images")
else:
print(" [!] No data found.")
if __name__ == "__main__":
import sys
target = sys.argv[1] if len(sys.argv) > 1 else "https://www.techtile.ai/"
test_crawl(target)
|