prince1604 commited on
Commit
d3e6b5e
·
1 Parent(s): e824b70

Optimize crawler speed: BFS + Reduced delays

Browse files
Files changed (1) hide show
  1. src/crawler.py +5 -4
src/crawler.py CHANGED
@@ -153,8 +153,8 @@ class Crawler:
153
  logger.info(f"Playwright navigating to {url}...")
154
  page.goto(url, timeout=60000, wait_until="domcontentloaded")
155
 
156
- # Wait for a bit of hydration
157
- page.wait_for_timeout(3000)
158
 
159
  # Log page title
160
  title = page.title()
@@ -203,8 +203,9 @@ class Crawler:
203
  logger.info(f"Starting crawl for domain: {base_domain}")
204
 
205
  while queue and pages_crawled < max_pages:
206
- pop_index = random.randint(0, len(queue) - 1)
207
- current_url = queue.pop(pop_index)
 
208
 
209
  logger.info(f"Crawling [{pages_crawled + 1}]: {current_url}")
210
 
 
153
  logger.info(f"Playwright navigating to {url}...")
154
  page.goto(url, timeout=60000, wait_until="domcontentloaded")
155
 
156
+ # Wait for a bit of hydration (Reduced for speed)
157
+ page.wait_for_timeout(1000)
158
 
159
  # Log page title
160
  title = page.title()
 
203
  logger.info(f"Starting crawl for domain: {base_domain}")
204
 
205
  while queue and pages_crawled < max_pages:
206
+ # OPTIMIZATION: Use BFS (First-In-First-Out) instead of Random
207
+ # This follows the site structure naturally and is generally faster/more logical
208
+ current_url = queue.pop(0)
209
 
210
  logger.info(f"Crawling [{pages_crawled + 1}]: {current_url}")
211