Spaces:
Paused
Paused
prince1604 commited on
Commit ·
d3e6b5e
1
Parent(s): e824b70
Optimize crawler speed: BFS + Reduced delays
Browse files- src/crawler.py +5 -4
src/crawler.py
CHANGED
|
@@ -153,8 +153,8 @@ class Crawler:
|
|
| 153 |
logger.info(f"Playwright navigating to {url}...")
|
| 154 |
page.goto(url, timeout=60000, wait_until="domcontentloaded")
|
| 155 |
|
| 156 |
-
# Wait for a bit of hydration
|
| 157 |
-
page.wait_for_timeout(
|
| 158 |
|
| 159 |
# Log page title
|
| 160 |
title = page.title()
|
|
@@ -203,8 +203,9 @@ class Crawler:
|
|
| 203 |
logger.info(f"Starting crawl for domain: {base_domain}")
|
| 204 |
|
| 205 |
while queue and pages_crawled < max_pages:
|
| 206 |
-
|
| 207 |
-
|
|
|
|
| 208 |
|
| 209 |
logger.info(f"Crawling [{pages_crawled + 1}]: {current_url}")
|
| 210 |
|
|
|
|
| 153 |
logger.info(f"Playwright navigating to {url}...")
|
| 154 |
page.goto(url, timeout=60000, wait_until="domcontentloaded")
|
| 155 |
|
| 156 |
+
# Wait for a bit of hydration (Reduced for speed)
|
| 157 |
+
page.wait_for_timeout(1000)
|
| 158 |
|
| 159 |
# Log page title
|
| 160 |
title = page.title()
|
|
|
|
| 203 |
logger.info(f"Starting crawl for domain: {base_domain}")
|
| 204 |
|
| 205 |
while queue and pages_crawled < max_pages:
|
| 206 |
+
# OPTIMIZATION: Use BFS (First-In-First-Out) instead of Random
|
| 207 |
+
# This follows the site structure naturally and is generally faster/more logical
|
| 208 |
+
current_url = queue.pop(0)
|
| 209 |
|
| 210 |
logger.info(f"Crawling [{pages_crawled + 1}]: {current_url}")
|
| 211 |
|