Spaces:
Sleeping
Sleeping
File size: 1,979 Bytes
c496840 f927995 c496840 f927995 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | """
Async helpers for Playwright browser-based scraping.
"""
import asyncio
import logging
from typing import List, Dict, Optional
from playwright.async_api import async_playwright
# Setup Logging
logger = logging.getLogger("ingestion.playwright")
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
async def fetch_page_html(
url: str,
wait_for_selector: Optional[str] = None,
timeout_ms: int = 30000
) -> str:
"""
Launches a headless browser, navigates to a URL, and returns the full HTML content.
"""
async with async_playwright() as p:
browser = await p.chromium.launch(headless=True)
context = await browser.new_context(user_agent=USER_AGENT)
page = await context.new_page()
try:
logger.info(f"Navigating to {url}...")
await page.goto(url, timeout=timeout_ms)
if wait_for_selector:
await page.wait_for_selector(wait_for_selector, timeout=timeout_ms)
html = await page.content()
return html
except Exception as e:
logger.error(f"Error fetching {url}: {e}")
return ""
finally:
await browser.close()
async def fetch_pages_concurrent(
urls: List[str],
max_concurrency: int = 3,
rate_limit_per_sec: float = 0.5
) -> Dict[str, str]:
"""
Fetches multiple pages concurrently with a semaphore and rate limiting.
"""
semaphore = asyncio.Semaphore(max_concurrency)
results = {}
async def _bound_fetch(url: str):
async with semaphore:
html = await fetch_page_html(url)
results[url] = html
# Rate limit wait
await asyncio.sleep(1.0 / rate_limit_per_sec)
tasks = [asyncio.create_task(_bound_fetch(url)) for url in urls]
await asyncio.gather(*tasks)
return results
|