| import asyncio |
| from crawlee.crawlers import ParselCrawler, ParselCrawlingContext |
| from urllib.parse import urljoin |
|
|
|
|
| async def crawl_parse_url(url: str): |
| """Scrape a single URL using Crawlee + ParselCrawler and return one record""" |
|
|
| result_container = {"data": None} |
|
|
| crawler = ParselCrawler(max_requests_per_crawl=1) |
|
|
|
|
|
|
| @crawler.router.default_handler |
| async def handle_page(ctx: ParselCrawlingContext): |
| selector = ctx.selector |
| req_url = ctx.request.url |
|
|
| |
| title = selector.xpath("//title/text()").get() or "No title" |
|
|
| |
| text = selector.xpath("string(//body)").get() |
| text = text.strip() if text else "" |
|
|
| |
| img_urls = [ |
| urljoin(req_url, src) for src in selector.xpath("//img/@src").getall() |
| ] |
|
|
| video_urls = [ |
| urljoin(req_url, src) for src in selector.xpath("//video/@src").getall() |
| ] + [ |
| urljoin(req_url, src) |
| for src in selector.xpath("//source[@type='video/mp4']/@src").getall() |
| ] |
|
|
| audio_urls = [ |
| urljoin(req_url, src) for src in selector.xpath("//audio/@src").getall() |
| ] + [ |
| urljoin(req_url, src) |
| for src in selector.xpath("//source[@type='audio/mpeg']/@src").getall() |
| ] |
|
|
| pdf_urls = [ |
| urljoin(req_url, href) |
| for href in selector.xpath("//a[contains(@href, '.pdf')]/@href").getall() |
| ] |
|
|
| |
| record = { |
| "page_url": req_url, |
| "title": title, |
| "content": text, |
| "img_url": img_urls, |
| "video_url": video_urls, |
| "audio_url": audio_urls, |
| "pdf_url": pdf_urls, |
| "agent_recommendation_rank": 4.2, |
| "agent_recommendation_notes": "Scraped successfully using Crawlee + ParselCrawler.", |
| "header": "Web Scraping Test", |
| "sub_header": "Crawlee Version", |
| } |
|
|
| result_container["data"] = record |
| await ctx.push_data(record) |
|
|
| await crawler.run([url]) |
|
|
| |
| return result_container["data"] or { |
| "page_url": url, |
| "title": "No title", |
| "content": "", |
| "img_url": [], |
| "video_url": [], |
| "audio_url": [], |
| "pdf_url": [], |
| "agent_recommendation_rank": 4.2, |
| "agent_recommendation_notes": "Scraped successfully using Crawlee + ParselCrawler.", |
| } |
|
|
