| # Prefix-Based Input Handling in Crawl4AI |
|
|
| This guide will walk you through using the Crawl4AI library to crawl web pages, local HTML files, and raw HTML strings. We'll demonstrate these capabilities using a Wikipedia page as an example. |
|
|
| ## Crawling a Web URL |
|
|
| To crawl a live web page, provide the URL starting with `http://` or `https://`, using a `CrawlerRunConfig` object: |
|
|
| ```python |
| import asyncio |
| from crawl4ai import AsyncWebCrawler |
| from crawl4ai.async_configs import CrawlerRunConfig |
| |
| async def crawl_web(): |
| config = CrawlerRunConfig(bypass_cache=True) |
| async with AsyncWebCrawler() as crawler: |
| result = await crawler.arun(url="https://en.wikipedia.org/wiki/apple", config=config) |
| if result.success: |
| print("Markdown Content:") |
| print(result.markdown) |
| else: |
| print(f"Failed to crawl: {result.error_message}") |
| |
| asyncio.run(crawl_web()) |
| ``` |
|
|
| ## Crawling a Local HTML File |
|
|
| To crawl a local HTML file, prefix the file path with `file://`. |
|
|
| ```python |
| import asyncio |
| from crawl4ai import AsyncWebCrawler |
| from crawl4ai.async_configs import CrawlerRunConfig |
| |
| async def crawl_local_file(): |
| local_file_path = "/path/to/apple.html" # Replace with your file path |
| file_url = f"file://{local_file_path}" |
| config = CrawlerRunConfig(bypass_cache=True) |
| |
| async with AsyncWebCrawler() as crawler: |
| result = await crawler.arun(url=file_url, config=config) |
| if result.success: |
| print("Markdown Content from Local File:") |
| print(result.markdown) |
| else: |
| print(f"Failed to crawl local file: {result.error_message}") |
| |
| asyncio.run(crawl_local_file()) |
| ``` |
|
|
| ## Crawling Raw HTML Content |
|
|
| To crawl raw HTML content, prefix the HTML string with `raw:`. |
|
|
| ```python |
| import asyncio |
| from crawl4ai import AsyncWebCrawler |
| from crawl4ai.async_configs import CrawlerRunConfig |
| |
| async def crawl_raw_html(): |
| raw_html = "<html><body><h1>Hello, World!</h1></body></html>" |
| raw_html_url = f"raw:{raw_html}" |
| config = CrawlerRunConfig(bypass_cache=True) |
| |
| async with AsyncWebCrawler() as crawler: |
| result = await crawler.arun(url=raw_html_url, config=config) |
| if result.success: |
| print("Markdown Content from Raw HTML:") |
| print(result.markdown) |
| else: |
| print(f"Failed to crawl raw HTML: {result.error_message}") |
| |
| asyncio.run(crawl_raw_html()) |
| ``` |
|
|
| --- |
|
|
| # Complete Example |
|
|
| Below is a comprehensive script that: |
|
|
| 1. Crawls the Wikipedia page for "Apple." |
| 2. Saves the HTML content to a local file (`apple.html`). |
| 3. Crawls the local HTML file and verifies the markdown length matches the original crawl. |
| 4. Crawls the raw HTML content from the saved file and verifies consistency. |
|
|
| ```python |
| import os |
| import sys |
| import asyncio |
| from pathlib import Path |
| from crawl4ai import AsyncWebCrawler |
| from crawl4ai.async_configs import CrawlerRunConfig |
| |
| async def main(): |
| wikipedia_url = "https://en.wikipedia.org/wiki/apple" |
| script_dir = Path(__file__).parent |
| html_file_path = script_dir / "apple.html" |
| |
| async with AsyncWebCrawler() as crawler: |
| # Step 1: Crawl the Web URL |
| print("\n=== Step 1: Crawling the Wikipedia URL ===") |
| web_config = CrawlerRunConfig(bypass_cache=True) |
| result = await crawler.arun(url=wikipedia_url, config=web_config) |
| |
| if not result.success: |
| print(f"Failed to crawl {wikipedia_url}: {result.error_message}") |
| return |
| |
| with open(html_file_path, 'w', encoding='utf-8') as f: |
| f.write(result.html) |
| web_crawl_length = len(result.markdown) |
| print(f"Length of markdown from web crawl: {web_crawl_length}\n") |
| |
| # Step 2: Crawl from the Local HTML File |
| print("=== Step 2: Crawling from the Local HTML File ===") |
| file_url = f"file://{html_file_path.resolve()}" |
| file_config = CrawlerRunConfig(bypass_cache=True) |
| local_result = await crawler.arun(url=file_url, config=file_config) |
| |
| if not local_result.success: |
| print(f"Failed to crawl local file {file_url}: {local_result.error_message}") |
| return |
| |
| local_crawl_length = len(local_result.markdown) |
| assert web_crawl_length == local_crawl_length, "Markdown length mismatch" |
| print("✅ Markdown length matches between web and local file crawl.\n") |
| |
| # Step 3: Crawl Using Raw HTML Content |
| print("=== Step 3: Crawling Using Raw HTML Content ===") |
| with open(html_file_path, 'r', encoding='utf-8') as f: |
| raw_html_content = f.read() |
| raw_html_url = f"raw:{raw_html_content}" |
| raw_config = CrawlerRunConfig(bypass_cache=True) |
| raw_result = await crawler.arun(url=raw_html_url, config=raw_config) |
| |
| if not raw_result.success: |
| print(f"Failed to crawl raw HTML content: {raw_result.error_message}") |
| return |
| |
| raw_crawl_length = len(raw_result.markdown) |
| assert web_crawl_length == raw_crawl_length, "Markdown length mismatch" |
| print("✅ Markdown length matches between web and raw HTML crawl.\n") |
| |
| print("All tests passed successfully!") |
| if html_file_path.exists(): |
| os.remove(html_file_path) |
| |
| if __name__ == "__main__": |
| asyncio.run(main()) |
| ``` |
|
|
| --- |
|
|
| # Conclusion |
|
|
| With the unified `url` parameter and prefix-based handling in **Crawl4AI**, you can seamlessly handle web URLs, local HTML files, and raw HTML content. Use `CrawlerRunConfig` for flexible and consistent configuration in all scenarios. |
