Spaces:
Sleeping
Sleeping
Download apps/python-sdk/tests/test_crawl_direct_kwargs.py from s-eto-hub/firecrawl-api: direct link, hf CLI and curl.
- Browser
- Download file 7.03 kB
-
https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/python-sdk/tests/test_crawl_direct_kwargs.py
- Command line
-
hf download hf://spaces/s-eto-hub/firecrawl-api/apps/python-sdk/tests/test_crawl_direct_kwargs.py
-
curl -L -o test_crawl_direct_kwargs.py https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/python-sdk/tests/test_crawl_direct_kwargs.py
7.03 kB
| """Tests for direct scrape kwargs on crawl() and start_crawl().""" | |
| import asyncio | |
| import inspect | |
| import pytest | |
| from firecrawl.v2.client import FirecrawlClient, _SCRAPE_OPTION_KEYS | |
| from firecrawl.v2.client_async import AsyncFirecrawlClient | |
| from firecrawl.v2.types import ScrapeOptions | |
| import firecrawl.v2.methods.crawl as crawl_mod | |
| from firecrawl.v2.methods.aio import crawl as async_crawl_mod | |
| def client(): | |
| return FirecrawlClient(api_key="fc-test", api_url="http://localhost:9") | |
| def async_client(): | |
| return AsyncFirecrawlClient(api_key="fc-test", api_url="http://localhost:9") | |
| class TestSyncCrawlDirectKwargs: | |
| def test_crawl_signature_accepts_formats(self): | |
| sig = inspect.signature(FirecrawlClient.crawl) | |
| assert "formats" in sig.parameters | |
| def test_start_crawl_signature_accepts_formats(self): | |
| sig = inspect.signature(FirecrawlClient.start_crawl) | |
| assert "formats" in sig.parameters | |
| def test_direct_kwargs_build_scrape_options(self, client, monkeypatch): | |
| captured = {} | |
| def fake_start_crawl(http, request): | |
| captured["scrape_options"] = request.scrape_options | |
| raise ConnectionError("captured") | |
| monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl) | |
| with pytest.raises(ConnectionError): | |
| client.start_crawl("https://example.com", formats=["markdown"]) | |
| assert captured["scrape_options"] is not None | |
| fmt_strs = [str(f) for f in captured["scrape_options"].formats] | |
| assert "markdown" in fmt_strs | |
| def test_explicit_scrape_options_wins(self, client, monkeypatch): | |
| captured = {} | |
| def fake_start_crawl(http, request): | |
| captured["scrape_options"] = request.scrape_options | |
| raise ConnectionError("captured") | |
| monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl) | |
| with pytest.raises(ConnectionError): | |
| client.start_crawl( | |
| "https://example.com", | |
| scrape_options=ScrapeOptions(formats=["html"]), | |
| formats=["markdown"], | |
| ) | |
| fmt_strs = [str(f) for f in captured["scrape_options"].formats] | |
| assert "html" in fmt_strs | |
| assert "markdown" not in fmt_strs | |
| def test_no_scrape_kwargs_means_no_scrape_options(self, client, monkeypatch): | |
| captured = {} | |
| def fake_start_crawl(http, request): | |
| captured["scrape_options"] = request.scrape_options | |
| raise ConnectionError("captured") | |
| monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl) | |
| with pytest.raises(ConnectionError): | |
| client.start_crawl("https://example.com", limit=10) | |
| assert captured["scrape_options"] is None | |
| def test_crawl_also_builds_scrape_options(self, client, monkeypatch): | |
| captured = {} | |
| def fake_crawl(http, request, poll_interval=2, timeout=None, request_timeout=None): | |
| captured["scrape_options"] = request.scrape_options | |
| raise ConnectionError("captured") | |
| monkeypatch.setattr(crawl_mod, "crawl", fake_crawl) | |
| with pytest.raises(ConnectionError): | |
| client.crawl("https://example.com", formats=["markdown"], only_main_content=True) | |
| assert captured["scrape_options"] is not None | |
| assert captured["scrape_options"].only_main_content is True | |
| def test_start_crawl_accepts_scrape_timeout(self, client, monkeypatch): | |
| captured = {} | |
| def fake_start_crawl(http, request): | |
| captured["scrape_options"] = request.scrape_options | |
| raise ConnectionError("captured") | |
| monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl) | |
| with pytest.raises(ConnectionError): | |
| client.start_crawl("https://example.com", timeout=5000) | |
| assert captured["scrape_options"] is not None | |
| assert captured["scrape_options"].timeout == 5000 | |
| def test_crawl_timeout_is_poll_timeout_not_scrape(self, client, monkeypatch): | |
| captured = {} | |
| def fake_crawl(http, request, poll_interval=2, timeout=None, request_timeout=None): | |
| captured["scrape_options"] = request.scrape_options | |
| captured["poll_timeout"] = timeout | |
| raise ConnectionError("captured") | |
| monkeypatch.setattr(crawl_mod, "crawl", fake_crawl) | |
| with pytest.raises(ConnectionError): | |
| client.crawl("https://example.com", timeout=60, formats=["markdown"]) | |
| assert captured["poll_timeout"] == 60 | |
| assert captured["scrape_options"] is not None | |
| assert captured["scrape_options"].timeout is None | |
| class TestAsyncCrawlDirectKwargs: | |
| def test_async_direct_kwargs_build_scrape_options(self, async_client, monkeypatch): | |
| captured = {} | |
| async def fake_start_crawl(http, request): | |
| captured["scrape_options"] = request.scrape_options | |
| raise ConnectionError("captured") | |
| monkeypatch.setattr(async_crawl_mod, "start_crawl", fake_start_crawl) | |
| with pytest.raises(ConnectionError): | |
| asyncio.get_event_loop().run_until_complete( | |
| async_client.start_crawl("https://example.com", formats=["markdown"]) | |
| ) | |
| assert captured["scrape_options"] is not None | |
| fmt_strs = [str(f) for f in captured["scrape_options"].formats] | |
| assert "markdown" in fmt_strs | |
| def test_async_explicit_scrape_options_wins(self, async_client, monkeypatch): | |
| captured = {} | |
| async def fake_start_crawl(http, request): | |
| captured["scrape_options"] = request.scrape_options | |
| raise ConnectionError("captured") | |
| monkeypatch.setattr(async_crawl_mod, "start_crawl", fake_start_crawl) | |
| with pytest.raises(ConnectionError): | |
| asyncio.get_event_loop().run_until_complete( | |
| async_client.start_crawl( | |
| "https://example.com", | |
| scrape_options=ScrapeOptions(formats=["html"]), | |
| formats=["markdown"], | |
| ) | |
| ) | |
| fmt_strs = [str(f) for f in captured["scrape_options"].formats] | |
| assert "html" in fmt_strs | |
| def test_async_integration_preserved_as_crawl_param(self, async_client, monkeypatch): | |
| captured = {} | |
| async def fake_start_crawl(http, request): | |
| captured["scrape_options"] = request.scrape_options | |
| captured["integration"] = request.integration | |
| raise ConnectionError("captured") | |
| monkeypatch.setattr(async_crawl_mod, "start_crawl", fake_start_crawl) | |
| with pytest.raises(ConnectionError): | |
| asyncio.get_event_loop().run_until_complete( | |
| async_client.start_crawl( | |
| "https://example.com", | |
| formats=["markdown"], | |
| integration="test-int", | |
| ) | |
| ) | |
| assert captured["integration"] == "test-int" | |
| assert captured["scrape_options"] is not None | |