Spaces:
Sleeping
Sleeping
File size: 7,031 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 | """Tests for direct scrape kwargs on crawl() and start_crawl()."""
import asyncio
import inspect
import pytest
from firecrawl.v2.client import FirecrawlClient, _SCRAPE_OPTION_KEYS
from firecrawl.v2.client_async import AsyncFirecrawlClient
from firecrawl.v2.types import ScrapeOptions
import firecrawl.v2.methods.crawl as crawl_mod
from firecrawl.v2.methods.aio import crawl as async_crawl_mod
@pytest.fixture
def client():
return FirecrawlClient(api_key="fc-test", api_url="http://localhost:9")
@pytest.fixture
def async_client():
return AsyncFirecrawlClient(api_key="fc-test", api_url="http://localhost:9")
class TestSyncCrawlDirectKwargs:
def test_crawl_signature_accepts_formats(self):
sig = inspect.signature(FirecrawlClient.crawl)
assert "formats" in sig.parameters
def test_start_crawl_signature_accepts_formats(self):
sig = inspect.signature(FirecrawlClient.start_crawl)
assert "formats" in sig.parameters
def test_direct_kwargs_build_scrape_options(self, client, monkeypatch):
captured = {}
def fake_start_crawl(http, request):
captured["scrape_options"] = request.scrape_options
raise ConnectionError("captured")
monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl)
with pytest.raises(ConnectionError):
client.start_crawl("https://example.com", formats=["markdown"])
assert captured["scrape_options"] is not None
fmt_strs = [str(f) for f in captured["scrape_options"].formats]
assert "markdown" in fmt_strs
def test_explicit_scrape_options_wins(self, client, monkeypatch):
captured = {}
def fake_start_crawl(http, request):
captured["scrape_options"] = request.scrape_options
raise ConnectionError("captured")
monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl)
with pytest.raises(ConnectionError):
client.start_crawl(
"https://example.com",
scrape_options=ScrapeOptions(formats=["html"]),
formats=["markdown"],
)
fmt_strs = [str(f) for f in captured["scrape_options"].formats]
assert "html" in fmt_strs
assert "markdown" not in fmt_strs
def test_no_scrape_kwargs_means_no_scrape_options(self, client, monkeypatch):
captured = {}
def fake_start_crawl(http, request):
captured["scrape_options"] = request.scrape_options
raise ConnectionError("captured")
monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl)
with pytest.raises(ConnectionError):
client.start_crawl("https://example.com", limit=10)
assert captured["scrape_options"] is None
def test_crawl_also_builds_scrape_options(self, client, monkeypatch):
captured = {}
def fake_crawl(http, request, poll_interval=2, timeout=None, request_timeout=None):
captured["scrape_options"] = request.scrape_options
raise ConnectionError("captured")
monkeypatch.setattr(crawl_mod, "crawl", fake_crawl)
with pytest.raises(ConnectionError):
client.crawl("https://example.com", formats=["markdown"], only_main_content=True)
assert captured["scrape_options"] is not None
assert captured["scrape_options"].only_main_content is True
def test_start_crawl_accepts_scrape_timeout(self, client, monkeypatch):
captured = {}
def fake_start_crawl(http, request):
captured["scrape_options"] = request.scrape_options
raise ConnectionError("captured")
monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl)
with pytest.raises(ConnectionError):
client.start_crawl("https://example.com", timeout=5000)
assert captured["scrape_options"] is not None
assert captured["scrape_options"].timeout == 5000
def test_crawl_timeout_is_poll_timeout_not_scrape(self, client, monkeypatch):
captured = {}
def fake_crawl(http, request, poll_interval=2, timeout=None, request_timeout=None):
captured["scrape_options"] = request.scrape_options
captured["poll_timeout"] = timeout
raise ConnectionError("captured")
monkeypatch.setattr(crawl_mod, "crawl", fake_crawl)
with pytest.raises(ConnectionError):
client.crawl("https://example.com", timeout=60, formats=["markdown"])
assert captured["poll_timeout"] == 60
assert captured["scrape_options"] is not None
assert captured["scrape_options"].timeout is None
class TestAsyncCrawlDirectKwargs:
def test_async_direct_kwargs_build_scrape_options(self, async_client, monkeypatch):
captured = {}
async def fake_start_crawl(http, request):
captured["scrape_options"] = request.scrape_options
raise ConnectionError("captured")
monkeypatch.setattr(async_crawl_mod, "start_crawl", fake_start_crawl)
with pytest.raises(ConnectionError):
asyncio.get_event_loop().run_until_complete(
async_client.start_crawl("https://example.com", formats=["markdown"])
)
assert captured["scrape_options"] is not None
fmt_strs = [str(f) for f in captured["scrape_options"].formats]
assert "markdown" in fmt_strs
def test_async_explicit_scrape_options_wins(self, async_client, monkeypatch):
captured = {}
async def fake_start_crawl(http, request):
captured["scrape_options"] = request.scrape_options
raise ConnectionError("captured")
monkeypatch.setattr(async_crawl_mod, "start_crawl", fake_start_crawl)
with pytest.raises(ConnectionError):
asyncio.get_event_loop().run_until_complete(
async_client.start_crawl(
"https://example.com",
scrape_options=ScrapeOptions(formats=["html"]),
formats=["markdown"],
)
)
fmt_strs = [str(f) for f in captured["scrape_options"].formats]
assert "html" in fmt_strs
def test_async_integration_preserved_as_crawl_param(self, async_client, monkeypatch):
captured = {}
async def fake_start_crawl(http, request):
captured["scrape_options"] = request.scrape_options
captured["integration"] = request.integration
raise ConnectionError("captured")
monkeypatch.setattr(async_crawl_mod, "start_crawl", fake_start_crawl)
with pytest.raises(ConnectionError):
asyncio.get_event_loop().run_until_complete(
async_client.start_crawl(
"https://example.com",
formats=["markdown"],
integration="test-int",
)
)
assert captured["integration"] == "test-int"
assert captured["scrape_options"] is not None
|