Spaces:
Sleeping
Sleeping
Download apps/python-sdk/firecrawl/v2/methods/aio/extract.py from s-eto-hub/firecrawl-api: direct link, hf CLI and curl.
- Browser
- Download file 5.9 kB
-
https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/python-sdk/firecrawl/v2/methods/aio/extract.py
- Command line
-
hf download hf://spaces/s-eto-hub/firecrawl-api/apps/python-sdk/firecrawl/v2/methods/aio/extract.py
-
curl -L -o extract.py https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/python-sdk/firecrawl/v2/methods/aio/extract.py
5.9 kB
| from typing import Any, Dict, List, Optional | |
| import asyncio | |
| import warnings | |
| from ...types import ExtractResponse, ScrapeOptions | |
| from ...utils.http_client_async import AsyncHttpClient | |
| from ...utils.validation import prepare_scrape_options | |
| _EXTRACT_DEPRECATION_MSG = ( | |
| "The extract endpoint is in maintenance mode and its use is discouraged. " | |
| "Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor " | |
| "to find a replacement." | |
| ) | |
| def _prepare_extract_request( | |
| urls: Optional[List[str]], | |
| *, | |
| prompt: Optional[str] = None, | |
| schema: Optional[Dict[str, Any]] = None, | |
| system_prompt: Optional[str] = None, | |
| allow_external_links: Optional[bool] = None, | |
| enable_web_search: Optional[bool] = None, | |
| show_sources: Optional[bool] = None, | |
| scrape_options: Optional[ScrapeOptions] = None, | |
| ignore_invalid_urls: Optional[bool] = None, | |
| integration: Optional[str] = None, | |
| ) -> Dict[str, Any]: | |
| body: Dict[str, Any] = {} | |
| if urls is not None: | |
| body["urls"] = urls | |
| if prompt is not None: | |
| body["prompt"] = prompt | |
| if schema is not None: | |
| body["schema"] = schema | |
| if system_prompt is not None: | |
| body["systemPrompt"] = system_prompt | |
| if allow_external_links is not None: | |
| body["allowExternalLinks"] = allow_external_links | |
| if enable_web_search is not None: | |
| body["enableWebSearch"] = enable_web_search | |
| if show_sources is not None: | |
| body["showSources"] = show_sources | |
| if ignore_invalid_urls is not None: | |
| body["ignoreInvalidURLs"] = ignore_invalid_urls | |
| if scrape_options is not None: | |
| prepared = prepare_scrape_options(scrape_options) | |
| if prepared: | |
| body["scrapeOptions"] = prepared | |
| if integration is not None and str(integration).strip(): | |
| body["integration"] = str(integration).strip() | |
| return body | |
| async def start_extract( | |
| client: AsyncHttpClient, | |
| urls: Optional[List[str]], | |
| *, | |
| prompt: Optional[str] = None, | |
| schema: Optional[Dict[str, Any]] = None, | |
| system_prompt: Optional[str] = None, | |
| allow_external_links: Optional[bool] = None, | |
| enable_web_search: Optional[bool] = None, | |
| show_sources: Optional[bool] = None, | |
| scrape_options: Optional[ScrapeOptions] = None, | |
| ignore_invalid_urls: Optional[bool] = None, | |
| integration: Optional[str] = None, | |
| ) -> ExtractResponse: | |
| """Start an extract job (non-blocking, async). | |
| .. deprecated:: | |
| The extract endpoint is in maintenance mode and its use is discouraged. | |
| Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor | |
| to find a replacement. | |
| """ | |
| warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2) | |
| body = _prepare_extract_request( | |
| urls, | |
| prompt=prompt, | |
| schema=schema, | |
| system_prompt=system_prompt, | |
| allow_external_links=allow_external_links, | |
| enable_web_search=enable_web_search, | |
| show_sources=show_sources, | |
| scrape_options=scrape_options, | |
| ignore_invalid_urls=ignore_invalid_urls, | |
| integration=integration, | |
| ) | |
| resp = await client.post("/v2/extract", body) | |
| return ExtractResponse(**resp.json()) | |
| async def get_extract_status(client: AsyncHttpClient, job_id: str) -> ExtractResponse: | |
| """Get the current status of an extract job (async). | |
| .. deprecated:: | |
| The extract endpoint is in maintenance mode and its use is discouraged. | |
| Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor | |
| to find a replacement. | |
| """ | |
| warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2) | |
| resp = await client.get(f"/v2/extract/{job_id}") | |
| return ExtractResponse(**resp.json()) | |
| async def wait_extract( | |
| client: AsyncHttpClient, | |
| job_id: str, | |
| *, | |
| poll_interval: int = 2, | |
| timeout: Optional[int] = None, | |
| ) -> ExtractResponse: | |
| start_ts = asyncio.get_event_loop().time() | |
| while True: | |
| status = await get_extract_status(client, job_id) | |
| if status.status in ("completed", "failed", "cancelled"): | |
| return status | |
| if timeout is not None and (asyncio.get_event_loop().time() - start_ts) > timeout: | |
| return status | |
| await asyncio.sleep(max(1, poll_interval)) | |
| async def extract( | |
| client: AsyncHttpClient, | |
| urls: Optional[List[str]], | |
| *, | |
| prompt: Optional[str] = None, | |
| schema: Optional[Dict[str, Any]] = None, | |
| system_prompt: Optional[str] = None, | |
| allow_external_links: Optional[bool] = None, | |
| enable_web_search: Optional[bool] = None, | |
| show_sources: Optional[bool] = None, | |
| scrape_options: Optional[ScrapeOptions] = None, | |
| ignore_invalid_urls: Optional[bool] = None, | |
| poll_interval: int = 2, | |
| timeout: Optional[int] = None, | |
| integration: Optional[str] = None, | |
| ) -> ExtractResponse: | |
| """Extract structured data and wait until completion (async). | |
| .. deprecated:: | |
| The extract endpoint is in maintenance mode and its use is discouraged. | |
| Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor | |
| to find a replacement. | |
| """ | |
| warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2) | |
| started = await start_extract( | |
| client, | |
| urls, | |
| prompt=prompt, | |
| schema=schema, | |
| system_prompt=system_prompt, | |
| allow_external_links=allow_external_links, | |
| enable_web_search=enable_web_search, | |
| show_sources=show_sources, | |
| scrape_options=scrape_options, | |
| ignore_invalid_urls=ignore_invalid_urls, | |
| integration=integration, | |
| ) | |
| job_id = getattr(started, "id", None) | |
| if not job_id: | |
| return started | |
| return await wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout) | |