Spaces:
Sleeping
Sleeping
Download apps/python-sdk/firecrawl/v2/methods/extract.py from s-eto-hub/firecrawl-api: direct link, hf CLI and curl.
- Browser
- Download file 6.9 kB
-
https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/python-sdk/firecrawl/v2/methods/extract.py
- Command line
-
hf download hf://spaces/s-eto-hub/firecrawl-api/apps/python-sdk/firecrawl/v2/methods/extract.py
-
curl -L -o extract.py https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/python-sdk/firecrawl/v2/methods/extract.py
6.9 kB
| from typing import Any, Dict, List, Optional | |
| import time | |
| import warnings | |
| from ..types import ExtractResponse, ScrapeOptions | |
| from ..types import AgentOptions | |
| from ..utils.http_client import HttpClient | |
| from ..utils.validation import prepare_scrape_options | |
| from ..utils.error_handler import handle_response_error | |
| _EXTRACT_DEPRECATION_MSG = ( | |
| "The extract endpoint is in maintenance mode and its use is discouraged. " | |
| "Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor " | |
| "to find a replacement." | |
| ) | |
| def _prepare_extract_request( | |
| urls: Optional[List[str]], | |
| *, | |
| prompt: Optional[str] = None, | |
| schema: Optional[Dict[str, Any]] = None, | |
| system_prompt: Optional[str] = None, | |
| allow_external_links: Optional[bool] = None, | |
| enable_web_search: Optional[bool] = None, | |
| show_sources: Optional[bool] = None, | |
| scrape_options: Optional[ScrapeOptions] = None, | |
| ignore_invalid_urls: Optional[bool] = None, | |
| integration: Optional[str] = None, | |
| agent: Optional[AgentOptions] = None, | |
| ) -> Dict[str, Any]: | |
| body: Dict[str, Any] = {} | |
| if urls is not None: | |
| body["urls"] = urls | |
| if prompt is not None: | |
| body["prompt"] = prompt | |
| if schema is not None: | |
| body["schema"] = schema | |
| if system_prompt is not None: | |
| body["systemPrompt"] = system_prompt | |
| if allow_external_links is not None: | |
| body["allowExternalLinks"] = allow_external_links | |
| if enable_web_search is not None: | |
| body["enableWebSearch"] = enable_web_search | |
| if show_sources is not None: | |
| body["showSources"] = show_sources | |
| if ignore_invalid_urls is not None: | |
| body["ignoreInvalidURLs"] = ignore_invalid_urls | |
| if scrape_options is not None: | |
| prepared = prepare_scrape_options(scrape_options) | |
| if prepared: | |
| body["scrapeOptions"] = prepared | |
| if integration is not None and str(integration).strip(): | |
| body["integration"] = str(integration).strip() | |
| if agent is not None: | |
| try: | |
| body["agent"] = agent.model_dump(exclude_none=True) # type: ignore[attr-defined] | |
| except AttributeError: | |
| body["agent"] = agent # fallback | |
| return body | |
| def _normalize_extract_response_payload(payload: Dict[str, Any]) -> Dict[str, Any]: | |
| out = dict(payload) | |
| if "expiresAt" in out and "expires_at" not in out: | |
| out["expires_at"] = out["expiresAt"] | |
| if "creditsUsed" in out and "credits_used" not in out: | |
| out["credits_used"] = out["creditsUsed"] | |
| if "tokensUsed" in out and "tokens_used" not in out: | |
| out["tokens_used"] = out["tokensUsed"] | |
| return out | |
| def start_extract( | |
| client: HttpClient, | |
| urls: Optional[List[str]], | |
| *, | |
| prompt: Optional[str] = None, | |
| schema: Optional[Dict[str, Any]] = None, | |
| system_prompt: Optional[str] = None, | |
| allow_external_links: Optional[bool] = None, | |
| enable_web_search: Optional[bool] = None, | |
| show_sources: Optional[bool] = None, | |
| scrape_options: Optional[ScrapeOptions] = None, | |
| ignore_invalid_urls: Optional[bool] = None, | |
| integration: Optional[str] = None, | |
| agent: Optional[AgentOptions] = None, | |
| ) -> ExtractResponse: | |
| """Start an extract job (non-blocking). | |
| .. deprecated:: | |
| The extract endpoint is in maintenance mode and its use is discouraged. | |
| Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor | |
| to find a replacement. | |
| """ | |
| warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2) | |
| body = _prepare_extract_request( | |
| urls, | |
| prompt=prompt, | |
| schema=schema, | |
| system_prompt=system_prompt, | |
| allow_external_links=allow_external_links, | |
| enable_web_search=enable_web_search, | |
| show_sources=show_sources, | |
| scrape_options=scrape_options, | |
| ignore_invalid_urls=ignore_invalid_urls, | |
| integration=integration, | |
| agent=agent, | |
| ) | |
| resp = client.post("/v2/extract", body) | |
| if not resp.ok: | |
| handle_response_error(resp, "extract") | |
| payload = _normalize_extract_response_payload(resp.json()) | |
| return ExtractResponse(**payload) | |
| def get_extract_status(client: HttpClient, job_id: str) -> ExtractResponse: | |
| """Get the current status of an extract job. | |
| .. deprecated:: | |
| The extract endpoint is in maintenance mode and its use is discouraged. | |
| Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor | |
| to find a replacement. | |
| """ | |
| warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2) | |
| resp = client.get(f"/v2/extract/{job_id}") | |
| if not resp.ok: | |
| handle_response_error(resp, "extract-status") | |
| payload = _normalize_extract_response_payload(resp.json()) | |
| return ExtractResponse(**payload) | |
| def wait_extract( | |
| client: HttpClient, | |
| job_id: str, | |
| *, | |
| poll_interval: int = 2, | |
| timeout: Optional[int] = None, | |
| ) -> ExtractResponse: | |
| start_ts = time.time() | |
| while True: | |
| status = get_extract_status(client, job_id) | |
| if status.status in ("completed", "failed", "cancelled"): | |
| return status | |
| if timeout is not None and (time.time() - start_ts) > timeout: | |
| return status | |
| time.sleep(max(1, poll_interval)) | |
| def extract( | |
| client: HttpClient, | |
| urls: Optional[List[str]], | |
| *, | |
| prompt: Optional[str] = None, | |
| schema: Optional[Dict[str, Any]] = None, | |
| system_prompt: Optional[str] = None, | |
| allow_external_links: Optional[bool] = None, | |
| enable_web_search: Optional[bool] = None, | |
| show_sources: Optional[bool] = None, | |
| scrape_options: Optional[ScrapeOptions] = None, | |
| ignore_invalid_urls: Optional[bool] = None, | |
| poll_interval: int = 2, | |
| timeout: Optional[int] = None, | |
| integration: Optional[str] = None, | |
| agent: Optional[AgentOptions] = None, | |
| ) -> ExtractResponse: | |
| """Extract structured data and wait until completion. | |
| .. deprecated:: | |
| The extract endpoint is in maintenance mode and its use is discouraged. | |
| Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor | |
| to find a replacement. | |
| """ | |
| warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2) | |
| started = start_extract( | |
| client, | |
| urls, | |
| prompt=prompt, | |
| schema=schema, | |
| system_prompt=system_prompt, | |
| allow_external_links=allow_external_links, | |
| enable_web_search=enable_web_search, | |
| show_sources=show_sources, | |
| scrape_options=scrape_options, | |
| ignore_invalid_urls=ignore_invalid_urls, | |
| integration=integration, | |
| agent=agent, | |
| ) | |
| job_id = getattr(started, "id", None) | |
| if not job_id: | |
| return started | |
| return wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout) | |