Spaces:
Sleeping
Sleeping
Download apps/python-sdk/firecrawl/v2/methods/scrape.py from s-eto-hub/firecrawl-api: direct link, hf CLI and curl.
- Browser
- Download file 6.25 kB
-
https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/python-sdk/firecrawl/v2/methods/scrape.py
- Command line
-
hf download hf://spaces/s-eto-hub/firecrawl-api/apps/python-sdk/firecrawl/v2/methods/scrape.py
-
curl -L -o scrape.py https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/python-sdk/firecrawl/v2/methods/scrape.py
6.25 kB
| """ | |
| Scraping functionality for Firecrawl v2 API. | |
| """ | |
| from typing import Optional, Dict, Any, Literal | |
| from ..types import ( | |
| ScrapeOptions, | |
| Document, | |
| BrowserExecuteResponse, | |
| BrowserDeleteResponse, | |
| ) | |
| from ..utils.normalize import normalize_document_input | |
| from ..utils import HttpClient, handle_response_error, prepare_scrape_options, validate_scrape_options | |
| def _prepare_scrape_request(url: str, options: Optional[ScrapeOptions] = None) -> Dict[str, Any]: | |
| """ | |
| Prepare a scrape request payload for v2 API. | |
| Args: | |
| url: URL to scrape | |
| options: ScrapeOptions (snake_case) to convert and include | |
| Returns: | |
| Request payload dictionary with camelCase fields | |
| """ | |
| if not url or not url.strip(): | |
| raise ValueError("URL cannot be empty") | |
| request_data: Dict[str, Any] = {"url": url.strip()} | |
| if options is not None: | |
| validated = validate_scrape_options(options) | |
| if validated is not None: | |
| opts = prepare_scrape_options(validated) | |
| if opts: | |
| request_data.update(opts) | |
| return request_data | |
| def scrape(client: HttpClient, url: str, options: Optional[ScrapeOptions] = None) -> Document: | |
| """ | |
| Scrape a single URL and return the document. | |
| The v2 API returns: { success: boolean, data: Document } | |
| We surface just the Document to callers. | |
| Args: | |
| client: HTTP client instance | |
| url: URL to scrape | |
| options: Scraping options (snake_case) | |
| Returns: | |
| Document | |
| """ | |
| payload = _prepare_scrape_request(url, options) | |
| response = client.post("/v2/scrape", payload) | |
| if not response.ok: | |
| handle_response_error(response, "scrape") | |
| body = response.json() | |
| if not body.get("success"): | |
| raise Exception(body.get("error", "Unknown error occurred")) | |
| document_data = body.get("data", {}) | |
| normalized = normalize_document_input(document_data) | |
| return Document(**normalized) | |
| def interact( | |
| client: HttpClient, | |
| job_id: str, | |
| code: Optional[str] = None, | |
| *, | |
| prompt: Optional[str] = None, | |
| language: Literal["python", "node", "bash"] = "node", | |
| timeout: Optional[int] = None, | |
| origin: Optional[str] = None, | |
| ) -> BrowserExecuteResponse: | |
| """ | |
| Interact with the scrape-bound browser session for a scrape job. | |
| Either ``code`` or ``prompt`` must be provided. When ``prompt`` is given | |
| the server runs an AI agent that translates the natural-language instruction | |
| into browser actions. | |
| Args: | |
| client: HTTP client instance | |
| job_id: Scrape job ID | |
| code: Code to execute (optional if prompt is provided) | |
| prompt: Natural-language instruction for the browser agent (optional if code is provided) | |
| language: Programming language ("python", "node", or "bash") | |
| timeout: Execution timeout in seconds (1-300) | |
| origin: Optional request origin tag | |
| Returns: | |
| BrowserExecuteResponse with execution output | |
| """ | |
| if not job_id or not job_id.strip(): | |
| raise ValueError("Job ID cannot be empty") | |
| has_code = code and code.strip() | |
| has_prompt = prompt and prompt.strip() | |
| if not has_code and not has_prompt: | |
| raise ValueError("Either 'code' or 'prompt' must be provided") | |
| body: Dict[str, Any] = { | |
| "language": language, | |
| } | |
| if has_code: | |
| body["code"] = code | |
| if has_prompt: | |
| body["prompt"] = prompt | |
| if timeout is not None: | |
| body["timeout"] = timeout | |
| if origin is not None: | |
| body["origin"] = origin | |
| response = client.post(f"/v2/scrape/{job_id}/interact", body) | |
| if not response.ok: | |
| handle_response_error(response, "interact with scrape browser") | |
| payload = response.json() | |
| if not payload.get("success"): | |
| raise Exception(payload.get("error", "Unknown error occurred")) | |
| normalized = dict(payload) | |
| if "exitCode" in normalized and "exit_code" not in normalized: | |
| normalized["exit_code"] = normalized["exitCode"] | |
| if "liveViewUrl" in normalized and "live_view_url" not in normalized: | |
| normalized["live_view_url"] = normalized["liveViewUrl"] | |
| if "interactiveLiveViewUrl" in normalized and "interactive_live_view_url" not in normalized: | |
| normalized["interactive_live_view_url"] = normalized["interactiveLiveViewUrl"] | |
| return BrowserExecuteResponse(**normalized) | |
| def stop_interaction( | |
| client: HttpClient, | |
| job_id: str, | |
| ) -> BrowserDeleteResponse: | |
| """ | |
| Stop the interaction session for a scrape job. | |
| Args: | |
| client: HTTP client instance | |
| job_id: Scrape job ID | |
| Returns: | |
| BrowserDeleteResponse | |
| """ | |
| if not job_id or not job_id.strip(): | |
| raise ValueError("Job ID cannot be empty") | |
| response = client.delete(f"/v2/scrape/{job_id}/interact") | |
| if not response.ok: | |
| handle_response_error(response, "stop interaction") | |
| payload = response.json() | |
| normalized = dict(payload) | |
| if "sessionDurationMs" in normalized and "session_duration_ms" not in normalized: | |
| normalized["session_duration_ms"] = normalized["sessionDurationMs"] | |
| if "creditsBilled" in normalized and "credits_billed" not in normalized: | |
| normalized["credits_billed"] = normalized["creditsBilled"] | |
| return BrowserDeleteResponse(**normalized) | |
| def stop_interactive_browser( | |
| client: HttpClient, | |
| job_id: str, | |
| ) -> BrowserDeleteResponse: | |
| """Deprecated alias for stop_interaction().""" | |
| return stop_interaction(client, job_id) | |
| def scrape_execute( | |
| client: HttpClient, | |
| job_id: str, | |
| code: Optional[str] = None, | |
| *, | |
| prompt: Optional[str] = None, | |
| language: Literal["python", "node", "bash"] = "node", | |
| timeout: Optional[int] = None, | |
| origin: Optional[str] = None, | |
| ) -> BrowserExecuteResponse: | |
| """Deprecated alias for interact().""" | |
| return interact( | |
| client, | |
| job_id, | |
| code, | |
| prompt=prompt, | |
| language=language, | |
| timeout=timeout, | |
| origin=origin, | |
| ) | |
| def delete_scrape_browser( | |
| client: HttpClient, | |
| job_id: str, | |
| ) -> BrowserDeleteResponse: | |
| """Deprecated alias for stop_interaction().""" | |
| return stop_interaction(client, job_id) |