Spaces:
Sleeping
Sleeping
Download apps/python-sdk/firecrawl/v2/methods/batch.py from s-eto-hub/firecrawl-api: direct link, hf CLI and curl.
- Browser
- Download file 16.4 kB
-
https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/python-sdk/firecrawl/v2/methods/batch.py
- Command line
-
hf download hf://spaces/s-eto-hub/firecrawl-api/apps/python-sdk/firecrawl/v2/methods/batch.py
-
curl -L -o batch.py https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/python-sdk/firecrawl/v2/methods/batch.py
16.4 kB
| """ | |
| Batch scraping functionality for Firecrawl v2 API. | |
| """ | |
| import time | |
| from typing import Optional, List, Callable, Dict, Any, Union | |
| from ..types import ( | |
| BatchScrapeRequest, | |
| BatchScrapeResponse, | |
| BatchScrapeJob, | |
| ScrapeOptions, | |
| Document, | |
| WebhookConfig, | |
| PaginationConfig, | |
| ) | |
| from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options | |
| from ..utils.normalize import normalize_document_input | |
| from ..types import CrawlErrorsResponse | |
| def _parse_batch_scrape_documents(data_list: Optional[List[Any]]) -> List[Document]: | |
| documents: List[Document] = [] | |
| for doc in data_list or []: | |
| if isinstance(doc, dict): | |
| normalized = normalize_document_input(doc) | |
| documents.append(Document(**normalized)) | |
| return documents | |
| def _parse_batch_scrape_status_response(body: Dict[str, Any]) -> Dict[str, Any]: | |
| if not body.get("success"): | |
| raise Exception(body.get("error", "Unknown error occurred")) | |
| return { | |
| "status": body.get("status"), | |
| "completed": body.get("completed", 0), | |
| "total": body.get("total", 0), | |
| "credits_used": body.get("creditsUsed"), | |
| "expires_at": body.get("expiresAt"), | |
| "next": body.get("next"), | |
| "data": _parse_batch_scrape_documents(body.get("data", []) or []), | |
| } | |
| def start_batch_scrape( | |
| client: HttpClient, | |
| urls: List[str], | |
| *, | |
| options: Optional[ScrapeOptions] = None, | |
| webhook: Optional[Union[str, WebhookConfig]] = None, | |
| append_to_id: Optional[str] = None, | |
| ignore_invalid_urls: Optional[bool] = None, | |
| max_concurrency: Optional[int] = None, | |
| zero_data_retention: Optional[bool] = None, | |
| integration: Optional[str] = None, | |
| idempotency_key: Optional[str] = None, | |
| ) -> BatchScrapeResponse: | |
| """ | |
| Start a batch scrape job for multiple URLs. | |
| Args: | |
| client: HTTP client instance | |
| urls: List of URLs to scrape | |
| options: Scraping options | |
| Returns: | |
| BatchScrapeResponse containing job information | |
| Raises: | |
| FirecrawlError: If the batch scrape operation fails to start | |
| """ | |
| # Prepare request data | |
| request_data = prepare_batch_scrape_request( | |
| urls, | |
| options=options, | |
| webhook=webhook, | |
| append_to_id=append_to_id, | |
| ignore_invalid_urls=ignore_invalid_urls, | |
| max_concurrency=max_concurrency, | |
| zero_data_retention=zero_data_retention, | |
| integration=integration, | |
| ) | |
| # Make the API request | |
| headers = client._prepare_headers(idempotency_key) # type: ignore[attr-defined] | |
| response = client.post("/v2/batch/scrape", request_data, headers=headers) | |
| # Handle errors | |
| if not response.ok: | |
| handle_response_error(response, "start batch scrape") | |
| # Parse response | |
| body = response.json() | |
| if not body.get("success"): | |
| raise Exception(body.get("error", "Unknown error occurred")) | |
| return BatchScrapeResponse( | |
| id=body.get("id"), | |
| url=body.get("url"), | |
| invalid_urls=body.get("invalidURLs") or None, | |
| ) | |
| def get_batch_scrape_status( | |
| client: HttpClient, | |
| job_id: str, | |
| pagination_config: Optional[PaginationConfig] = None | |
| ) -> BatchScrapeJob: | |
| """ | |
| Get the status of a batch scrape job. | |
| Args: | |
| client: HTTP client instance | |
| job_id: ID of the batch scrape job | |
| pagination_config: Optional configuration for pagination behavior | |
| Returns: | |
| BatchScrapeJob containing job status and data | |
| Raises: | |
| FirecrawlError: If the status check fails | |
| """ | |
| # Make the API request | |
| response = client.get(f"/v2/batch/scrape/{job_id}") | |
| # Handle errors | |
| if not response.ok: | |
| handle_response_error(response, "get batch scrape status") | |
| # Parse response | |
| body = response.json() | |
| payload = _parse_batch_scrape_status_response(body) | |
| documents = payload["data"] | |
| # Handle pagination if requested | |
| auto_paginate = pagination_config.auto_paginate if pagination_config else True | |
| if auto_paginate and payload["next"]: | |
| documents = _fetch_all_batch_pages( | |
| client, | |
| payload["next"], | |
| documents, | |
| pagination_config | |
| ) | |
| return BatchScrapeJob( | |
| status=payload["status"], | |
| completed=payload["completed"], | |
| total=payload["total"], | |
| credits_used=payload["credits_used"], | |
| expires_at=payload["expires_at"], | |
| next=payload["next"] if not auto_paginate else None, | |
| data=documents, | |
| ) | |
| def get_batch_scrape_status_page( | |
| client: HttpClient, | |
| next_url: str, | |
| *, | |
| request_timeout: Optional[float] = None, | |
| ) -> BatchScrapeJob: | |
| """ | |
| Fetch a single page of batch scrape results using the provided next URL. | |
| Args: | |
| client: HTTP client instance | |
| next_url: Opaque next URL from a prior batch scrape status response | |
| request_timeout: Timeout (in seconds) for the HTTP request | |
| Returns: | |
| BatchScrapeJob with the page data and next URL (if any) | |
| Raises: | |
| Exception: If the request fails or returns an error response | |
| """ | |
| response = client.get(next_url, timeout=request_timeout) | |
| if not response.ok: | |
| handle_response_error(response, "get batch scrape status page") | |
| body = response.json() | |
| payload = _parse_batch_scrape_status_response(body) | |
| return BatchScrapeJob( | |
| status=payload["status"], | |
| completed=payload["completed"], | |
| total=payload["total"], | |
| credits_used=payload["credits_used"], | |
| expires_at=payload["expires_at"], | |
| next=payload["next"], | |
| data=payload["data"], | |
| ) | |
| def _fetch_all_batch_pages( | |
| client: HttpClient, | |
| next_url: str, | |
| initial_documents: List[Document], | |
| pagination_config: Optional[PaginationConfig] = None | |
| ) -> List[Document]: | |
| """ | |
| Fetch all pages of batch scrape results. | |
| Args: | |
| client: HTTP client instance | |
| next_url: URL for the next page | |
| initial_documents: Documents from the first page | |
| pagination_config: Optional configuration for pagination limits | |
| Returns: | |
| List of all documents from all pages | |
| """ | |
| documents = initial_documents.copy() | |
| current_url = next_url | |
| page_count = 0 | |
| # Apply pagination limits | |
| max_pages = pagination_config.max_pages if pagination_config else None | |
| max_results = pagination_config.max_results if pagination_config else None | |
| max_wait_time = pagination_config.max_wait_time if pagination_config else None | |
| start_time = time.monotonic() | |
| while current_url: | |
| # Check pagination limits (treat 0 as a valid limit) | |
| if (max_pages is not None) and page_count >= max_pages: | |
| break | |
| if (max_wait_time is not None) and (time.monotonic() - start_time) > max_wait_time: | |
| break | |
| # Fetch next page | |
| response = client.get(current_url) | |
| if not response.ok: | |
| # Log error but continue with what we have | |
| import logging | |
| logger = logging.getLogger("firecrawl") | |
| logger.warning("Failed to fetch next page", extra={"status_code": response.status_code}) | |
| break | |
| page_data = response.json() | |
| try: | |
| page_payload = _parse_batch_scrape_status_response(page_data) | |
| except Exception: | |
| break | |
| # Add documents from this page | |
| for document in page_payload["data"]: | |
| # Check max_results limit | |
| if max_results is not None and len(documents) >= max_results: | |
| break | |
| documents.append(document) | |
| # Check if we hit max_results limit after adding all docs from this page | |
| if max_results is not None and len(documents) >= max_results: | |
| break | |
| # Get next URL | |
| current_url = page_payload["next"] | |
| page_count += 1 | |
| return documents | |
| def cancel_batch_scrape( | |
| client: HttpClient, | |
| job_id: str | |
| ) -> bool: | |
| """ | |
| Cancel a running batch scrape job. | |
| Args: | |
| client: HTTP client instance | |
| job_id: ID of the batch scrape job to cancel | |
| Returns: | |
| BatchScrapeStatusResponse with updated status | |
| Raises: | |
| FirecrawlError: If the cancellation fails | |
| """ | |
| # Make the API request | |
| response = client.delete(f"/v2/batch/scrape/{job_id}") | |
| # Handle errors | |
| if not response.ok: | |
| handle_response_error(response, "cancel batch scrape") | |
| # Parse response | |
| body = response.json() | |
| return body.get("status") == "cancelled" | |
| def wait_for_batch_completion( | |
| client: HttpClient, | |
| job_id: str, | |
| poll_interval: int = 2, | |
| timeout: Optional[int] = None | |
| ) -> BatchScrapeJob: | |
| """ | |
| Wait for a batch scrape job to complete, polling for status updates. | |
| Args: | |
| client: HTTP client instance | |
| job_id: ID of the batch scrape job | |
| poll_interval: Seconds between status checks | |
| timeout: Maximum seconds to wait (None for no timeout) | |
| Returns: | |
| BatchScrapeStatusResponse when job completes | |
| Raises: | |
| FirecrawlError: If the job fails or timeout is reached | |
| TimeoutError: If timeout is reached | |
| """ | |
| start_time = time.monotonic() | |
| while True: | |
| status_job = get_batch_scrape_status(client, job_id) | |
| # Check if job is complete | |
| if status_job.status in ["completed", "failed", "cancelled"]: | |
| return status_job | |
| # Check timeout | |
| if timeout and (time.monotonic() - start_time) > timeout: | |
| raise TimeoutError(f"Batch scrape job {job_id} did not complete within {timeout} seconds") | |
| # Wait before next poll | |
| time.sleep(poll_interval) | |
| def batch_scrape( | |
| client: HttpClient, | |
| urls: List[str], | |
| *, | |
| options: Optional[ScrapeOptions] = None, | |
| webhook: Optional[Union[str, WebhookConfig]] = None, | |
| append_to_id: Optional[str] = None, | |
| ignore_invalid_urls: Optional[bool] = None, | |
| max_concurrency: Optional[int] = None, | |
| zero_data_retention: Optional[bool] = None, | |
| integration: Optional[str] = None, | |
| idempotency_key: Optional[str] = None, | |
| poll_interval: int = 2, | |
| timeout: Optional[int] = None | |
| ) -> BatchScrapeJob: | |
| """ | |
| Start a batch scrape job and wait for it to complete. | |
| Args: | |
| client: HTTP client instance | |
| urls: List of URLs to scrape | |
| options: Scraping options | |
| poll_interval: Seconds between status checks | |
| timeout: Maximum seconds to wait (None for no timeout) | |
| Returns: | |
| BatchScrapeStatusResponse when job completes | |
| Raises: | |
| FirecrawlError: If the batch scrape fails to start or complete | |
| TimeoutError: If timeout is reached | |
| """ | |
| # Start the batch scrape | |
| start = start_batch_scrape( | |
| client, | |
| urls, | |
| options=options, | |
| webhook=webhook, | |
| append_to_id=append_to_id, | |
| ignore_invalid_urls=ignore_invalid_urls, | |
| max_concurrency=max_concurrency, | |
| zero_data_retention=zero_data_retention, | |
| integration=integration, | |
| idempotency_key=idempotency_key, | |
| ) | |
| job_id = start.id | |
| # Wait for completion | |
| return wait_for_batch_completion( | |
| client, job_id, poll_interval, timeout | |
| ) | |
| def validate_batch_urls(urls: List[str]) -> List[str]: | |
| """ | |
| Validate and normalize a list of URLs for batch scraping. | |
| Args: | |
| urls: List of URLs to validate | |
| Returns: | |
| Validated list of URLs | |
| Raises: | |
| ValueError: If URLs are invalid | |
| """ | |
| if not urls: | |
| raise ValueError("URLs list cannot be empty") | |
| validated_urls = [] | |
| for url in urls: | |
| if not url or not isinstance(url, str): | |
| raise ValueError(f"Invalid URL: {url}") | |
| # Basic URL validation | |
| if not (url.startswith("http://") or url.startswith("https://")): | |
| raise ValueError(f"URL must start with http:// or https://: {url}") | |
| validated_urls.append(url.strip()) | |
| return validated_urls | |
| def prepare_batch_scrape_request( | |
| urls: List[str], | |
| *, | |
| options: Optional[ScrapeOptions] = None, | |
| webhook: Optional[Union[str, WebhookConfig]] = None, | |
| append_to_id: Optional[str] = None, | |
| ignore_invalid_urls: Optional[bool] = None, | |
| max_concurrency: Optional[int] = None, | |
| zero_data_retention: Optional[bool] = None, | |
| integration: Optional[str] = None, | |
| ) -> dict: | |
| """ | |
| Prepare a batch scrape request payload. | |
| Args: | |
| urls: List of URLs to scrape | |
| options: Scraping options | |
| Returns: | |
| Request payload dictionary | |
| """ | |
| validated_urls = validate_batch_urls(urls) | |
| request_data: Dict[str, Any] = {"urls": validated_urls} | |
| # Flatten scrape options at the top level (v2 behavior) | |
| if options: | |
| scrape_data = prepare_scrape_options(options) | |
| if scrape_data: | |
| request_data.update(scrape_data) | |
| # Batch-specific fields | |
| if webhook is not None: | |
| if isinstance(webhook, str): | |
| request_data["webhook"] = webhook | |
| else: | |
| request_data["webhook"] = webhook.model_dump(exclude_none=True) | |
| if append_to_id is not None: | |
| request_data["appendToId"] = append_to_id | |
| if ignore_invalid_urls is not None: | |
| request_data["ignoreInvalidURLs"] = ignore_invalid_urls | |
| if max_concurrency is not None: | |
| request_data["maxConcurrency"] = max_concurrency | |
| if zero_data_retention is not None: | |
| request_data["zeroDataRetention"] = zero_data_retention | |
| if integration is not None: | |
| request_data["integration"] = str(integration).strip() | |
| return request_data | |
| def chunk_urls(urls: List[str], chunk_size: int = 100) -> List[List[str]]: | |
| """ | |
| Split a large list of URLs into smaller chunks for batch processing. | |
| Args: | |
| urls: List of URLs to chunk | |
| chunk_size: Maximum size of each chunk | |
| Returns: | |
| List of URL chunks | |
| """ | |
| chunks = [] | |
| for i in range(0, len(urls), chunk_size): | |
| chunks.append(urls[i:i + chunk_size]) | |
| return chunks | |
| def process_large_batch( | |
| client: HttpClient, | |
| urls: List[str], | |
| options: Optional[ScrapeOptions] = None, | |
| chunk_size: int = 100, | |
| poll_interval: int = 2, | |
| timeout: Optional[int] = None | |
| ) -> List[Document]: | |
| """ | |
| Process a large batch of URLs by splitting into smaller chunks. | |
| Args: | |
| client: HTTP client instance | |
| urls: List of URLs to scrape | |
| options: Scraping options | |
| chunk_size: Size of each batch chunk | |
| poll_interval: Seconds between status checks | |
| timeout: Maximum seconds to wait per chunk | |
| Returns: | |
| List of all scraped documents | |
| Raises: | |
| FirecrawlError: If any chunk fails | |
| """ | |
| url_chunks = chunk_urls(urls, chunk_size) | |
| all_documents = [] | |
| completed_chunks = 0 | |
| for chunk in url_chunks: | |
| # Process this chunk | |
| result = batch_scrape( | |
| client, | |
| chunk, | |
| options=options, | |
| poll_interval=poll_interval, | |
| timeout=timeout, | |
| ) | |
| # Add documents from this chunk | |
| if result.data: | |
| all_documents.extend(result.data) | |
| completed_chunks += 1 | |
| return all_documents | |
| def get_batch_scrape_errors(client: HttpClient, job_id: str) -> CrawlErrorsResponse: | |
| """ | |
| Get errors for a batch scrape job. | |
| Args: | |
| client: HTTP client instance | |
| job_id: ID of the batch scrape job | |
| Returns: | |
| CrawlErrorsResponse with errors and robots-blocked URLs | |
| """ | |
| response = client.get(f"/v2/batch/scrape/{job_id}/errors") | |
| if not response.ok: | |
| handle_response_error(response, "get batch scrape errors") | |
| body = response.json() | |
| payload = body.get("data", body) | |
| normalized = { | |
| "errors": payload.get("errors", []), | |
| "robots_blocked": payload.get("robotsBlocked", payload.get("robots_blocked", [])), | |
| } | |
| return CrawlErrorsResponse(**normalized) | |