Spaces:
Sleeping
Sleeping
File size: 6,253 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 | """
Scraping functionality for Firecrawl v2 API.
"""
from typing import Optional, Dict, Any, Literal
from ..types import (
ScrapeOptions,
Document,
BrowserExecuteResponse,
BrowserDeleteResponse,
)
from ..utils.normalize import normalize_document_input
from ..utils import HttpClient, handle_response_error, prepare_scrape_options, validate_scrape_options
def _prepare_scrape_request(url: str, options: Optional[ScrapeOptions] = None) -> Dict[str, Any]:
"""
Prepare a scrape request payload for v2 API.
Args:
url: URL to scrape
options: ScrapeOptions (snake_case) to convert and include
Returns:
Request payload dictionary with camelCase fields
"""
if not url or not url.strip():
raise ValueError("URL cannot be empty")
request_data: Dict[str, Any] = {"url": url.strip()}
if options is not None:
validated = validate_scrape_options(options)
if validated is not None:
opts = prepare_scrape_options(validated)
if opts:
request_data.update(opts)
return request_data
def scrape(client: HttpClient, url: str, options: Optional[ScrapeOptions] = None) -> Document:
"""
Scrape a single URL and return the document.
The v2 API returns: { success: boolean, data: Document }
We surface just the Document to callers.
Args:
client: HTTP client instance
url: URL to scrape
options: Scraping options (snake_case)
Returns:
Document
"""
payload = _prepare_scrape_request(url, options)
response = client.post("/v2/scrape", payload)
if not response.ok:
handle_response_error(response, "scrape")
body = response.json()
if not body.get("success"):
raise Exception(body.get("error", "Unknown error occurred"))
document_data = body.get("data", {})
normalized = normalize_document_input(document_data)
return Document(**normalized)
def interact(
client: HttpClient,
job_id: str,
code: Optional[str] = None,
*,
prompt: Optional[str] = None,
language: Literal["python", "node", "bash"] = "node",
timeout: Optional[int] = None,
origin: Optional[str] = None,
) -> BrowserExecuteResponse:
"""
Interact with the scrape-bound browser session for a scrape job.
Either ``code`` or ``prompt`` must be provided. When ``prompt`` is given
the server runs an AI agent that translates the natural-language instruction
into browser actions.
Args:
client: HTTP client instance
job_id: Scrape job ID
code: Code to execute (optional if prompt is provided)
prompt: Natural-language instruction for the browser agent (optional if code is provided)
language: Programming language ("python", "node", or "bash")
timeout: Execution timeout in seconds (1-300)
origin: Optional request origin tag
Returns:
BrowserExecuteResponse with execution output
"""
if not job_id or not job_id.strip():
raise ValueError("Job ID cannot be empty")
has_code = code and code.strip()
has_prompt = prompt and prompt.strip()
if not has_code and not has_prompt:
raise ValueError("Either 'code' or 'prompt' must be provided")
body: Dict[str, Any] = {
"language": language,
}
if has_code:
body["code"] = code
if has_prompt:
body["prompt"] = prompt
if timeout is not None:
body["timeout"] = timeout
if origin is not None:
body["origin"] = origin
response = client.post(f"/v2/scrape/{job_id}/interact", body)
if not response.ok:
handle_response_error(response, "interact with scrape browser")
payload = response.json()
if not payload.get("success"):
raise Exception(payload.get("error", "Unknown error occurred"))
normalized = dict(payload)
if "exitCode" in normalized and "exit_code" not in normalized:
normalized["exit_code"] = normalized["exitCode"]
if "liveViewUrl" in normalized and "live_view_url" not in normalized:
normalized["live_view_url"] = normalized["liveViewUrl"]
if "interactiveLiveViewUrl" in normalized and "interactive_live_view_url" not in normalized:
normalized["interactive_live_view_url"] = normalized["interactiveLiveViewUrl"]
return BrowserExecuteResponse(**normalized)
def stop_interaction(
client: HttpClient,
job_id: str,
) -> BrowserDeleteResponse:
"""
Stop the interaction session for a scrape job.
Args:
client: HTTP client instance
job_id: Scrape job ID
Returns:
BrowserDeleteResponse
"""
if not job_id or not job_id.strip():
raise ValueError("Job ID cannot be empty")
response = client.delete(f"/v2/scrape/{job_id}/interact")
if not response.ok:
handle_response_error(response, "stop interaction")
payload = response.json()
normalized = dict(payload)
if "sessionDurationMs" in normalized and "session_duration_ms" not in normalized:
normalized["session_duration_ms"] = normalized["sessionDurationMs"]
if "creditsBilled" in normalized and "credits_billed" not in normalized:
normalized["credits_billed"] = normalized["creditsBilled"]
return BrowserDeleteResponse(**normalized)
def stop_interactive_browser(
client: HttpClient,
job_id: str,
) -> BrowserDeleteResponse:
"""Deprecated alias for stop_interaction()."""
return stop_interaction(client, job_id)
def scrape_execute(
client: HttpClient,
job_id: str,
code: Optional[str] = None,
*,
prompt: Optional[str] = None,
language: Literal["python", "node", "bash"] = "node",
timeout: Optional[int] = None,
origin: Optional[str] = None,
) -> BrowserExecuteResponse:
"""Deprecated alias for interact()."""
return interact(
client,
job_id,
code,
prompt=prompt,
language=language,
timeout=timeout,
origin=origin,
)
def delete_scrape_browser(
client: HttpClient,
job_id: str,
) -> BrowserDeleteResponse:
"""Deprecated alias for stop_interaction()."""
return stop_interaction(client, job_id) |