File size: 6,253 Bytes
097eaf0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
"""
Scraping functionality for Firecrawl v2 API.
"""

from typing import Optional, Dict, Any, Literal
from ..types import (
    ScrapeOptions,
    Document,
    BrowserExecuteResponse,
    BrowserDeleteResponse,
)
from ..utils.normalize import normalize_document_input
from ..utils import HttpClient, handle_response_error, prepare_scrape_options, validate_scrape_options


def _prepare_scrape_request(url: str, options: Optional[ScrapeOptions] = None) -> Dict[str, Any]:
    """
    Prepare a scrape request payload for v2 API.
    
    Args:
        url: URL to scrape
        options: ScrapeOptions (snake_case) to convert and include
        
    Returns:
        Request payload dictionary with camelCase fields
    """
    if not url or not url.strip():
        raise ValueError("URL cannot be empty")

    request_data: Dict[str, Any] = {"url": url.strip()}

    if options is not None:
        validated = validate_scrape_options(options)
        if validated is not None:
            opts = prepare_scrape_options(validated)
            if opts:
                request_data.update(opts)

    return request_data

def scrape(client: HttpClient, url: str, options: Optional[ScrapeOptions] = None) -> Document:
    """
    Scrape a single URL and return the document.
    
    The v2 API returns: { success: boolean, data: Document }
    We surface just the Document to callers.
    
    Args:
        client: HTTP client instance
        url: URL to scrape
        options: Scraping options (snake_case)
        
    Returns:
        Document
    """
    payload = _prepare_scrape_request(url, options)

    response = client.post("/v2/scrape", payload)

    if not response.ok:
        handle_response_error(response, "scrape")

    body = response.json()
    if not body.get("success"):
        raise Exception(body.get("error", "Unknown error occurred"))

    document_data = body.get("data", {})
    normalized = normalize_document_input(document_data)
    return Document(**normalized)


def interact(
    client: HttpClient,
    job_id: str,
    code: Optional[str] = None,
    *,
    prompt: Optional[str] = None,
    language: Literal["python", "node", "bash"] = "node",
    timeout: Optional[int] = None,
    origin: Optional[str] = None,
) -> BrowserExecuteResponse:
    """
    Interact with the scrape-bound browser session for a scrape job.

    Either ``code`` or ``prompt`` must be provided.  When ``prompt`` is given
    the server runs an AI agent that translates the natural-language instruction
    into browser actions.

    Args:
        client: HTTP client instance
        job_id: Scrape job ID
        code: Code to execute (optional if prompt is provided)
        prompt: Natural-language instruction for the browser agent (optional if code is provided)
        language: Programming language ("python", "node", or "bash")
        timeout: Execution timeout in seconds (1-300)
        origin: Optional request origin tag

    Returns:
        BrowserExecuteResponse with execution output
    """
    if not job_id or not job_id.strip():
        raise ValueError("Job ID cannot be empty")
    has_code = code and code.strip()
    has_prompt = prompt and prompt.strip()
    if not has_code and not has_prompt:
        raise ValueError("Either 'code' or 'prompt' must be provided")

    body: Dict[str, Any] = {
        "language": language,
    }
    if has_code:
        body["code"] = code
    if has_prompt:
        body["prompt"] = prompt
    if timeout is not None:
        body["timeout"] = timeout
    if origin is not None:
        body["origin"] = origin

    response = client.post(f"/v2/scrape/{job_id}/interact", body)
    if not response.ok:
        handle_response_error(response, "interact with scrape browser")

    payload = response.json()
    if not payload.get("success"):
        raise Exception(payload.get("error", "Unknown error occurred"))

    normalized = dict(payload)
    if "exitCode" in normalized and "exit_code" not in normalized:
        normalized["exit_code"] = normalized["exitCode"]
    if "liveViewUrl" in normalized and "live_view_url" not in normalized:
        normalized["live_view_url"] = normalized["liveViewUrl"]
    if "interactiveLiveViewUrl" in normalized and "interactive_live_view_url" not in normalized:
        normalized["interactive_live_view_url"] = normalized["interactiveLiveViewUrl"]
    return BrowserExecuteResponse(**normalized)


def stop_interaction(
    client: HttpClient,
    job_id: str,
) -> BrowserDeleteResponse:
    """
    Stop the interaction session for a scrape job.

    Args:
        client: HTTP client instance
        job_id: Scrape job ID

    Returns:
        BrowserDeleteResponse
    """
    if not job_id or not job_id.strip():
        raise ValueError("Job ID cannot be empty")

    response = client.delete(f"/v2/scrape/{job_id}/interact")
    if not response.ok:
        handle_response_error(response, "stop interaction")

    payload = response.json()
    normalized = dict(payload)
    if "sessionDurationMs" in normalized and "session_duration_ms" not in normalized:
        normalized["session_duration_ms"] = normalized["sessionDurationMs"]
    if "creditsBilled" in normalized and "credits_billed" not in normalized:
        normalized["credits_billed"] = normalized["creditsBilled"]

    return BrowserDeleteResponse(**normalized)


def stop_interactive_browser(
    client: HttpClient,
    job_id: str,
) -> BrowserDeleteResponse:
    """Deprecated alias for stop_interaction()."""
    return stop_interaction(client, job_id)


def scrape_execute(
    client: HttpClient,
    job_id: str,
    code: Optional[str] = None,
    *,
    prompt: Optional[str] = None,
    language: Literal["python", "node", "bash"] = "node",
    timeout: Optional[int] = None,
    origin: Optional[str] = None,
) -> BrowserExecuteResponse:
    """Deprecated alias for interact()."""
    return interact(
        client,
        job_id,
        code,
        prompt=prompt,
        language=language,
        timeout=timeout,
        origin=origin,
    )


def delete_scrape_browser(
    client: HttpClient,
    job_id: str,
) -> BrowserDeleteResponse:
    """Deprecated alias for stop_interaction()."""
    return stop_interaction(client, job_id)