File size: 6,898 Bytes
097eaf0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
from typing import Any, Dict, List, Optional
import time
import warnings

from ..types import ExtractResponse, ScrapeOptions
from ..types import AgentOptions
from ..utils.http_client import HttpClient
from ..utils.validation import prepare_scrape_options
from ..utils.error_handler import handle_response_error

_EXTRACT_DEPRECATION_MSG = (
    "The extract endpoint is in maintenance mode and its use is discouraged. "
    "Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor "
    "to find a replacement."
)


def _prepare_extract_request(
    urls: Optional[List[str]],
    *,
    prompt: Optional[str] = None,
    schema: Optional[Dict[str, Any]] = None,
    system_prompt: Optional[str] = None,
    allow_external_links: Optional[bool] = None,
    enable_web_search: Optional[bool] = None,
    show_sources: Optional[bool] = None,
    scrape_options: Optional[ScrapeOptions] = None,
    ignore_invalid_urls: Optional[bool] = None,
    integration: Optional[str] = None,
    agent: Optional[AgentOptions] = None,
) -> Dict[str, Any]:
    body: Dict[str, Any] = {}
    if urls is not None:
        body["urls"] = urls
    if prompt is not None:
        body["prompt"] = prompt
    if schema is not None:
        body["schema"] = schema
    if system_prompt is not None:
        body["systemPrompt"] = system_prompt
    if allow_external_links is not None:
        body["allowExternalLinks"] = allow_external_links
    if enable_web_search is not None:
        body["enableWebSearch"] = enable_web_search
    if show_sources is not None:
        body["showSources"] = show_sources
    if ignore_invalid_urls is not None:
        body["ignoreInvalidURLs"] = ignore_invalid_urls
    if scrape_options is not None:
        prepared = prepare_scrape_options(scrape_options)
        if prepared:
            body["scrapeOptions"] = prepared
    if integration is not None and str(integration).strip():
        body["integration"] = str(integration).strip()
    if agent is not None:
        try:
            body["agent"] = agent.model_dump(exclude_none=True)  # type: ignore[attr-defined]
        except AttributeError:
            body["agent"] = agent  # fallback
    return body


def _normalize_extract_response_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
    out = dict(payload)
    if "expiresAt" in out and "expires_at" not in out:
        out["expires_at"] = out["expiresAt"]
    if "creditsUsed" in out and "credits_used" not in out:
        out["credits_used"] = out["creditsUsed"]
    if "tokensUsed" in out and "tokens_used" not in out:
        out["tokens_used"] = out["tokensUsed"]
    return out


def start_extract(
    client: HttpClient,
    urls: Optional[List[str]],
    *,
    prompt: Optional[str] = None,
    schema: Optional[Dict[str, Any]] = None,
    system_prompt: Optional[str] = None,
    allow_external_links: Optional[bool] = None,
    enable_web_search: Optional[bool] = None,
    show_sources: Optional[bool] = None,
    scrape_options: Optional[ScrapeOptions] = None,
    ignore_invalid_urls: Optional[bool] = None,
    integration: Optional[str] = None,
    agent: Optional[AgentOptions] = None,
) -> ExtractResponse:
    """Start an extract job (non-blocking).

    .. deprecated::
        The extract endpoint is in maintenance mode and its use is discouraged.
        Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
        to find a replacement.
    """
    warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
    body = _prepare_extract_request(
        urls,
        prompt=prompt,
        schema=schema,
        system_prompt=system_prompt,
        allow_external_links=allow_external_links,
        enable_web_search=enable_web_search,
        show_sources=show_sources,
        scrape_options=scrape_options,
        ignore_invalid_urls=ignore_invalid_urls,
        integration=integration,
        agent=agent,
    )
    resp = client.post("/v2/extract", body)
    if not resp.ok:
        handle_response_error(resp, "extract")
    payload = _normalize_extract_response_payload(resp.json())
    return ExtractResponse(**payload)


def get_extract_status(client: HttpClient, job_id: str) -> ExtractResponse:
    """Get the current status of an extract job.

    .. deprecated::
        The extract endpoint is in maintenance mode and its use is discouraged.
        Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
        to find a replacement.
    """
    warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
    resp = client.get(f"/v2/extract/{job_id}")
    if not resp.ok:
        handle_response_error(resp, "extract-status")
    payload = _normalize_extract_response_payload(resp.json())
    return ExtractResponse(**payload)


def wait_extract(
    client: HttpClient,
    job_id: str,
    *,
    poll_interval: int = 2,
    timeout: Optional[int] = None,
) -> ExtractResponse:
    start_ts = time.time()
    while True:
        status = get_extract_status(client, job_id)
        if status.status in ("completed", "failed", "cancelled"):
            return status
        if timeout is not None and (time.time() - start_ts) > timeout:
            return status
        time.sleep(max(1, poll_interval))


def extract(
    client: HttpClient,
    urls: Optional[List[str]],
    *,
    prompt: Optional[str] = None,
    schema: Optional[Dict[str, Any]] = None,
    system_prompt: Optional[str] = None,
    allow_external_links: Optional[bool] = None,
    enable_web_search: Optional[bool] = None,
    show_sources: Optional[bool] = None,
    scrape_options: Optional[ScrapeOptions] = None,
    ignore_invalid_urls: Optional[bool] = None,
    poll_interval: int = 2,
    timeout: Optional[int] = None,
    integration: Optional[str] = None,
    agent: Optional[AgentOptions] = None,
) -> ExtractResponse:
    """Extract structured data and wait until completion.

    .. deprecated::
        The extract endpoint is in maintenance mode and its use is discouraged.
        Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
        to find a replacement.
    """
    warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
    started = start_extract(
        client,
        urls,
        prompt=prompt,
        schema=schema,
        system_prompt=system_prompt,
        allow_external_links=allow_external_links,
        enable_web_search=enable_web_search,
        show_sources=show_sources,
        scrape_options=scrape_options,
        ignore_invalid_urls=ignore_invalid_urls,
        integration=integration,
        agent=agent,
    )
    job_id = getattr(started, "id", None)
    if not job_id:
        return started
    return wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)