Spaces:
Sleeping
Sleeping
File size: 10,370 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 | """
Async WebSocket watcher with async iterator interface for v2 jobs.
Usage:
async for snapshot in AsyncWatcher(client, job_id, kind="crawl"):
print(snapshot.status)
"""
import asyncio
import inspect
import json
import time
from typing import AsyncIterator, Dict, List, Literal, Optional
import websockets
from websockets.exceptions import ConnectionClosed, ConnectionClosedOK, ConnectionClosedError
from .types import BatchScrapeJob, CrawlJob, Document
from .utils.normalize import normalize_document_input
JobKind = Literal["crawl", "batch"]
class AsyncWatcher:
def __init__(
self,
client: object,
job_id: str,
*,
kind: JobKind = "crawl",
poll_interval: int = 2,
timeout: Optional[int] = None,
) -> None:
self._client = client
self._job_id = job_id
self._kind = kind
self._timeout = timeout
self._poll_interval: float = max(0.0, float(poll_interval)) # Guard against negative values
http_client = getattr(client, "http_client", None)
if http_client is not None:
self._api_url = getattr(http_client, "api_url", None)
self._api_key = getattr(http_client, "api_key", None)
else:
# Allow passing the top-level Firecrawl client directly
self._api_url = getattr(client, "api_url", None)
self._api_key = getattr(client, "api_key", None)
self._status: str = "scraping"
self._data: List[Dict] = []
def __aiter__(self) -> AsyncIterator[object]:
return self._iterate()
def _build_ws_url(self) -> str:
if not self._api_url:
raise ValueError("API URL is required for WebSocket watcher")
ws_base = self._api_url.replace("https://", "wss://").replace("http://", "ws://", 1)
if self._kind == "crawl":
return f"{ws_base}/v2/crawl/{self._job_id}"
return f"{ws_base}/v2/batch/scrape/{self._job_id}"
async def _iterate(self) -> AsyncIterator[object]:
uri = self._build_ws_url()
headers_list = []
if self._api_key:
headers_list.append(("Authorization", f"Bearer {self._api_key}"))
# Attempt to establish WS; on failure, fall back to HTTP polling immediately
try:
async with websockets.connect(uri, max_size=None, additional_headers=headers_list) as websocket:
deadline = asyncio.get_event_loop().time() + self._timeout if self._timeout else None
# Pre-yield a snapshot if available to ensure progress is visible
try:
pre = await self._fetch_job_status()
yield pre
if pre.status in ("completed", "failed", "cancelled"):
return
except Exception:
pass
while True:
try:
if deadline is not None:
remaining = max(0.0, deadline - asyncio.get_event_loop().time())
timeout = min(self._poll_interval, remaining) if remaining > 0 else 0.0
else:
timeout = self._poll_interval
msg = await asyncio.wait_for(websocket.recv(), timeout=timeout)
except asyncio.TimeoutError:
# Quiet period: poll HTTP once
job = await self._safe_fetch()
if job is not None:
yield job
if job.status in ("completed", "failed", "cancelled"):
return
if deadline is not None and asyncio.get_event_loop().time() >= deadline:
return
continue
except (ConnectionClosedOK, ConnectionClosed, ConnectionClosedError):
# Graceful/abrupt close: poll HTTP until terminal (bounded by timeout)
deadline = time.time() + (self._timeout or 30)
while True:
try:
job = await self._fetch_job_status()
yield job
if job.status in ("completed", "failed", "cancelled"):
return
except Exception:
return
if time.time() >= deadline:
return
await asyncio.sleep(1)
try:
body = json.loads(msg)
except Exception:
continue
msg_type = body.get("type")
if msg_type == "error":
self._status = "failed"
# Yield a terminal snapshot
if self._kind == "crawl":
yield CrawlJob(status="failed", completed=0, total=0, credits_used=0, expires_at=None, next=None, data=[])
else:
yield BatchScrapeJob(status="failed", completed=0, total=0, credits_used=0, expires_at=None, next=None, data=[])
return
elif msg_type == "catchup":
d = body.get("data", {})
self._status = d.get("status", self._status)
docs_in = d.get("data", []) or []
self._data.extend(docs_in)
# Fall through to emit a snapshot below
elif msg_type == "document":
doc = body.get("data")
if isinstance(doc, dict):
self._data.append(doc)
# Fall through to emit a snapshot below
elif msg_type == "done":
self._status = "completed"
raw_payload = body.get("data", {}) or {}
docs_in = raw_payload.get("data", []) or []
if isinstance(docs_in, list) and docs_in:
for doc in docs_in:
if isinstance(doc, dict):
self._data.append(doc)
# Emit final snapshot then end
yield self._make_snapshot(status="completed", payload=raw_payload, docs_override=self._data)
return
# Generic snapshot emit for status messages and periodic progress
payload = body.get("data", body)
status_str = payload.get("status", body.get("status", self._status))
snapshot = self._make_snapshot(status=status_str, payload=payload)
yield snapshot
if status_str in ("completed", "failed", "cancelled"):
return
except Exception:
# WS connect failure: fallback to HTTP polling loop until terminal/timeout
deadline = time.time() + (self._timeout or 30)
while True:
try:
job = await self._fetch_job_status()
yield job
if job.status in ("completed", "failed", "cancelled"):
return
except Exception:
return
if time.time() >= deadline:
return
await asyncio.sleep(1)
async def _fetch_job_status(self):
if self._kind == "crawl":
return await self._call_status_method("get_crawl_status")
return await self._call_status_method("get_batch_scrape_status")
async def _call_status_method(self, method_name: str):
# Try on client directly
meth = getattr(self._client, method_name, None)
if meth is not None:
try:
result = meth(self._job_id)
except TypeError:
result = None
if result is not None:
if inspect.isawaitable(result):
return await result
return result
# Fallback: if we couldn't call directly, try to_thread
return await asyncio.to_thread(meth, self._job_id)
# Try on client.v2
v2 = getattr(self._client, "v2", None)
if v2 is not None:
meth = getattr(v2, method_name, None)
if meth is not None:
try:
result = meth(self._job_id)
except TypeError:
result = None
if result is not None:
if inspect.isawaitable(result):
return await result
return result
return await asyncio.to_thread(meth, self._job_id)
raise RuntimeError(f"Client does not expose {method_name}")
async def _safe_fetch(self):
try:
return await self._fetch_job_status()
except Exception:
return None
def _make_snapshot(self, *, status: str, payload: Dict, docs_override: Optional[List[Dict]] = None):
docs = []
source_docs = docs_override if docs_override is not None else payload.get("data", []) or []
for doc in source_docs:
if isinstance(doc, dict):
d = normalize_document_input(doc)
docs.append(Document(**d))
if self._kind == "crawl":
return CrawlJob(
status=status,
completed=payload.get("completed", 0),
total=payload.get("total", 0),
credits_used=payload.get("creditsUsed", 0),
expires_at=payload.get("expiresAt"),
next=payload.get("next"),
data=docs,
)
return BatchScrapeJob(
status=status,
completed=payload.get("completed", 0),
total=payload.get("total", 0),
credits_used=payload.get("creditsUsed"),
expires_at=payload.get("expiresAt"),
next=payload.get("next"),
data=docs,
)
|