Spaces:
Sleeping
Sleeping
File size: 2,118 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | #!/usr/bin/env python3
"""
Minimal websocket watcher examples (sync and async).
Env:
FIRECRAWL_API_KEY
FIRECRAWL_API_URL
"""
import os
import time
import asyncio
from dotenv import load_dotenv
from firecrawl import Firecrawl, Watcher, AsyncWatcher
def example_watcher() -> None:
api_key = os.getenv("FIRECRAWL_API_KEY")
api_url = os.getenv("FIRECRAWL_API_URL")
if not api_key or not api_url:
raise ValueError("FIRECRAWL_API_KEY and FIRECRAWL_API_URL must be set")
client = Firecrawl(api_key=api_key, api_url=api_url)
# Start a small crawl job
job = client.start_crawl("https://docs.firecrawl.dev", limit=2)
events = {"document": 0, "done": 0}
statuses = []
w: Watcher = client.watcher(job.id, kind="crawl", poll_interval=1, timeout=180)
w.add_event_listener("document", lambda d: events.__setitem__("document", events["document"] + 1))
w.add_event_listener("done", lambda d: events.__setitem__("done", events["done"] + 1))
w.add_listener(lambda s: statuses.append(s.status))
w.start()
# Wait until terminal
deadline = time.time() + 180
while time.time() < deadline:
if statuses and statuses[-1] in ("completed", "failed"):
break
time.sleep(1)
w.stop()
print("sync watcher:", {"last_status": statuses[-1] if statuses else None, **events})
async def example_async_watcher() -> None:
"""
Example of using the async watcher.
"""
api_key = os.getenv("FIRECRAWL_API_KEY")
api_url = os.getenv("FIRECRAWL_API_URL")
if not api_key or not api_url:
raise ValueError("FIRECRAWL_API_KEY and FIRECRAWL_API_URL must be set")
client = Firecrawl(api_key=api_key, api_url=api_url)
# Start a small crawl job
job = client.start_crawl("https://docs.firecrawl.dev", limit=2)
async for snapshot in AsyncWatcher(client, job.id, kind="crawl"):
print("async watcher:", snapshot.status, f"docs={len(snapshot.data)}")
def main() -> None:
load_dotenv()
example_watcher()
asyncio.run(example_async_watcher())
if __name__ == "__main__":
main()
|