Spaces:
Configuration error
Configuration error
File size: 1,341 Bytes
10bc2e0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 | """Firecrawl client - scrape, crawl, search, extract (rate-limit aware)."""
import os
import time
import requests
BASE = os.getenv("FIRECRAWL_API_BASE", "https://api.firecrawl.dev")
KEY = os.getenv("FIRECRAWL_API_KEY")
def _h():
if not KEY:
raise RuntimeError("FIRECRAWL_API_KEY not set")
return {"Authorization": f"Bearer {KEY}", "Content-Type": "application/json"}
def scrape(url, formats=("markdown",), **kwargs):
return requests.post(f"{BASE}/v1/scrape", headers=_h(), json={"url": url, "formats": list(formats), **kwargs}, timeout=120).json()
def crawl(url, max_depth=1, limit=50, **kwargs):
"""Firecrawl full-crawl. Respects rate limits (429s) with backoff."""
r = requests.post(f"{BASE}/v1/crawl", headers=_h(), json={"url": url, "maxDepth": max_depth, "limit": limit, **kwargs}, timeout=120)
if r.status_code == 429:
time.sleep(5)
r = requests.post(f"{BASE}/v1/crawl", headers=_h(), json={"url": url, "maxDepth": max_depth, "limit": limit, **kwargs}, timeout=120)
r.raise_for_status()
return r.json()
def crawl_status(job_id: str):
return requests.get(f"{BASE}/v1/crawl/{job_id}", headers=_h(), timeout=60).json()
def search(query, limit=10):
return requests.post(f"{BASE}/v1/search", headers=_h(), json={"query": query, "limit": limit}, timeout=120).json()
|