Spaces:
Configuration error
Configuration error
| """Firecrawl client - scrape, crawl, search, extract (rate-limit aware).""" | |
| import os | |
| import time | |
| import requests | |
| BASE = os.getenv("FIRECRAWL_API_BASE", "https://api.firecrawl.dev") | |
| KEY = os.getenv("FIRECRAWL_API_KEY") | |
| def _h(): | |
| if not KEY: | |
| raise RuntimeError("FIRECRAWL_API_KEY not set") | |
| return {"Authorization": f"Bearer {KEY}", "Content-Type": "application/json"} | |
| def scrape(url, formats=("markdown",), **kwargs): | |
| return requests.post(f"{BASE}/v1/scrape", headers=_h(), json={"url": url, "formats": list(formats), **kwargs}, timeout=120).json() | |
| def crawl(url, max_depth=1, limit=50, **kwargs): | |
| """Firecrawl full-crawl. Respects rate limits (429s) with backoff.""" | |
| r = requests.post(f"{BASE}/v1/crawl", headers=_h(), json={"url": url, "maxDepth": max_depth, "limit": limit, **kwargs}, timeout=120) | |
| if r.status_code == 429: | |
| time.sleep(5) | |
| r = requests.post(f"{BASE}/v1/crawl", headers=_h(), json={"url": url, "maxDepth": max_depth, "limit": limit, **kwargs}, timeout=120) | |
| r.raise_for_status() | |
| return r.json() | |
| def crawl_status(job_id: str): | |
| return requests.get(f"{BASE}/v1/crawl/{job_id}", headers=_h(), timeout=60).json() | |
| def search(query, limit=10): | |
| return requests.post(f"{BASE}/v1/search", headers=_h(), json={"query": query, "limit": limit}, timeout=120).json() | |