Spaces:
Sleeping
Sleeping
File size: 4,146 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 | from firecrawl.client import Firecrawl
# Initialize Firecrawl client
firecrawl = Firecrawl(api_key="YOUR_API_KEY")
# =============================================================================
# SCRAPE EXAMPLES
# =============================================================================
# Basic scraping - Get markdown content
print("=== Basic Scrape (Markdown) ===")
# doc = firecrawl.scrape("https://firecrawl.dev", formats=["markdown"])
# print(doc.markdown)
# Scraping with location settings
print("\n=== Scrape with Location Settings ===")
# doc = firecrawl.scrape('https://docs.firecrawl.dev',
# formats=['markdown'],
# location={
# 'country': 'US',
# 'languages': ['en']
# }
# )
# print(doc)
# Scraping with JSON extraction using Pydantic schema
print("\n=== Scrape with JSON Schema (Pydantic) ===")
# from pydantic import BaseModel
#
# class JsonSchema(BaseModel):
# company_mission: str
# supports_sso: bool
# is_open_source: bool
# is_in_yc: bool
#
# result = firecrawl.scrape(
# 'https://firecrawl.dev',
# formats=[{
# "type": "json",
# "schema": JsonSchema
# }],
# only_main_content=False,
# max_age=120000000
# )
# print(result)
# Scraping with JSON extraction using prompt
print("\n=== Scrape with JSON Prompt ===")
# result = firecrawl.scrape(
# 'https://firecrawl.dev',
# formats=[{
# "type": "json",
# "prompt": "Extract the company mission from the page."
# }],
# only_main_content=False,
# timeout=120000
# )
# print(result.json["companyMission"])
# Advanced scraping with multiple formats and options
print("\n=== Advanced Scrape (Multiple Formats + Options) ===")
# response = firecrawl.scrape('https://docs.firecrawl.dev',
# formats=[
# 'markdown',
# { 'type': 'json', 'schema': { 'type': 'object', 'properties': { 'title': { 'type': 'string' } } } }
# ],
# proxy='auto',
# max_age=600000,
# only_main_content=True
# )
# print(response)
# =============================================================================
# CRAWL EXAMPLES
# =============================================================================
# Basic crawling
print("\n=== Basic Crawl ===")
# docs = firecrawl.crawl(url="https://docs.firecrawl.dev", limit=10)
# print(docs)
# =============================================================================
# BATCH SCRAPE EXAMPLES
# =============================================================================
# Batch scraping multiple URLs
print("\n=== Batch Scrape ===")
# from firecrawl import Firecrawl
#
# firecrawl = Firecrawl(api_key="fc-YOUR-API-KEY")
#
# job = firecrawl.batch_scrape([
# "https://firecrawl.dev",
# "https://docs.firecrawl.dev",
# ], formats=["markdown"], poll_interval=2, wait_timeout=120)
# print(job)
# =============================================================================
# SEARCH EXAMPLES
# =============================================================================
# Search functionality
print("\n=== Search ===")
# search = firecrawl.search(query="firecrawl", sources=[{"type": "web"}], limit=5)
# print(search.web[0].title)
# =============================================================================
# MAP EXAMPLES
# =============================================================================
# Website mapping
print("\n=== Map Website ===")
# res = firecrawl.map(url="https://firecrawl.dev", limit=50, sitemap="include", search="price")
# print(res.links[0].url)
# print(res.links[1].url)
# print(res.links[2].url)
# print(res.links[3].url)
# =============================================================================
# EXTRACT EXAMPLES
# =============================================================================
# Data extraction with schema
print("\n=== Extract with Schema ===")
# schema = {
# "type": "object",
# "properties": {"title": {"type": "string"}},
# "required": ["title"],
# }
#
# res = firecrawl.extract(
# urls=["https://docs.firecrawl.dev"],
# prompt="Extract the page title",
# schema=schema,
# )
# print(res.data["title"]) |