Spaces:
Sleeping
Sleeping
File size: 6,998 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 | //! Example usage of Firecrawl API.
//!
//! Run with: cargo run --example example
use firecrawl::{
AgentModel, AgentOptions, BatchScrapeOptions, Client, CrawlOptions, Format, MapOptions,
ScrapeOptions, SearchOptions, SitemapMode,
};
use serde::Deserialize;
use serde_json::json;
#[tokio::main]
async fn main() {
// Initialize the Client with the API key
let client = Client::new("fc-YOUR-API-KEY").expect("Failed to initialize Client");
// Or, connect to a self-hosted instance:
// let client = Client::new_selfhosted("http://localhost:3002", None::<&str>)
// .expect("Failed to initialize Client");
// Example 1: Simple Scrape
println!("=== Example 1: Simple Scrape ===");
let result = client.scrape("https://example.com", None).await;
match result {
Ok(doc) => println!("Markdown: {:?}", doc.markdown),
Err(e) => eprintln!("Scrape failed: {}", e),
}
// Example 2: Scrape with Options
println!("\n=== Example 2: Scrape with Options ===");
let options = ScrapeOptions {
formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),
only_main_content: Some(true),
..Default::default()
};
let result = client.scrape("https://example.com", options).await;
match result {
Ok(doc) => {
println!("Links: {:?}", doc.links);
}
Err(e) => eprintln!("Scrape with options failed: {}", e),
}
// Example 3: Scrape with JSON Schema Extraction
println!("\n=== Example 3: JSON Schema Extraction ===");
let schema = json!({
"type": "object",
"properties": {
"title": { "type": "string" },
"description": { "type": "string" }
}
});
let result = client
.scrape_with_schema(
"https://example.com",
schema,
Some("Extract title and description"),
)
.await;
match result {
Ok(data) => println!(
"Extracted: {}",
serde_json::to_string_pretty(&data).unwrap()
),
Err(e) => eprintln!("Schema extraction failed: {}", e),
}
// Example 4: Search
println!("\n=== Example 4: Search ===");
let options = SearchOptions {
limit: Some(5),
..Default::default()
};
let result = client.search("rust programming", options).await;
match result {
Ok(response) => {
if let Some(web) = response.data.web {
for item in web {
match item {
firecrawl::SearchResultOrDocument::WebResult(r) => {
println!("Result: {} - {}", r.url, r.title.unwrap_or_default());
}
firecrawl::SearchResultOrDocument::Document(d) => {
if let Some(meta) = d.metadata {
println!("Document: {:?}", meta.title);
}
}
}
}
}
}
Err(e) => eprintln!("Search failed: {}", e),
}
// Example 5: Map a Website
println!("\n=== Example 5: Map ===");
let options = MapOptions {
sitemap: Some(SitemapMode::Include),
limit: Some(20),
..Default::default()
};
let result = client.map("https://example.com", options).await;
match result {
Ok(response) => {
println!("Found {} links", response.links.len());
for link in response.links.iter().take(5) {
println!(" - {}", link.url);
}
}
Err(e) => eprintln!("Map failed: {}", e),
}
// Example 6: Crawl a Website
println!("\n=== Example 6: Crawl ===");
let options = CrawlOptions {
limit: Some(5),
sitemap: Some(SitemapMode::Include),
poll_interval: Some(3000),
..Default::default()
};
let result = client.crawl("https://example.com", options).await;
match result {
Ok(job) => {
println!("Crawled {} pages", job.data.len());
for doc in job.data.iter().take(3) {
if let Some(meta) = &doc.metadata {
println!(" - {:?}", meta.source_url);
}
}
}
Err(e) => eprintln!("Crawl failed: {}", e),
}
// Example 7: Batch Scrape
println!("\n=== Example 7: Batch Scrape ===");
let urls = vec![
"https://example.com".to_string(),
"https://example.org".to_string(),
];
let options = BatchScrapeOptions {
options: Some(ScrapeOptions {
formats: Some(vec![Format::Markdown]),
..Default::default()
}),
poll_interval: Some(2000),
..Default::default()
};
let result = client.batch_scrape(urls, options).await;
match result {
Ok(job) => {
println!("Batch scraped {} pages", job.data.len());
}
Err(e) => eprintln!("Batch scrape failed: {}", e),
}
// Example 8: Agent (Autonomous Web Browsing)
println!("\n=== Example 8: Agent ===");
let options = AgentOptions {
urls: Some(vec!["https://example.com".to_string()]),
prompt: "Describe what this website is about and list any key features mentioned"
.to_string(),
model: Some(AgentModel::Spark1Pro),
timeout: Some(60),
..Default::default()
};
let result = client.agent(options).await;
match result {
Ok(response) => {
println!("Agent status: {:?}", response.status);
if let Some(data) = response.data {
println!("Result: {}", serde_json::to_string_pretty(&data).unwrap());
}
}
Err(e) => eprintln!("Agent failed: {}", e),
}
// Example 9: Agent with Typed Schema
println!("\n=== Example 9: Agent with Typed Schema ===");
#[derive(Debug, Deserialize)]
struct CompanyInfo {
name: String,
description: Option<String>,
industry: Option<String>,
}
let schema = json!({
"type": "object",
"properties": {
"name": { "type": "string" },
"description": { "type": "string" },
"industry": { "type": "string" }
},
"required": ["name"]
});
let result: Result<Option<CompanyInfo>, _> = client
.agent_with_schema(
vec!["https://firecrawl.dev".to_string()],
"Extract company information from this website",
schema,
)
.await;
match result {
Ok(Some(info)) => {
println!("Company: {}", info.name);
println!("Description: {:?}", info.description);
println!("Industry: {:?}", info.industry);
}
Ok(None) => println!("No data extracted"),
Err(e) => eprintln!("Agent with schema failed: {}", e),
}
println!("\n=== All examples completed ===");
}
|