Spaces:
Sleeping
Sleeping
Download apps/rust-sdk/src/scrape.rs from s-eto-hub/firecrawl-api: direct link, hf CLI and curl.
- Browser
- Download file 24.9 kB
-
https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/rust-sdk/src/scrape.rs
- Command line
-
hf download hf://spaces/s-eto-hub/firecrawl-api/apps/rust-sdk/src/scrape.rs
-
curl -L -o scrape.rs https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/rust-sdk/src/scrape.rs
24.9 kB
| //! Scrape endpoint for Firecrawl API v2. | |
| use serde::{Deserialize, Serialize}; | |
| use serde_json::Value; | |
| use std::collections::HashMap; | |
| use crate::client::Client; | |
| use crate::types::{ | |
| Action, AttributeSelector, ChangeTrackingOptions, Document, Format, JsonOptions, | |
| LocationConfig, ProfileConfig, ProxyType, ScreenshotOptions, | |
| }; | |
| use crate::FirecrawlError; | |
| /// Options for scraping a URL. | |
| pub struct ScrapeOptions { | |
| /// Output formats to include in the response. | |
| pub formats: Option<Vec<Format>>, | |
| /// Additional HTTP headers to send with the request. | |
| pub headers: Option<HashMap<String, String>>, | |
| /// HTML tags to exclusively include in the output. | |
| pub include_tags: Option<Vec<String>>, | |
| /// HTML tags to exclude from the output. | |
| pub exclude_tags: Option<Vec<String>>, | |
| /// Only extract the main content of the page. | |
| pub only_main_content: Option<bool>, | |
| /// Timeout in milliseconds before returning an error. | |
| pub timeout: Option<u32>, | |
| /// Time to wait after page load before scraping (milliseconds). | |
| pub wait_for: Option<u32>, | |
| /// Emulate a mobile device. | |
| pub mobile: Option<bool>, | |
| /// Parser configurations (e.g., for PDFs). | |
| pub parsers: Option<Vec<ParserConfig>>, | |
| /// Browser automation actions to perform before scraping. | |
| pub actions: Option<Vec<Action>>, | |
| /// Location configuration for proxy routing. | |
| pub location: Option<LocationConfig>, | |
| /// Skip TLS certificate verification. | |
| pub skip_tls_verification: Option<bool>, | |
| /// Remove base64-encoded images from the output. | |
| pub remove_base64_images: Option<bool>, | |
| /// Enable fast mode for quicker scrapes with reduced accuracy. | |
| pub fast_mode: Option<bool>, | |
| /// Block advertisements on the page. | |
| pub block_ads: Option<bool>, | |
| /// Proxy type to use. | |
| pub proxy: Option<ProxyType>, | |
| /// Maximum age of cached content to accept (seconds). | |
| pub max_age: Option<u32>, | |
| /// Minimum age of cached content to accept (seconds). | |
| pub min_age: Option<u32>, | |
| /// Store the result in cache for future requests. | |
| pub store_in_cache: Option<bool>, | |
| /// Lockdown mode: serve only previously cached results, never make outbound requests. | |
| pub lockdown: Option<bool>, | |
| /// Persistent browser profile for maintaining state across scrapes. | |
| pub profile: Option<ProfileConfig>, | |
| /// Integration identifier for tracking. | |
| pub integration: Option<String>, | |
| /// JSON extraction options. | |
| pub json_options: Option<JsonOptions>, | |
| /// Screenshot options. | |
| pub screenshot_options: Option<ScreenshotOptions>, | |
| /// Change tracking options. | |
| pub change_tracking_options: Option<ChangeTrackingOptions>, | |
| /// Attribute selectors for extraction. | |
| pub attribute_selectors: Option<Vec<AttributeSelector>>, | |
| } | |
| /// Parser configuration for document parsing. | |
| pub enum ParserConfig { | |
| /// Simple parser type string. | |
| Simple(String), | |
| /// PDF parser with options. | |
| Pdf { | |
| parser_type: String, | |
| mode: Option<String>, | |
| max_pages: Option<u32>, | |
| }, | |
| } | |
| /// Request body for scrape endpoint. | |
| struct ScrapeRequest { | |
| url: String, | |
| options: ScrapeOptions, | |
| } | |
| /// Response from scrape endpoint. | |
| struct ScrapeResponse { | |
| success: bool, | |
| data: Document, | |
| warning: Option<String>, | |
| } | |
| /// Supported languages for scrape-bound browser execution. | |
| pub enum ScrapeExecuteLanguage { | |
| Python, | |
| Node, | |
| Bash, | |
| } | |
| /// Options for executing code or a prompt in a scrape-bound browser session. | |
| /// | |
| /// At least one of `code` or `prompt` must be provided. | |
| pub struct ScrapeExecuteOptions { | |
| /// Code to execute (optional if `prompt` is provided). | |
| pub code: Option<String>, | |
| /// Natural-language instruction for the browser agent (optional if `code` is provided). | |
| pub prompt: Option<String>, | |
| /// Runtime language for the code. | |
| pub language: Option<ScrapeExecuteLanguage>, | |
| /// Execution timeout in seconds. | |
| pub timeout: Option<u32>, | |
| /// Optional origin tag for request attribution. | |
| pub origin: Option<String>, | |
| } | |
| /// Response from scrape-bound browser execution. | |
| pub struct ScrapeExecuteResponse { | |
| /// Whether the request succeeded. | |
| pub success: bool, | |
| /// Live-view URL for the browser session. | |
| pub live_view_url: Option<String>, | |
| /// Interactive live-view URL for the browser session. | |
| pub interactive_live_view_url: Option<String>, | |
| /// Agent output when a prompt was used. | |
| pub output: Option<String>, | |
| /// Captured stdout from execution. | |
| pub stdout: Option<String>, | |
| /// Optional execution result payload. | |
| pub result: Option<String>, | |
| /// Captured stderr from execution. | |
| pub stderr: Option<String>, | |
| /// Process exit code. | |
| pub exit_code: Option<i32>, | |
| /// Whether execution was killed by timeout or system. | |
| pub killed: Option<bool>, | |
| /// Error message when execution fails. | |
| pub error: Option<String>, | |
| } | |
| /// Response from deleting a scrape-bound browser session. | |
| pub struct ScrapeBrowserDeleteResponse { | |
| /// Whether the delete request succeeded. | |
| pub success: bool, | |
| /// Session duration in milliseconds when available. | |
| pub session_duration_ms: Option<u64>, | |
| /// Credits billed when available. | |
| pub credits_billed: Option<u32>, | |
| /// Error message when deletion fails. | |
| pub error: Option<String>, | |
| } | |
| impl Client { | |
| /// Scrapes a URL and returns the content in the requested formats. | |
| /// | |
| /// # Arguments | |
| /// | |
| /// * `url` - The URL to scrape. | |
| /// * `options` - Optional scrape configuration. | |
| /// | |
| /// # Returns | |
| /// | |
| /// A `Document` containing the scraped content. | |
| /// | |
| /// # Example | |
| /// | |
| /// ```no_run | |
| /// use firecrawl::{Client, ScrapeOptions, Format}; | |
| /// | |
| /// #[tokio::main] | |
| /// async fn main() -> Result<(), Box<dyn std::error::Error>> { | |
| /// let client = Client::new("your-api-key")?; | |
| /// | |
| /// // Simple scrape | |
| /// let document = client.scrape("https://example.com", None).await?; | |
| /// println!("Markdown: {:?}", document.markdown); | |
| /// | |
| /// // Scrape with options | |
| /// let options = ScrapeOptions { | |
| /// formats: Some(vec![Format::Markdown, Format::Html, Format::Links]), | |
| /// only_main_content: Some(true), | |
| /// ..Default::default() | |
| /// }; | |
| /// let document = client.scrape("https://example.com", options).await?; | |
| /// println!("Links: {:?}", document.links); | |
| /// | |
| /// Ok(()) | |
| /// } | |
| /// ``` | |
| pub async fn scrape( | |
| &self, | |
| url: impl AsRef<str>, | |
| options: impl Into<Option<ScrapeOptions>>, | |
| ) -> Result<Document, FirecrawlError> { | |
| let body = ScrapeRequest { | |
| url: url.as_ref().to_string(), | |
| options: options.into().unwrap_or_default(), | |
| }; | |
| let headers = self.prepare_headers(None); | |
| let response = self | |
| .client | |
| .post(self.url("/scrape")) | |
| .headers(headers) | |
| .json(&body) | |
| .send() | |
| .await | |
| .map_err(|e| FirecrawlError::HttpError(format!("Scraping {:?}", url.as_ref()), e))?; | |
| let response: ScrapeResponse = self.handle_response(response, "scrape").await?; | |
| Ok(response.data) | |
| } | |
| /// Scrapes a URL with a JSON schema for structured extraction. | |
| /// | |
| /// This is a convenience method that combines scraping with JSON extraction. | |
| /// | |
| /// # Arguments | |
| /// | |
| /// * `url` - The URL to scrape. | |
| /// * `schema` - JSON schema for the extraction. | |
| /// * `prompt` - Optional extraction prompt. | |
| /// | |
| /// # Returns | |
| /// | |
| /// The extracted JSON value matching the schema. | |
| /// | |
| /// # Example | |
| /// | |
| /// ```no_run | |
| /// use firecrawl::Client; | |
| /// use serde_json::json; | |
| /// | |
| /// #[tokio::main] | |
| /// async fn main() -> Result<(), Box<dyn std::error::Error>> { | |
| /// let client = Client::new("your-api-key")?; | |
| /// | |
| /// let schema = json!({ | |
| /// "type": "object", | |
| /// "properties": { | |
| /// "title": { "type": "string" }, | |
| /// "price": { "type": "number" } | |
| /// } | |
| /// }); | |
| /// | |
| /// let data = client.scrape_with_schema( | |
| /// "https://example.com/product", | |
| /// schema, | |
| /// Some("Extract the product title and price") | |
| /// ).await?; | |
| /// | |
| /// println!("Extracted: {}", data); | |
| /// | |
| /// Ok(()) | |
| /// } | |
| /// ``` | |
| pub async fn scrape_with_schema( | |
| &self, | |
| url: impl AsRef<str>, | |
| schema: Value, | |
| prompt: Option<impl AsRef<str>>, | |
| ) -> Result<Value, FirecrawlError> { | |
| let options = ScrapeOptions { | |
| formats: Some(vec![Format::Json]), | |
| json_options: Some(JsonOptions { | |
| schema: Some(schema), | |
| prompt: prompt.map(|p| p.as_ref().to_string()), | |
| ..Default::default() | |
| }), | |
| ..Default::default() | |
| }; | |
| let document = self.scrape(url, options).await?; | |
| Ok(document.json.unwrap_or(Value::Null)) | |
| } | |
| /// Interacts with the browser session associated with a scrape job. | |
| /// | |
| /// # Arguments | |
| /// | |
| /// * `job_id` - The scrape job ID. | |
| /// * `options` - Execution options including code and runtime config. | |
| /// | |
| /// # Returns | |
| /// | |
| /// A `ScrapeExecuteResponse` containing execution output. | |
| pub async fn interact( | |
| &self, | |
| job_id: impl AsRef<str>, | |
| options: ScrapeExecuteOptions, | |
| ) -> Result<ScrapeExecuteResponse, FirecrawlError> { | |
| let has_code = options.code.as_ref().is_some_and(|c| !c.trim().is_empty()); | |
| let has_prompt = options | |
| .prompt | |
| .as_ref() | |
| .is_some_and(|p| !p.trim().is_empty()); | |
| if !has_code && !has_prompt { | |
| return Err(FirecrawlError::Misuse( | |
| "Either 'code' or 'prompt' must be provided".into(), | |
| )); | |
| } | |
| let mut body = options; | |
| if body.language.is_none() { | |
| body.language = Some(ScrapeExecuteLanguage::Node); | |
| } | |
| let response = self | |
| .client | |
| .post(self.url(&format!("/scrape/{}/interact", job_id.as_ref()))) | |
| .headers(self.prepare_headers(None)) | |
| .json(&body) | |
| .send() | |
| .await | |
| .map_err(|e| { | |
| FirecrawlError::HttpError( | |
| format!("Interacting with scrape browser for {}", job_id.as_ref()), | |
| e, | |
| ) | |
| })?; | |
| self.handle_response(response, "scrape interact").await | |
| } | |
| /// Stops the interaction session associated with a scrape job. | |
| /// | |
| /// # Arguments | |
| /// | |
| /// * `job_id` - The scrape job ID. | |
| /// | |
| /// # Returns | |
| /// | |
| /// A `ScrapeBrowserDeleteResponse` indicating stop status. | |
| pub async fn stop_interaction( | |
| &self, | |
| job_id: impl AsRef<str>, | |
| ) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> { | |
| let response = self | |
| .client | |
| .delete(self.url(&format!("/scrape/{}/interact", job_id.as_ref()))) | |
| .headers(self.prepare_headers(None)) | |
| .send() | |
| .await | |
| .map_err(|e| { | |
| FirecrawlError::HttpError( | |
| format!("Stopping interaction for {}", job_id.as_ref()), | |
| e, | |
| ) | |
| })?; | |
| self.handle_response(response, "stop interaction").await | |
| } | |
| /// Deprecated alias for [`Client::interact`]. | |
| pub async fn scrape_execute( | |
| &self, | |
| job_id: impl AsRef<str>, | |
| options: ScrapeExecuteOptions, | |
| ) -> Result<ScrapeExecuteResponse, FirecrawlError> { | |
| self.interact(job_id, options).await | |
| } | |
| /// Deprecated alias for [`Client::stop_interaction`]. | |
| pub async fn stop_interactive_browser( | |
| &self, | |
| job_id: impl AsRef<str>, | |
| ) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> { | |
| self.stop_interaction(job_id).await | |
| } | |
| /// Deprecated alias for [`Client::stop_interaction`]. | |
| pub async fn delete_scrape_browser( | |
| &self, | |
| job_id: impl AsRef<str>, | |
| ) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> { | |
| self.stop_interaction(job_id).await | |
| } | |
| } | |
| mod tests { | |
| use super::*; | |
| use crate::{HighlightsFormat, QueryFormat, QueryFormatMode, QuestionFormat}; | |
| use serde_json::json; | |
| fn test_query_format_serializes_mode() { | |
| let options = ScrapeOptions { | |
| formats: Some(vec![Format::Query(QueryFormat { | |
| prompt: "What is Firecrawl?".to_string(), | |
| mode: Some(QueryFormatMode::DirectQuote), | |
| })]), | |
| ..Default::default() | |
| }; | |
| let payload = serde_json::to_value(options).unwrap(); | |
| assert_eq!( | |
| payload["formats"][0], | |
| json!({ | |
| "type": "query", | |
| "prompt": "What is Firecrawl?", | |
| "mode": "directQuote" | |
| }) | |
| ); | |
| } | |
| fn test_question_and_highlights_formats_serialize() { | |
| let options = ScrapeOptions { | |
| formats: Some(vec![ | |
| Format::Question(QuestionFormat { | |
| question: "What is Firecrawl?".to_string(), | |
| }), | |
| Format::Highlights(HighlightsFormat { | |
| query: "What is Firecrawl?".to_string(), | |
| }), | |
| ]), | |
| ..Default::default() | |
| }; | |
| let payload = serde_json::to_value(options).unwrap(); | |
| assert_eq!( | |
| payload["formats"][0], | |
| json!({ | |
| "type": "question", | |
| "question": "What is Firecrawl?" | |
| }) | |
| ); | |
| assert_eq!( | |
| payload["formats"][1], | |
| json!({ | |
| "type": "highlights", | |
| "query": "What is Firecrawl?" | |
| }) | |
| ); | |
| } | |
| async fn test_scrape_with_mock() { | |
| let mut server = mockito::Server::new_async().await; | |
| let mock = server | |
| .mock("POST", "/v2/scrape") | |
| .with_status(200) | |
| .with_header("content-type", "application/json") | |
| .with_body( | |
| json!({ | |
| "success": true, | |
| "data": { | |
| "markdown": "# Example Domain\n\nThis is an example.", | |
| "metadata": { | |
| "sourceURL": "https://example.com", | |
| "statusCode": 200, | |
| "title": "Example Domain" | |
| } | |
| } | |
| }) | |
| .to_string(), | |
| ) | |
| .create(); | |
| let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap(); | |
| let document = client.scrape("https://example.com", None).await.unwrap(); | |
| assert!(document.markdown.is_some()); | |
| assert!(document.markdown.unwrap().contains("Example Domain")); | |
| mock.assert(); | |
| } | |
| async fn test_scrape_with_options() { | |
| let mut server = mockito::Server::new_async().await; | |
| let mock = server | |
| .mock("POST", "/v2/scrape") | |
| .with_status(200) | |
| .with_header("content-type", "application/json") | |
| .with_body( | |
| json!({ | |
| "success": true, | |
| "data": { | |
| "markdown": "# Test", | |
| "html": "<h1>Test</h1>", | |
| "links": ["https://example.com/page1", "https://example.com/page2"], | |
| "metadata": { | |
| "sourceURL": "https://example.com", | |
| "statusCode": 200 | |
| } | |
| } | |
| }) | |
| .to_string(), | |
| ) | |
| .create(); | |
| let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap(); | |
| let options = ScrapeOptions { | |
| formats: Some(vec![Format::Markdown, Format::Html, Format::Links]), | |
| only_main_content: Some(true), | |
| ..Default::default() | |
| }; | |
| let document = client.scrape("https://example.com", options).await.unwrap(); | |
| assert!(document.markdown.is_some()); | |
| assert!(document.html.is_some()); | |
| assert!(document.links.is_some()); | |
| assert_eq!(document.links.unwrap().len(), 2); | |
| mock.assert(); | |
| } | |
| async fn test_scrape_with_schema() { | |
| let mut server = mockito::Server::new_async().await; | |
| let mock = server | |
| .mock("POST", "/v2/scrape") | |
| .with_status(200) | |
| .with_header("content-type", "application/json") | |
| .with_body( | |
| json!({ | |
| "success": true, | |
| "data": { | |
| "json": { | |
| "title": "Product Name", | |
| "price": 99.99 | |
| }, | |
| "metadata": { | |
| "sourceURL": "https://example.com/product", | |
| "statusCode": 200 | |
| } | |
| } | |
| }) | |
| .to_string(), | |
| ) | |
| .create(); | |
| let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap(); | |
| let schema = json!({ | |
| "type": "object", | |
| "properties": { | |
| "title": { "type": "string" }, | |
| "price": { "type": "number" } | |
| } | |
| }); | |
| let data = client | |
| .scrape_with_schema( | |
| "https://example.com/product", | |
| schema, | |
| Some("Extract product info"), | |
| ) | |
| .await | |
| .unwrap(); | |
| assert_eq!(data["title"], "Product Name"); | |
| assert_eq!(data["price"], 99.99); | |
| mock.assert(); | |
| } | |
| async fn test_scrape_error_response() { | |
| let mut server = mockito::Server::new_async().await; | |
| let mock = server | |
| .mock("POST", "/v2/scrape") | |
| .with_status(400) | |
| .with_header("content-type", "application/json") | |
| .with_body( | |
| json!({ | |
| "success": false, | |
| "error": "Invalid URL" | |
| }) | |
| .to_string(), | |
| ) | |
| .create(); | |
| let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap(); | |
| let result = client.scrape("invalid-url", None).await; | |
| assert!(result.is_err()); | |
| mock.assert(); | |
| } | |
| async fn test_interact_with_mock() { | |
| let mut server = mockito::Server::new_async().await; | |
| let mock = server | |
| .mock("POST", "/v2/scrape/job-123/interact") | |
| .with_status(200) | |
| .with_header("content-type", "application/json") | |
| .with_body( | |
| json!({ | |
| "success": true, | |
| "stdout": "ok", | |
| "result": "done", | |
| "stderr": "", | |
| "exitCode": 0, | |
| "killed": false | |
| }) | |
| .to_string(), | |
| ) | |
| .create(); | |
| let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap(); | |
| let response = client | |
| .interact( | |
| "job-123", | |
| ScrapeExecuteOptions { | |
| code: Some("console.log('ok')".to_string()), | |
| timeout: Some(30), | |
| ..Default::default() | |
| }, | |
| ) | |
| .await | |
| .unwrap(); | |
| assert!(response.success); | |
| assert_eq!(response.exit_code, Some(0)); | |
| assert_eq!(response.result, Some("done".to_string())); | |
| mock.assert(); | |
| } | |
| async fn test_interact_with_prompt() { | |
| let mut server = mockito::Server::new_async().await; | |
| let mock = server | |
| .mock("POST", "/v2/scrape/job-789/interact") | |
| .with_status(200) | |
| .with_header("content-type", "application/json") | |
| .with_body( | |
| json!({ | |
| "success": true, | |
| "output": "Clicked the login button", | |
| "liveViewUrl": "https://live.example.com/view", | |
| "interactiveLiveViewUrl": "https://live.example.com/interactive", | |
| "stdout": "", | |
| "exitCode": 0, | |
| "killed": false | |
| }) | |
| .to_string(), | |
| ) | |
| .create(); | |
| let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap(); | |
| let response = client | |
| .interact( | |
| "job-789", | |
| ScrapeExecuteOptions { | |
| prompt: Some("Click the login button".to_string()), | |
| ..Default::default() | |
| }, | |
| ) | |
| .await | |
| .unwrap(); | |
| assert!(response.success); | |
| assert_eq!( | |
| response.output, | |
| Some("Clicked the login button".to_string()) | |
| ); | |
| assert_eq!( | |
| response.live_view_url, | |
| Some("https://live.example.com/view".to_string()) | |
| ); | |
| assert_eq!( | |
| response.interactive_live_view_url, | |
| Some("https://live.example.com/interactive".to_string()) | |
| ); | |
| mock.assert(); | |
| } | |
| async fn test_stop_interaction_with_mock() { | |
| let mut server = mockito::Server::new_async().await; | |
| let mock = server | |
| .mock("DELETE", "/v2/scrape/job-123/interact") | |
| .with_status(200) | |
| .with_header("content-type", "application/json") | |
| .with_body( | |
| json!({ | |
| "success": true, | |
| "sessionDurationMs": 1200, | |
| "creditsBilled": 3 | |
| }) | |
| .to_string(), | |
| ) | |
| .create(); | |
| let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap(); | |
| let response = client.stop_interaction("job-123").await.unwrap(); | |
| assert!(response.success); | |
| assert_eq!(response.session_duration_ms, Some(1200)); | |
| assert_eq!(response.credits_billed, Some(3)); | |
| mock.assert(); | |
| } | |
| async fn test_interact_error_response() { | |
| let mut server = mockito::Server::new_async().await; | |
| let mock = server | |
| .mock("POST", "/v2/scrape/job-404/interact") | |
| .with_status(404) | |
| .with_header("content-type", "application/json") | |
| .with_body( | |
| json!({ | |
| "success": false, | |
| "error": "Job not found." | |
| }) | |
| .to_string(), | |
| ) | |
| .create(); | |
| let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap(); | |
| let result = client | |
| .interact( | |
| "job-404", | |
| ScrapeExecuteOptions { | |
| code: Some("console.log('ok')".to_string()), | |
| ..Default::default() | |
| }, | |
| ) | |
| .await; | |
| assert!(result.is_err()); | |
| mock.assert(); | |
| } | |
| } | |