Spaces:
Sleeping
Sleeping
Download apps/rust-sdk/src/parse.rs from s-eto-hub/firecrawl-api: direct link, hf CLI and curl.
- Browser
- Download file 9.06 kB
-
https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/rust-sdk/src/parse.rs
- Command line
-
hf download hf://spaces/s-eto-hub/firecrawl-api/apps/rust-sdk/src/parse.rs
-
curl -L -o parse.rs https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/rust-sdk/src/parse.rs
9.06 kB
| //! Parse endpoint for Firecrawl API v2. | |
| use std::path::Path; | |
| use reqwest::multipart::{Form, Part}; | |
| use serde::{Deserialize, Serialize}; | |
| use std::collections::HashMap; | |
| use super::client::Client; | |
| use super::scrape::ParserConfig; | |
| use super::types::{AttributeSelector, Document, JsonOptions}; | |
| use crate::FirecrawlError; | |
| /// Uploaded file payload for the `/v2/parse` endpoint. | |
| pub struct ParseFile { | |
| pub filename: String, | |
| pub bytes: Vec<u8>, | |
| pub content_type: Option<String>, | |
| } | |
| impl ParseFile { | |
| /// Build a parse file from in-memory bytes. | |
| pub fn from_bytes(filename: impl Into<String>, bytes: Vec<u8>) -> Self { | |
| Self { | |
| filename: filename.into(), | |
| bytes, | |
| content_type: None, | |
| } | |
| } | |
| /// Build a parse file by reading bytes from disk. | |
| pub fn from_path(path: impl AsRef<Path>) -> Result<Self, FirecrawlError> { | |
| let path_ref = path.as_ref(); | |
| let bytes = std::fs::read(path_ref).map_err(|e| { | |
| FirecrawlError::Misuse(format!( | |
| "Failed to read parse file {}: {}", | |
| path_ref.display(), | |
| e | |
| )) | |
| })?; | |
| let filename = path_ref | |
| .file_name() | |
| .and_then(|name| name.to_str()) | |
| .ok_or_else(|| { | |
| FirecrawlError::Misuse("Could not derive a valid filename from path".to_string()) | |
| })? | |
| .to_string(); | |
| Ok(Self { | |
| filename, | |
| bytes, | |
| content_type: None, | |
| }) | |
| } | |
| /// Attach a content type hint (e.g. `text/html`, `application/pdf`). | |
| pub fn with_content_type(mut self, content_type: impl Into<String>) -> Self { | |
| self.content_type = Some(content_type.into()); | |
| self | |
| } | |
| } | |
| /// Response from parse endpoint. | |
| struct ParseResponse { | |
| success: bool, | |
| data: Document, | |
| warning: Option<String>, | |
| } | |
| /// Proxy settings accepted by `/v2/parse`. | |
| pub enum ParseProxyType { | |
| Basic, | |
| Auto, | |
| } | |
| /// Output formats accepted by `/v2/parse`. | |
| pub enum ParseFormat { | |
| Markdown, | |
| Html, | |
| RawHtml, | |
| Links, | |
| Images, | |
| Summary, | |
| Json, | |
| Attributes, | |
| } | |
| /// Options accepted by the `/v2/parse` endpoint. | |
| /// | |
| /// This intentionally omits scrape-only fields that `/v2/parse` rejects | |
| /// (e.g. actions, waitFor, location, and screenshot/branding/changeTracking options). | |
| pub struct ParseOptions { | |
| /// Output formats to include in the response. | |
| pub formats: Option<Vec<ParseFormat>>, | |
| /// Additional HTTP headers. | |
| pub headers: Option<HashMap<String, String>>, | |
| /// HTML tags to include. | |
| pub include_tags: Option<Vec<String>>, | |
| /// HTML tags to exclude. | |
| pub exclude_tags: Option<Vec<String>>, | |
| /// Extract only the main content. | |
| pub only_main_content: Option<bool>, | |
| /// Timeout in milliseconds. | |
| pub timeout: Option<u32>, | |
| /// Parser configurations (e.g. PDF parser). | |
| pub parsers: Option<Vec<ParserConfig>>, | |
| /// Skip TLS verification. | |
| pub skip_tls_verification: Option<bool>, | |
| /// Remove base64 images. | |
| pub remove_base64_images: Option<bool>, | |
| /// Fast mode. | |
| pub fast_mode: Option<bool>, | |
| /// Mock fixture id to use. | |
| pub use_mock: Option<String>, | |
| /// Block ads. | |
| pub block_ads: Option<bool>, | |
| /// Proxy type. | |
| pub proxy: Option<ParseProxyType>, | |
| /// Integration identifier. | |
| pub integration: Option<String>, | |
| /// Request origin identifier. | |
| pub origin: Option<String>, | |
| /// Zero data retention mode. | |
| pub zero_data_retention: Option<bool>, | |
| /// JSON extraction options. | |
| pub json_options: Option<JsonOptions>, | |
| /// Attribute selectors for extraction. | |
| pub attribute_selectors: Option<Vec<AttributeSelector>>, | |
| } | |
| impl Client { | |
| /// Parse an uploaded file and return the extracted document. | |
| pub async fn parse( | |
| &self, | |
| file: ParseFile, | |
| options: impl Into<Option<ParseOptions>>, | |
| ) -> Result<Document, FirecrawlError> { | |
| let resolved_filename = file.filename.trim().to_string(); | |
| if resolved_filename.is_empty() { | |
| return Err(FirecrawlError::Misuse( | |
| "filename cannot be empty".to_string(), | |
| )); | |
| } | |
| if file.bytes.is_empty() { | |
| return Err(FirecrawlError::Misuse( | |
| "file content cannot be empty".to_string(), | |
| )); | |
| } | |
| let options = options.into().unwrap_or_default(); | |
| let options_json = | |
| serde_json::to_string(&options).map_err(FirecrawlError::ResponseParseError)?; | |
| let mut part = Part::bytes(file.bytes).file_name(resolved_filename); | |
| if let Some(content_type) = file.content_type { | |
| part = part.mime_str(&content_type).map_err(|e| { | |
| FirecrawlError::Misuse(format!("Invalid content type for parse file: {}", e)) | |
| })?; | |
| } | |
| let form = Form::new().text("options", options_json).part("file", part); | |
| let headers = self.prepare_multipart_headers(None); | |
| let response = self | |
| .client | |
| .post(self.url("/parse")) | |
| .headers(headers) | |
| .multipart(form) | |
| .send() | |
| .await | |
| .map_err(|e| FirecrawlError::HttpError("Parsing uploaded file".to_string(), e))?; | |
| let response: ParseResponse = self.handle_response(response, "parse").await?; | |
| Ok(response.data) | |
| } | |
| } | |
| mod tests { | |
| use super::*; | |
| use mockito::Matcher; | |
| use serde_json::json; | |
| async fn test_parse_with_mock() { | |
| let mut server = mockito::Server::new_async().await; | |
| let mock = server | |
| .mock("POST", "/v2/parse") | |
| .match_header( | |
| "content-type", | |
| Matcher::Regex("multipart/form-data".to_string()), | |
| ) | |
| .with_status(200) | |
| .with_header("content-type", "application/json") | |
| .with_body( | |
| json!({ | |
| "success": true, | |
| "data": { | |
| "markdown": "# Parsed File", | |
| "metadata": { | |
| "sourceURL": "https://parse.firecrawl.dev/uploads/upload.html", | |
| "statusCode": 200 | |
| } | |
| } | |
| }) | |
| .to_string(), | |
| ) | |
| .create(); | |
| let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap(); | |
| let file = ParseFile::from_bytes("upload.html", b"<html><body>ok</body></html>".to_vec()) | |
| .with_content_type("text/html"); | |
| let doc = client.parse(file, None).await.unwrap(); | |
| assert!(doc.markdown.is_some()); | |
| assert!(doc.markdown.unwrap().contains("Parsed File")); | |
| mock.assert(); | |
| } | |
| fn test_parse_file_from_missing_path() { | |
| let result = ParseFile::from_path("/tmp/this-file-should-not-exist-for-parse-sdk-test"); | |
| assert!(result.is_err()); | |
| } | |
| async fn test_parse_error_response() { | |
| let mut server = mockito::Server::new_async().await; | |
| let mock = server | |
| .mock("POST", "/v2/parse") | |
| .match_header( | |
| "content-type", | |
| Matcher::Regex("multipart/form-data".to_string()), | |
| ) | |
| .with_status(400) | |
| .with_header("content-type", "application/json") | |
| .with_body( | |
| json!({ | |
| "success": false, | |
| "error": "Unsupported upload type." | |
| }) | |
| .to_string(), | |
| ) | |
| .create(); | |
| let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap(); | |
| let file = ParseFile::from_bytes("upload.xyz", b"not a real file".to_vec()); | |
| let result = client.parse(file, None).await; | |
| assert!(result.is_err()); | |
| mock.assert(); | |
| } | |
| fn test_parse_rejects_empty_bytes() { | |
| let file = ParseFile::from_bytes("empty.html", vec![]); | |
| let rt = tokio::runtime::Runtime::new().unwrap(); | |
| let result = rt.block_on(async { | |
| let client = Client::new_selfhosted("http://localhost:9999", Some("k")).unwrap(); | |
| client.parse(file, None).await | |
| }); | |
| assert!(result.is_err()); | |
| let err_msg = format!("{}", result.unwrap_err()); | |
| assert!( | |
| err_msg.contains("empty"), | |
| "Expected empty file error, got: {}", | |
| err_msg | |
| ); | |
| } | |
| } | |