Download src/parser.rs from misukisu/docparse: direct link, hf CLI and curl.
- Browser
- Download file 6.12 kB
-
https://huggingface.co/spaces/misukisu/docparse/resolve/74fa4cd6ff6bdb071afea7cb584d2d770b32a41e/src/parser.rs
- Command line
-
hf download hf://spaces/misukisu/docparse@74fa4cd6ff6bdb071afea7cb584d2d770b32a41e/src/parser.rs
-
curl -L -o parser.rs https://huggingface.co/spaces/misukisu/docparse/resolve/74fa4cd6ff6bdb071afea7cb584d2d770b32a41e/src/parser.rs
6.12 kB
| use anyhow::{anyhow, Context, Result}; | |
| use memmap2::Mmap; | |
| use quick_xml::events::Event; | |
| use quick_xml::reader::Reader; | |
| use std::fs::File; | |
| use std::io::{BufRead, Cursor, Read}; | |
| use std::path::Path; | |
| use thiserror::Error; | |
| pub enum ParserError { | |
| Io( std::io::Error), | |
| UnsupportedFormat(String), | |
| CorruptDocument(String), | |
| } | |
| pub struct ParsedDocument { | |
| pub path: String, | |
| pub title: String, | |
| pub content: String, | |
| pub extension: String, | |
| pub size_bytes: u64, | |
| } | |
| pub struct DocumentParser; | |
| impl DocumentParser { | |
| /// Ingests a file with zero-copy mmap for large files or standard read for small files. | |
| pub fn parse_file<P: AsRef<Path>>(path: P) -> Result<ParsedDocument> { | |
| let path_ref = path.as_ref(); | |
| let metadata = std::fs::metadata(path_ref) | |
| .with_context(|| format!("Failed to read metadata for {:?}", path_ref))?; | |
| let size_bytes = metadata.len(); | |
| if size_bytes == 0 { | |
| return Ok(ParsedDocument { | |
| path: path_ref.to_string_lossy().to_string(), | |
| title: Self::extract_title(path_ref), | |
| content: String::new(), | |
| extension: Self::get_extension(path_ref), | |
| size_bytes: 0, | |
| }); | |
| } | |
| let extension = Self::get_extension(path_ref); | |
| let title = Self::extract_title(path_ref); | |
| let file = File::open(path_ref)?; | |
| // Zero-copy memory map for files >= 16 KB; heap read for smaller files | |
| let content = if size_bytes >= 16 * 1024 { | |
| let mmap = unsafe { Mmap::map(&file)? }; | |
| Self::extract_content_from_bytes(&mmap, &extension)? | |
| } else { | |
| let mut buffer = Vec::with_capacity(size_bytes as usize); | |
| let mut reader = std::io::BufReader::new(file); | |
| reader.read_to_end(&mut buffer)?; | |
| Self::extract_content_from_bytes(&buffer, &extension)? | |
| }; | |
| Ok(ParsedDocument { | |
| path: path_ref.to_string_lossy().to_string(), | |
| title, | |
| content, | |
| extension, | |
| size_bytes, | |
| }) | |
| } | |
| fn extract_content_from_bytes(bytes: &[u8], extension: &str) -> Result<String> { | |
| match extension.to_lowercase().as_str() { | |
| "txt" | "md" | "markdown" | "log" | "rs" | "py" | "js" | "ts" | "toml" | "yaml" | "yml" => { | |
| match std::str::from_utf8(bytes) { | |
| Ok(valid_str) => Ok(valid_str.to_string()), | |
| Err(_) => Ok(String::from_utf8_lossy(bytes).into_owned()), | |
| } | |
| } | |
| "json" => { | |
| let val: serde_json::Value = serde_json::from_slice(bytes) | |
| .context("Invalid JSON document")?; | |
| if let Some(text) = val.as_str() { | |
| Ok(text.to_string()) | |
| } else { | |
| Ok(serde_json::to_string_pretty(&val)?) | |
| } | |
| } | |
| "csv" => Self::parse_csv(bytes), | |
| "docx" => Self::parse_docx(bytes), | |
| "pdf" => Self::parse_pdf(bytes), | |
| ext => Err(ParserError::UnsupportedFormat(ext.to_string()).into()), | |
| } | |
| } | |
| fn parse_csv(bytes: &[u8]) -> Result<String> { | |
| let mut rdr = csv::ReaderBuilder::new() | |
| .flexible(true) | |
| .has_headers(true) | |
| .from_reader(Cursor::new(bytes)); | |
| let mut output = String::with_capacity(bytes.len()); | |
| if let Ok(headers) = rdr.headers() { | |
| output.push_str(&headers.iter().collect::<Vec<_>>().join(" | ")); | |
| output.push('\n'); | |
| } | |
| for result in rdr.records() { | |
| let record = result?; | |
| output.push_str(&record.iter().collect::<Vec<_>>().join(" | ")); | |
| output.push('\n'); | |
| } | |
| Ok(output) | |
| } | |
| fn parse_docx(bytes: &[u8]) -> Result<String> { | |
| let cursor = Cursor::new(bytes); | |
| let mut archive = zip::ZipArchive::new(cursor) | |
| .context("Failed to open DOCX as ZIP archive")?; | |
| let mut document_xml = archive | |
| .by_name("word/document.xml") | |
| .context("Missing word/document.xml inside DOCX archive")?; | |
| let mut xml_bytes = Vec::new(); | |
| document_xml.read_to_end(&mut xml_bytes)?; | |
| let mut reader = Reader::from_reader(Cursor::new(xml_bytes)); | |
| reader.config_mut().trim_text(true); | |
| let mut txt = String::new(); | |
| let mut buf = Vec::new(); | |
| let mut in_text_node = false; | |
| loop { | |
| match reader.read_event_into(&mut buf) { | |
| Ok(Event::Start(ref e)) if e.name().as_ref() == b"w:t" => { | |
| in_text_node = true; | |
| } | |
| Ok(Event::End(ref e)) if e.name().as_ref() == b"w:t" => { | |
| in_text_node = false; | |
| } | |
| Ok(Event::End(ref e)) if e.name().as_ref() == b"w:p" => { | |
| txt.push('\n'); | |
| } | |
| Ok(Event::Text(e)) if in_text_node => { | |
| if let Ok(s) = e.unescape() { | |
| txt.push_str(&s); | |
| } | |
| } | |
| Ok(Event::Eof) => break, | |
| Err(e) => return Err(anyhow!("XML Parsing error in DOCX: {}", e)), | |
| _ => (), | |
| } | |
| buf.clear(); | |
| } | |
| Ok(txt) | |
| } | |
| fn parse_pdf(bytes: &[u8]) -> Result<String> { | |
| pdf_extract::extract_text_from_mem(bytes) | |
| .context("Failed to extract text stream from PDF") | |
| } | |
| fn extract_title(path: &Path) -> String { | |
| path.file_name() | |
| .map(|f| f.to_string_lossy().to_string()) | |
| .unwrap_or_else(|| "Untitled".to_string()) | |
| } | |
| fn get_extension(path: &Path) -> String { | |
| path.extension() | |
| .map(|e| e.to_string_lossy().to_string()) | |
| .unwrap_or_else(|| "txt".to_string()) | |
| } | |
| } |