thio / core /utils /fast_parse /parser.py
kyrlos adel
cva1
7cfdc4c
Raw History Blame Contribute Delete
33.8 kB
from __future__ import annotations
import io
import os
import re
import mimetypes
from dataclasses import dataclass, field
from enum import Enum, auto
from pathlib import Path
from typing import Any, Dict, List, Optional, Union, BinaryIO
import chardet
from .config import FastParseConfig, DEFAULT_CONFIG
class FileType(Enum):
TEXT = auto()
PDF = auto()
WORD = auto()
EXCEL = auto()
PRESENTATION = auto()
IMAGE = auto()
BINARY = auto()
UNKNOWN = auto()
class ParseError(Exception):
def __init__(self, message: str, error_code: str = "PARSE_ERROR", details: Optional[Dict[str, Any]] = None):
super().__init__(message)
self.message = message
self.error_code = error_code
self.details = details or {}
@dataclass
class ParseResult:
success: bool
content: str
file_type: FileType
filename: str
mime_type: str
file_size: int
metadata: Dict[str, Any] = field(default_factory=dict)
warnings: List[str] = field(default_factory=list)
error: Optional[str] = None
@property
def is_empty(self) -> bool:
return not self.content or not self.content.strip()
@property
def char_count(self) -> int:
return len(self.content) if self.content else 0
@property
def line_count(self) -> int:
return len(self.content.splitlines()) if self.content else 0
def to_dict(self) -> Dict[str, Any]:
return {
"success": self.success,
"content": self.content,
"file_type": self.file_type.name,
"filename": self.filename,
"mime_type": self.mime_type,
"file_size": self.file_size,
"metadata": self.metadata,
"warnings": self.warnings,
"error": self.error,
"char_count": self.char_count,
"line_count": self.line_count,
}
class FastParse:
__slots__ = ("_config", "_extension_map")
def __init__(self, config: Optional[FastParseConfig] = None):
self._config = config or DEFAULT_CONFIG
self._extension_map = self._build_extension_map()
def _build_extension_map(self) -> Dict[str, FileType]:
ext_map: Dict[str, FileType] = {}
for ext in self._config.text_extensions:
ext_map[ext.lower()] = FileType.TEXT
for ext in self._config.pdf_extensions:
ext_map[ext.lower()] = FileType.PDF
for ext in self._config.word_extensions:
ext_map[ext.lower()] = FileType.WORD
for ext in self._config.excel_extensions:
ext_map[ext.lower()] = FileType.EXCEL
for ext in self._config.presentation_extensions:
ext_map[ext.lower()] = FileType.PRESENTATION
for ext in self._config.image_extensions:
ext_map[ext.lower()] = FileType.IMAGE
for ext in self._config.binary_extensions:
ext_map[ext.lower()] = FileType.BINARY
return ext_map
def detect_file_type(self, filename: str, mime_type: Optional[str] = None) -> FileType:
ext = Path(filename).suffix.lower()
if ext in self._extension_map:
return self._extension_map[ext]
if mime_type:
mime_lower = mime_type.lower()
if mime_lower.startswith("text/"):
return FileType.TEXT
if mime_lower == "application/pdf":
return FileType.PDF
if mime_lower in ("application/msword", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"):
return FileType.WORD
if mime_lower in ("application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"):
return FileType.EXCEL
if mime_lower.startswith("image/"):
return FileType.IMAGE
if mime_lower == "application/octet-stream":
return FileType.BINARY
return FileType.UNKNOWN
def parse(
self,
content: Union[bytes, BinaryIO, str],
filename: str,
mime_type: Optional[str] = None,
) -> ParseResult:
if isinstance(content, str):
file_bytes = content.encode("utf-8")
elif hasattr(content, "read"):
file_bytes = content.read()
else:
file_bytes = content
file_size = len(file_bytes)
if file_size > self._config.max_file_size_bytes:
return ParseResult(
success=False,
content="",
file_type=FileType.UNKNOWN,
filename=filename,
mime_type=mime_type or "application/octet-stream",
file_size=file_size,
error=f"File exceeds maximum size limit of {self._config.max_file_size_bytes / (1024*1024):.1f}MB",
)
if not mime_type:
mime_type, _ = mimetypes.guess_type(filename)
mime_type = mime_type or "application/octet-stream"
file_type = self.detect_file_type(filename, mime_type)
try:
if file_type == FileType.TEXT:
return self._parse_text(file_bytes, filename, mime_type, file_size)
elif file_type == FileType.PDF:
return self._parse_pdf(file_bytes, filename, mime_type, file_size)
elif file_type == FileType.WORD:
return self._parse_word(file_bytes, filename, mime_type, file_size)
elif file_type == FileType.EXCEL:
return self._parse_excel(file_bytes, filename, mime_type, file_size)
elif file_type == FileType.PRESENTATION:
return self._parse_presentation(file_bytes, filename, mime_type, file_size)
elif file_type == FileType.IMAGE:
return self._parse_image(file_bytes, filename, mime_type, file_size)
elif file_type == FileType.BINARY:
return self._parse_binary(file_bytes, filename, mime_type, file_size)
else:
return self._parse_unknown(file_bytes, filename, mime_type, file_size)
except ParseError as e:
return ParseResult(
success=False,
content="",
file_type=file_type,
filename=filename,
mime_type=mime_type,
file_size=file_size,
error=e.message,
)
except Exception as e:
return ParseResult(
success=False,
content="",
file_type=file_type,
filename=filename,
mime_type=mime_type,
file_size=file_size,
error=f"Parsing failed: {str(e)}",
)
def parse_file(self, file_path: Union[str, Path]) -> ParseResult:
path = Path(file_path)
if not path.exists():
return ParseResult(
success=False,
content="",
file_type=FileType.UNKNOWN,
filename=path.name,
mime_type="application/octet-stream",
file_size=0,
error=f"File not found: {file_path}",
)
file_size = path.stat().st_size
if file_size > self._config.max_file_size_bytes:
return ParseResult(
success=False,
content="",
file_type=FileType.UNKNOWN,
filename=path.name,
mime_type="application/octet-stream",
file_size=file_size,
error=f"File exceeds maximum size limit of {self._config.max_file_size_bytes / (1024*1024):.1f}MB",
)
with open(path, "rb") as f:
content = f.read()
return self.parse(content, path.name)
def _check_script_injection(self, content: str) -> List[str]:
if not self._config.enable_script_detection:
return []
warnings = []
content_lower = content.lower()
for pattern in self._config.dangerous_patterns:
if pattern.lower() in content_lower:
warnings.append(f"Potentially dangerous pattern detected: {pattern}")
return warnings
def _parse_text(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
try:
detected = chardet.detect(data[:min(len(data), 10000)])
encoding = detected.get("encoding") or "utf-8"
confidence = detected.get("confidence", 0)
except Exception:
encoding = "utf-8"
confidence = 0.5
try:
content = data.decode(encoding)
except (UnicodeDecodeError, LookupError):
try:
content = data.decode("utf-8", errors="replace")
encoding = "utf-8 (fallback)"
except Exception:
content = data.decode("latin-1", errors="replace")
encoding = "latin-1 (fallback)"
if len(content) > self._config.max_text_chars:
content = content[:self._config.max_text_chars]
truncated = True
else:
truncated = False
warnings = self._check_script_injection(content)
ext = Path(filename).suffix.lower()
language = self._detect_language(ext)
metadata = {
"encoding": encoding,
"encoding_confidence": confidence,
"language": language,
"truncated": truncated,
}
if truncated:
warnings.append(f"Content truncated to {self._config.max_text_chars:,} characters")
return ParseResult(
success=True,
content=content,
file_type=FileType.TEXT,
filename=filename,
mime_type=mime_type,
file_size=file_size,
metadata=metadata,
warnings=warnings,
)
def _detect_language(self, ext: str) -> str:
language_map = {
".py": "python", ".pyw": "python", ".pyi": "python",
".js": "javascript", ".jsx": "javascript", ".mjs": "javascript",
".ts": "typescript", ".tsx": "typescript",
".java": "java", ".kt": "kotlin", ".scala": "scala",
".c": "c", ".h": "c", ".cpp": "cpp", ".hpp": "cpp",
".cs": "csharp", ".fs": "fsharp",
".go": "go", ".rs": "rust", ".swift": "swift",
".rb": "ruby", ".php": "php", ".pl": "perl",
".lua": "lua", ".r": "r", ".jl": "julia",
".sh": "bash", ".bash": "bash", ".zsh": "zsh",
".sql": "sql", ".graphql": "graphql",
".html": "html", ".htm": "html",
".css": "css", ".scss": "scss", ".sass": "sass",
".json": "json", ".yaml": "yaml", ".yml": "yaml",
".xml": "xml", ".md": "markdown", ".rst": "rst",
".vue": "vue", ".svelte": "svelte",
}
return language_map.get(ext, "text")
def _parse_pdf(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
try:
import PyPDF2
except ImportError:
raise ParseError("PyPDF2 not installed", "MISSING_DEPENDENCY")
try:
reader = PyPDF2.PdfReader(io.BytesIO(data))
except Exception as e:
raise ParseError(f"Invalid or corrupted PDF: {str(e)}", "INVALID_PDF")
total_pages = len(reader.pages)
pages_to_process = min(total_pages, self._config.max_pdf_pages)
text_parts: List[str] = []
for i in range(pages_to_process):
try:
page = reader.pages[i]
page_text = page.extract_text() or ""
if page_text.strip():
text_parts.append(f"--- Page {i + 1} ---\n{page_text}")
except Exception:
text_parts.append(f"--- Page {i + 1} ---\n[Error extracting text from this page]")
content = "\n\n".join(text_parts)
warnings = self._check_script_injection(content)
pdf_metadata = {}
if reader.metadata:
for key in ["/Title", "/Author", "/Subject", "/Creator", "/Producer", "/CreationDate"]:
val = reader.metadata.get(key)
if val:
pdf_metadata[key.lstrip("/")] = str(val)
metadata = {
"total_pages": total_pages,
"pages_processed": pages_to_process,
"pdf_metadata": pdf_metadata,
"truncated": pages_to_process < total_pages,
}
if pages_to_process < total_pages:
warnings.append(f"Only processed {pages_to_process} of {total_pages} pages")
return ParseResult(
success=True,
content=content,
file_type=FileType.PDF,
filename=filename,
mime_type=mime_type,
file_size=file_size,
metadata=metadata,
warnings=warnings,
)
def _parse_word(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
ext = Path(filename).suffix.lower()
if ext == ".docx":
return self._parse_docx(data, filename, mime_type, file_size)
elif ext == ".doc":
return self._parse_doc_legacy(data, filename, mime_type, file_size)
elif ext == ".rtf":
return self._parse_rtf(data, filename, mime_type, file_size)
elif ext == ".odt":
return self._parse_odt(data, filename, mime_type, file_size)
else:
raise ParseError(f"Unsupported Word format: {ext}", "UNSUPPORTED_FORMAT")
def _parse_docx(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
try:
import docx
except ImportError:
raise ParseError("python-docx not installed", "MISSING_DEPENDENCY")
try:
doc = docx.Document(io.BytesIO(data))
except Exception as e:
raise ParseError(f"Invalid or corrupted DOCX: {str(e)}", "INVALID_DOCX")
paragraphs = [p.text for p in doc.paragraphs if p.text.strip()]
tables_text: List[str] = []
for i, table in enumerate(doc.tables):
table_rows: List[str] = []
for row in table.rows:
cells = [cell.text.strip() for cell in row.cells]
table_rows.append(" | ".join(cells))
if table_rows:
tables_text.append(f"[Table {i + 1}]\n" + "\n".join(table_rows))
content_parts = paragraphs
if tables_text:
content_parts.extend(["", "--- Tables ---"] + tables_text)
content = "\n".join(content_parts)
warnings = self._check_script_injection(content)
core_props = doc.core_properties
metadata = {
"title": core_props.title or "",
"author": core_props.author or "",
"subject": core_props.subject or "",
"created": str(core_props.created) if core_props.created else "",
"modified": str(core_props.modified) if core_props.modified else "",
"paragraph_count": len(paragraphs),
"table_count": len(doc.tables),
}
return ParseResult(
success=True,
content=content,
file_type=FileType.WORD,
filename=filename,
mime_type=mime_type,
file_size=file_size,
metadata=metadata,
warnings=warnings,
)
def _parse_doc_legacy(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
return ParseResult(
success=True,
content="[Legacy .doc format - conversion required for full text extraction]",
file_type=FileType.WORD,
filename=filename,
mime_type=mime_type,
file_size=file_size,
metadata={"format": "legacy_doc", "requires_conversion": True},
warnings=["Legacy .doc format has limited support. Consider converting to .docx"],
)
def _parse_rtf(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
try:
text = data.decode("utf-8", errors="replace")
text = re.sub(r"\\[a-z]+\d*\s?", "", text)
text = re.sub(r"\{[^}]*\}", "", text)
text = text.replace("\\par", "\n").replace("\\tab", "\t")
text = re.sub(r"[{}]", "", text)
content = text.strip()
except Exception as e:
raise ParseError(f"Failed to parse RTF: {str(e)}", "RTF_PARSE_ERROR")
warnings = self._check_script_injection(content)
return ParseResult(
success=True,
content=content,
file_type=FileType.WORD,
filename=filename,
mime_type=mime_type,
file_size=file_size,
metadata={"format": "rtf"},
warnings=warnings,
)
def _parse_odt(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
try:
import zipfile
from xml.etree import ElementTree
except ImportError:
raise ParseError("Required libraries not available", "MISSING_DEPENDENCY")
try:
with zipfile.ZipFile(io.BytesIO(data)) as zf:
content_xml = zf.read("content.xml")
tree = ElementTree.fromstring(content_xml)
ns = {"text": "urn:oasis:names:tc:opendocument:xmlns:text:1.0"}
paragraphs = tree.findall(".//text:p", ns)
text_parts = []
for p in paragraphs:
text = "".join(p.itertext())
if text.strip():
text_parts.append(text)
content = "\n".join(text_parts)
except Exception as e:
raise ParseError(f"Failed to parse ODT: {str(e)}", "ODT_PARSE_ERROR")
warnings = self._check_script_injection(content)
return ParseResult(
success=True,
content=content,
file_type=FileType.WORD,
filename=filename,
mime_type=mime_type,
file_size=file_size,
metadata={"format": "odt"},
warnings=warnings,
)
def _parse_excel(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
ext = Path(filename).suffix.lower()
if ext == ".csv":
return self._parse_csv(data, filename, mime_type, file_size)
elif ext in (".xlsx", ".xlsm", ".xlsb"):
return self._parse_xlsx(data, filename, mime_type, file_size)
elif ext == ".xls":
return self._parse_xls_legacy(data, filename, mime_type, file_size)
elif ext == ".ods":
return self._parse_ods(data, filename, mime_type, file_size)
else:
raise ParseError(f"Unsupported Excel format: {ext}", "UNSUPPORTED_FORMAT")
def _parse_csv(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
try:
detected = chardet.detect(data[:10000])
encoding = detected.get("encoding") or "utf-8"
content = data.decode(encoding, errors="replace")
except Exception:
content = data.decode("utf-8", errors="replace")
lines = content.splitlines()
if len(lines) > self._config.max_excel_rows:
lines = lines[:self._config.max_excel_rows]
truncated = True
else:
truncated = False
content = "\n".join(lines)
warnings = self._check_script_injection(content)
metadata = {
"format": "csv",
"row_count": len(lines),
"truncated": truncated,
}
if truncated:
warnings.append(f"Rows limited to {self._config.max_excel_rows:,}")
return ParseResult(
success=True,
content=content,
file_type=FileType.EXCEL,
filename=filename,
mime_type=mime_type,
file_size=file_size,
metadata=metadata,
warnings=warnings,
)
def _parse_xlsx(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
try:
import openpyxl
except ImportError:
raise ParseError("openpyxl not installed", "MISSING_DEPENDENCY")
try:
wb = openpyxl.load_workbook(io.BytesIO(data), read_only=True, data_only=True)
except Exception as e:
raise ParseError(f"Invalid or corrupted Excel file: {str(e)}", "INVALID_EXCEL")
sheets_to_process = wb.sheetnames[:self._config.max_excel_sheets]
content_parts: List[str] = []
total_rows = 0
for sheet_name in sheets_to_process:
ws = wb[sheet_name]
sheet_content = [f"=== Sheet: {sheet_name} ==="]
row_count = 0
for row in ws.iter_rows(values_only=True):
if total_rows >= self._config.max_excel_rows:
break
cells = [str(cell) if cell is not None else "" for cell in row]
if any(c.strip() for c in cells):
sheet_content.append(" | ".join(cells))
row_count += 1
total_rows += 1
if row_count > 0:
content_parts.extend(sheet_content)
content_parts.append("")
wb.close()
content = "\n".join(content_parts)
warnings = self._check_script_injection(content)
metadata = {
"format": "xlsx",
"sheet_count": len(wb.sheetnames),
"sheets_processed": len(sheets_to_process),
"total_rows": total_rows,
"truncated": total_rows >= self._config.max_excel_rows,
}
if total_rows >= self._config.max_excel_rows:
warnings.append(f"Rows limited to {self._config.max_excel_rows:,}")
return ParseResult(
success=True,
content=content,
file_type=FileType.EXCEL,
filename=filename,
mime_type=mime_type,
file_size=file_size,
metadata=metadata,
warnings=warnings,
)
def _parse_xls_legacy(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
return ParseResult(
success=True,
content="[Legacy .xls format - conversion required for full data extraction]",
file_type=FileType.EXCEL,
filename=filename,
mime_type=mime_type,
file_size=file_size,
metadata={"format": "legacy_xls", "requires_conversion": True},
warnings=["Legacy .xls format has limited support. Consider converting to .xlsx"],
)
def _parse_ods(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
try:
import zipfile
from xml.etree import ElementTree
except ImportError:
raise ParseError("Required libraries not available", "MISSING_DEPENDENCY")
try:
with zipfile.ZipFile(io.BytesIO(data)) as zf:
content_xml = zf.read("content.xml")
tree = ElementTree.fromstring(content_xml)
ns = {
"table": "urn:oasis:names:tc:opendocument:xmlns:table:1.0",
"text": "urn:oasis:names:tc:opendocument:xmlns:text:1.0",
}
tables = tree.findall(".//table:table", ns)
content_parts: List[str] = []
for table in tables[:self._config.max_excel_sheets]:
name = table.get("{urn:oasis:names:tc:opendocument:xmlns:table:1.0}name", "Sheet")
content_parts.append(f"=== Sheet: {name} ===")
rows = table.findall(".//table:table-row", ns)
for row in rows[:self._config.max_excel_rows]:
cells = row.findall("table:table-cell", ns)
cell_texts = []
for cell in cells:
text = "".join(cell.itertext())
cell_texts.append(text.strip())
if any(cell_texts):
content_parts.append(" | ".join(cell_texts))
content_parts.append("")
content = "\n".join(content_parts)
except Exception as e:
raise ParseError(f"Failed to parse ODS: {str(e)}", "ODS_PARSE_ERROR")
warnings = self._check_script_injection(content)
return ParseResult(
success=True,
content=content,
file_type=FileType.EXCEL,
filename=filename,
mime_type=mime_type,
file_size=file_size,
metadata={"format": "ods"},
warnings=warnings,
)
def _parse_presentation(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
ext = Path(filename).suffix.lower()
if ext == ".pptx":
return self._parse_pptx(data, filename, mime_type, file_size)
elif ext == ".ppt":
return self._parse_ppt_legacy(data, filename, mime_type, file_size)
elif ext == ".odp":
return self._parse_odp(data, filename, mime_type, file_size)
else:
raise ParseError(f"Unsupported presentation format: {ext}", "UNSUPPORTED_FORMAT")
def _parse_pptx(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
try:
from pptx import Presentation
except ImportError:
raise ParseError("python-pptx not installed", "MISSING_DEPENDENCY")
try:
prs = Presentation(io.BytesIO(data))
except Exception as e:
raise ParseError(f"Invalid or corrupted PPTX: {str(e)}", "INVALID_PPTX")
slides_content: List[str] = []
for i, slide in enumerate(prs.slides):
slide_text: List[str] = [f"--- Slide {i + 1} ---"]
for shape in slide.shapes:
if hasattr(shape, "text") and shape.text.strip():
slide_text.append(shape.text.strip())
if shape.has_table:
table_rows: List[str] = []
for row in shape.table.rows:
cells = [cell.text.strip() for cell in row.cells]
table_rows.append(" | ".join(cells))
if table_rows:
slide_text.append("[Table]\n" + "\n".join(table_rows))
if len(slide_text) > 1:
slides_content.append("\n".join(slide_text))
content = "\n\n".join(slides_content)
warnings = self._check_script_injection(content)
metadata = {
"format": "pptx",
"slide_count": len(prs.slides),
}
return ParseResult(
success=True,
content=content,
file_type=FileType.PRESENTATION,
filename=filename,
mime_type=mime_type,
file_size=file_size,
metadata=metadata,
warnings=warnings,
)
def _parse_ppt_legacy(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
return ParseResult(
success=True,
content="[Legacy .ppt format - conversion required for full text extraction]",
file_type=FileType.PRESENTATION,
filename=filename,
mime_type=mime_type,
file_size=file_size,
metadata={"format": "legacy_ppt", "requires_conversion": True},
warnings=["Legacy .ppt format has limited support. Consider converting to .pptx"],
)
def _parse_odp(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
try:
import zipfile
from xml.etree import ElementTree
except ImportError:
raise ParseError("Required libraries not available", "MISSING_DEPENDENCY")
try:
with zipfile.ZipFile(io.BytesIO(data)) as zf:
content_xml = zf.read("content.xml")
tree = ElementTree.fromstring(content_xml)
ns = {
"draw": "urn:oasis:names:tc:opendocument:xmlns:drawing:1.0",
"text": "urn:oasis:names:tc:opendocument:xmlns:text:1.0",
}
pages = tree.findall(".//draw:page", ns)
slides_content: List[str] = []
for i, page in enumerate(pages):
slide_text: List[str] = [f"--- Slide {i + 1} ---"]
for text_elem in page.findall(".//text:p", ns):
text = "".join(text_elem.itertext())
if text.strip():
slide_text.append(text.strip())
if len(slide_text) > 1:
slides_content.append("\n".join(slide_text))
content = "\n\n".join(slides_content)
except Exception as e:
raise ParseError(f"Failed to parse ODP: {str(e)}", "ODP_PARSE_ERROR")
warnings = self._check_script_injection(content)
return ParseResult(
success=True,
content=content,
file_type=FileType.PRESENTATION,
filename=filename,
mime_type=mime_type,
file_size=file_size,
metadata={"format": "odp"},
warnings=warnings,
)
def _parse_image(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
metadata: Dict[str, Any] = {
"format": Path(filename).suffix.lower().lstrip("."),
"is_image": True,
}
try:
from PIL import Image
img = Image.open(io.BytesIO(data))
metadata.update({
"width": img.width,
"height": img.height,
"mode": img.mode,
"format_detail": img.format,
})
except ImportError:
pass
except Exception:
pass
return ParseResult(
success=True,
content="",
file_type=FileType.IMAGE,
filename=filename,
mime_type=mime_type,
file_size=file_size,
metadata=metadata,
warnings=[],
)
def _parse_binary(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
return ParseResult(
success=True,
content=f"[Binary file: {filename}]\nSize: {file_size:,} bytes\nType: {mime_type}\n\n[Binary content cannot be displayed as text]",
file_type=FileType.BINARY,
filename=filename,
mime_type=mime_type,
file_size=file_size,
metadata={"format": "binary"},
warnings=[],
)
def _parse_unknown(self, data: bytes, filename: str, mime_type: str, file_size: int) -> ParseResult:
is_text = self._detect_text_content(data[:min(len(data), 8192)])
if is_text:
return self._parse_text(data, filename, mime_type, file_size)
else:
return self._parse_binary(data, filename, mime_type, file_size)
def _detect_text_content(self, sample: bytes) -> bool:
if not sample:
return False
try:
detected = chardet.detect(sample)
if detected.get("confidence", 0) > 0.7:
decoded = sample.decode(detected.get("encoding") or "utf-8")
printable_count = sum(1 for c in decoded if c.isprintable() or c.isspace())
return printable_count / len(decoded) > 0.85
except Exception:
pass
return False
_default_parser: Optional[FastParse] = None
def get_parser(config: Optional[FastParseConfig] = None) -> FastParse:
global _default_parser
if config is not None:
return FastParse(config)
if _default_parser is None:
_default_parser = FastParse()
return _default_parser
def parse(
content: Union[bytes, BinaryIO, str],
filename: str,
mime_type: Optional[str] = None,
config: Optional[FastParseConfig] = None,
) -> ParseResult:
return get_parser(config).parse(content, filename, mime_type)
def parse_file(
file_path: Union[str, Path],
config: Optional[FastParseConfig] = None,
) -> ParseResult:
return get_parser(config).parse_file(file_path)