nniback / core /tools /sb_document_parser.py
Kyrlos11's picture
Upload 842 files
22087d0 verified
Raw History Blame Contribute Delete
8.1 kB
from chunkr_ai import Chunkr
from typing import Dict, Any
from core.agentpress.tool import ToolResult, openapi_schema
from core.agentpress.thread_manager import ThreadManager
from core.sandbox.tool_base import SandboxToolsBase
from core.utils.logger import logger
class SandboxDocumentParserTool(SandboxToolsBase):
def __init__(self, project_id: str, thread_manager: ThreadManager):
super().__init__(project_id, thread_manager)
self.chunkr = Chunkr()
@openapi_schema({
"type": "function",
"function": {
"name": "parse_document",
"description": "Parse any document type from a URL and extract text, metadata, tables, and structured data. Auto-detects format from URL.",
"parameters": {
"type": "object",
"properties": {
"url": {
"type": "string",
"description": "URL to the document file to parse (must start with http:// or https://)"
},
"extract_tables": {
"type": "boolean",
"description": "Whether to extract tables from the document",
"default": False
},
"extract_structured_data": {
"type": "boolean",
"description": "Whether to extract structured data like forms, key-value pairs, etc.",
"default": False
}
},
"required": ["url"]
}
}
})
async def parse_document(
self,
url: str,
extract_tables: bool = False,
extract_structured_data: bool = False,
options: Dict[str, Any] = None
) -> ToolResult:
try:
await self._ensure_sandbox()
if options is None:
options = {}
# Validate that the input is a URL
if not url.startswith(('http://', 'https://')):
return self.fail_response(f"Invalid URL: {url}. URL must start with http:// or https://")
logger.info(f"Parsing document from URL: {url}")
logger.debug(f"Uploading to Chunkr AI: {url}")
# Upload to Chunkr and get the task result
task = await self.chunkr.upload(url)
logger.debug(f"Chunkr task completed successfully")
# Extract meaningful content from the Chunkr response
parsed_content = self._extract_meaningful_content(task, extract_tables, extract_structured_data)
return self.success_response({
"message": f"Successfully parsed document from URL: {url}",
"content": parsed_content
})
except Exception as e:
logger.error(f"Error parsing document from URL {url}: {e}", exc_info=True)
return self.fail_response(f"Error parsing document from URL: {str(e)}")
def _extract_meaningful_content(self, task, extract_tables: bool, extract_structured_data: bool) -> Dict[str, Any]:
"""Extract meaningful content from Chunkr task response without overwhelming detail."""
try:
content = {
"document_info": {},
"text_content": [],
"structure": [],
"tables": [],
"metadata": {}
}
# Check if task has output
if not hasattr(task, 'output') or not task.output:
return {"error": "No content found in document"}
output = task.output
# Extract basic document info
content["document_info"] = {
"total_chunks": len(output.chunks) if hasattr(output, 'chunks') else 0,
"status": "completed",
"processing_time": str(getattr(task, 'finished_at', 'N/A'))
}
# Process chunks to extract text and structure
if hasattr(output, 'chunks') and output.chunks:
for chunk in output.chunks:
if hasattr(chunk, 'segments'):
for segment in chunk.segments:
segment_info = {
"type": str(getattr(segment, 'segment_type', 'unknown')),
"content": getattr(segment, 'content', ''),
"page": getattr(segment, 'page_number', 1)
}
# Clean up content (remove excessive dots and formatting artifacts)
clean_content = self._clean_content(segment_info["content"])
if clean_content and len(clean_content.strip()) > 3: # Skip very short/empty content
segment_info["content"] = clean_content
# Categorize by segment type
seg_type = segment_info["type"].lower()
if 'title' in seg_type:
content["structure"].append({
"type": "heading",
"content": clean_content,
"page": segment_info["page"]
})
elif 'table' in seg_type and extract_tables:
# Extract table data if available
table_data = {
"content": clean_content,
"page": segment_info["page"]
}
if hasattr(segment, 'html') and segment.html:
table_data["html"] = segment.html
content["tables"].append(table_data)
elif 'text' in seg_type:
content["text_content"].append({
"text": clean_content,
"page": segment_info["page"]
})
# Summarize content for easier consumption
content["summary"] = self._create_content_summary(content)
return content
except Exception as e:
logger.error(f"Error extracting content from Chunkr response: {e}", exc_info=True)
return {"error": f"Failed to extract content: {str(e)}"}
def _clean_content(self, content: str) -> str:
"""Clean up content by removing excessive formatting artifacts."""
if not content:
return ""
# Remove excessive dots (common in TOC entries)
content = content.strip()
# Skip content that's mostly dots
if content.count('.') > len(content) * 0.5:
return ""
# Remove standalone dots and clean up spacing
if content in ['.', '..', '...'] or len(content.strip('.').strip()) == 0:
return ""
return content
def _create_content_summary(self, content: Dict[str, Any]) -> Dict[str, Any]:
"""Create a summary of the extracted content."""
return {
"total_pages": max([item.get("page", 1) for item in content["text_content"]] + [1]),
"headings_count": len(content["structure"]),
"text_sections": len(content["text_content"]),
"tables_count": len(content["tables"]),
"main_headings": [item["content"] for item in content["structure"][:5]] # First 5 headings
}