File size: 8,101 Bytes
7cfdc4c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 | from chunkr_ai import Chunkr
from typing import Dict, Any
from core.agentpress.tool import ToolResult, openapi_schema
from core.agentpress.thread_manager import ThreadManager
from core.sandbox.tool_base import SandboxToolsBase
from core.utils.logger import logger
class SandboxDocumentParserTool(SandboxToolsBase):
def __init__(self, project_id: str, thread_manager: ThreadManager):
super().__init__(project_id, thread_manager)
self.chunkr = Chunkr()
@openapi_schema({
"type": "function",
"function": {
"name": "parse_document",
"description": "Parse any document type from a URL and extract text, metadata, tables, and structured data. Auto-detects format from URL.",
"parameters": {
"type": "object",
"properties": {
"url": {
"type": "string",
"description": "URL to the document file to parse (must start with http:// or https://)"
},
"extract_tables": {
"type": "boolean",
"description": "Whether to extract tables from the document",
"default": False
},
"extract_structured_data": {
"type": "boolean",
"description": "Whether to extract structured data like forms, key-value pairs, etc.",
"default": False
}
},
"required": ["url"]
}
}
})
async def parse_document(
self,
url: str,
extract_tables: bool = False,
extract_structured_data: bool = False,
options: Dict[str, Any] = None
) -> ToolResult:
try:
await self._ensure_sandbox()
if options is None:
options = {}
# Validate that the input is a URL
if not url.startswith(('http://', 'https://')):
return self.fail_response(f"Invalid URL: {url}. URL must start with http:// or https://")
logger.info(f"Parsing document from URL: {url}")
logger.debug(f"Uploading to Chunkr AI: {url}")
# Upload to Chunkr and get the task result
task = await self.chunkr.upload(url)
logger.debug(f"Chunkr task completed successfully")
# Extract meaningful content from the Chunkr response
parsed_content = self._extract_meaningful_content(task, extract_tables, extract_structured_data)
return self.success_response({
"message": f"Successfully parsed document from URL: {url}",
"content": parsed_content
})
except Exception as e:
logger.error(f"Error parsing document from URL {url}: {e}", exc_info=True)
return self.fail_response(f"Error parsing document from URL: {str(e)}")
def _extract_meaningful_content(self, task, extract_tables: bool, extract_structured_data: bool) -> Dict[str, Any]:
"""Extract meaningful content from Chunkr task response without overwhelming detail."""
try:
content = {
"document_info": {},
"text_content": [],
"structure": [],
"tables": [],
"metadata": {}
}
# Check if task has output
if not hasattr(task, 'output') or not task.output:
return {"error": "No content found in document"}
output = task.output
# Extract basic document info
content["document_info"] = {
"total_chunks": len(output.chunks) if hasattr(output, 'chunks') else 0,
"status": "completed",
"processing_time": str(getattr(task, 'finished_at', 'N/A'))
}
# Process chunks to extract text and structure
if hasattr(output, 'chunks') and output.chunks:
for chunk in output.chunks:
if hasattr(chunk, 'segments'):
for segment in chunk.segments:
segment_info = {
"type": str(getattr(segment, 'segment_type', 'unknown')),
"content": getattr(segment, 'content', ''),
"page": getattr(segment, 'page_number', 1)
}
# Clean up content (remove excessive dots and formatting artifacts)
clean_content = self._clean_content(segment_info["content"])
if clean_content and len(clean_content.strip()) > 3: # Skip very short/empty content
segment_info["content"] = clean_content
# Categorize by segment type
seg_type = segment_info["type"].lower()
if 'title' in seg_type:
content["structure"].append({
"type": "heading",
"content": clean_content,
"page": segment_info["page"]
})
elif 'table' in seg_type and extract_tables:
# Extract table data if available
table_data = {
"content": clean_content,
"page": segment_info["page"]
}
if hasattr(segment, 'html') and segment.html:
table_data["html"] = segment.html
content["tables"].append(table_data)
elif 'text' in seg_type:
content["text_content"].append({
"text": clean_content,
"page": segment_info["page"]
})
# Summarize content for easier consumption
content["summary"] = self._create_content_summary(content)
return content
except Exception as e:
logger.error(f"Error extracting content from Chunkr response: {e}", exc_info=True)
return {"error": f"Failed to extract content: {str(e)}"}
def _clean_content(self, content: str) -> str:
"""Clean up content by removing excessive formatting artifacts."""
if not content:
return ""
# Remove excessive dots (common in TOC entries)
content = content.strip()
# Skip content that's mostly dots
if content.count('.') > len(content) * 0.5:
return ""
# Remove standalone dots and clean up spacing
if content in ['.', '..', '...'] or len(content.strip('.').strip()) == 0:
return ""
return content
def _create_content_summary(self, content: Dict[str, Any]) -> Dict[str, Any]:
"""Create a summary of the extracted content."""
return {
"total_pages": max([item.get("page", 1) for item in content["text_content"]] + [1]),
"headings_count": len(content["structure"]),
"text_sections": len(content["text_content"]),
"tables_count": len(content["tables"]),
"main_headings": [item["content"] for item in content["structure"][:5]] # First 5 headings
} |