File size: 8,101 Bytes
22087d0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
from chunkr_ai import Chunkr
from typing import Dict, Any

from core.agentpress.tool import ToolResult, openapi_schema
from core.agentpress.thread_manager import ThreadManager
from core.sandbox.tool_base import SandboxToolsBase
from core.utils.logger import logger


class SandboxDocumentParserTool(SandboxToolsBase):
    def __init__(self, project_id: str, thread_manager: ThreadManager):
        super().__init__(project_id, thread_manager)
        self.chunkr = Chunkr()

    @openapi_schema({
        "type": "function",
        "function": {
            "name": "parse_document",
            "description": "Parse any document type from a URL and extract text, metadata, tables, and structured data. Auto-detects format from URL.",
            "parameters": {
                "type": "object",
                "properties": {
                    "url": {
                        "type": "string",
                        "description": "URL to the document file to parse (must start with http:// or https://)"
                    },
                    "extract_tables": {
                        "type": "boolean",
                        "description": "Whether to extract tables from the document",
                        "default": False
                    },
                    "extract_structured_data": {
                        "type": "boolean",
                        "description": "Whether to extract structured data like forms, key-value pairs, etc.",
                        "default": False
                    }
                },
                "required": ["url"]
            }
        }
    })
    async def parse_document(
        self, 
        url: str, 
        extract_tables: bool = False, 
        extract_structured_data: bool = False,
        options: Dict[str, Any] = None
    ) -> ToolResult:
        try:
            await self._ensure_sandbox()
            
            if options is None:
                options = {}
            
            # Validate that the input is a URL
            if not url.startswith(('http://', 'https://')):
                return self.fail_response(f"Invalid URL: {url}. URL must start with http:// or https://")
            
            logger.info(f"Parsing document from URL: {url}")
            
            logger.debug(f"Uploading to Chunkr AI: {url}")
            
            # Upload to Chunkr and get the task result
            task = await self.chunkr.upload(url)
            
            logger.debug(f"Chunkr task completed successfully")
            
            # Extract meaningful content from the Chunkr response
            parsed_content = self._extract_meaningful_content(task, extract_tables, extract_structured_data)
            
            return self.success_response({
                "message": f"Successfully parsed document from URL: {url}",
                "content": parsed_content
            })
            
        except Exception as e:
            logger.error(f"Error parsing document from URL {url}: {e}", exc_info=True)
            return self.fail_response(f"Error parsing document from URL: {str(e)}")
    
    def _extract_meaningful_content(self, task, extract_tables: bool, extract_structured_data: bool) -> Dict[str, Any]:
        """Extract meaningful content from Chunkr task response without overwhelming detail."""
        try:
            content = {
                "document_info": {},
                "text_content": [],
                "structure": [],
                "tables": [],
                "metadata": {}
            }
            
            # Check if task has output
            if not hasattr(task, 'output') or not task.output:
                return {"error": "No content found in document"}
            
            output = task.output
            
            # Extract basic document info
            content["document_info"] = {
                "total_chunks": len(output.chunks) if hasattr(output, 'chunks') else 0,
                "status": "completed",
                "processing_time": str(getattr(task, 'finished_at', 'N/A'))
            }
            
            # Process chunks to extract text and structure
            if hasattr(output, 'chunks') and output.chunks:
                for chunk in output.chunks:
                    if hasattr(chunk, 'segments'):
                        for segment in chunk.segments:
                            segment_info = {
                                "type": str(getattr(segment, 'segment_type', 'unknown')),
                                "content": getattr(segment, 'content', ''),
                                "page": getattr(segment, 'page_number', 1)
                            }
                            
                            # Clean up content (remove excessive dots and formatting artifacts)
                            clean_content = self._clean_content(segment_info["content"])
                            
                            if clean_content and len(clean_content.strip()) > 3:  # Skip very short/empty content
                                segment_info["content"] = clean_content
                                
                                # Categorize by segment type
                                seg_type = segment_info["type"].lower()
                                
                                if 'title' in seg_type:
                                    content["structure"].append({
                                        "type": "heading",
                                        "content": clean_content,
                                        "page": segment_info["page"]
                                    })
                                elif 'table' in seg_type and extract_tables:
                                    # Extract table data if available
                                    table_data = {
                                        "content": clean_content,
                                        "page": segment_info["page"]
                                    }
                                    if hasattr(segment, 'html') and segment.html:
                                        table_data["html"] = segment.html
                                    content["tables"].append(table_data)
                                elif 'text' in seg_type:
                                    content["text_content"].append({
                                        "text": clean_content,
                                        "page": segment_info["page"]
                                    })
            
            # Summarize content for easier consumption
            content["summary"] = self._create_content_summary(content)
            
            return content
            
        except Exception as e:
            logger.error(f"Error extracting content from Chunkr response: {e}", exc_info=True)
            return {"error": f"Failed to extract content: {str(e)}"}
    
    def _clean_content(self, content: str) -> str:
        """Clean up content by removing excessive formatting artifacts."""
        if not content:
            return ""
        
        # Remove excessive dots (common in TOC entries)
        content = content.strip()
        
        # Skip content that's mostly dots
        if content.count('.') > len(content) * 0.5:
            return ""
        
        # Remove standalone dots and clean up spacing
        if content in ['.', '..', '...'] or len(content.strip('.').strip()) == 0:
            return ""
        
        return content
    
    def _create_content_summary(self, content: Dict[str, Any]) -> Dict[str, Any]:
        """Create a summary of the extracted content."""
        return {
            "total_pages": max([item.get("page", 1) for item in content["text_content"]] + [1]),
            "headings_count": len(content["structure"]),
            "text_sections": len(content["text_content"]),
            "tables_count": len(content["tables"]),
            "main_headings": [item["content"] for item in content["structure"][:5]]  # First 5 headings
        }