File size: 12,936 Bytes
c86be6a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
import fitz  # PyMuPDF - keep for fallback
import tiktoken
import os
from typing import List, Dict, Any
from pathlib import Path
import logging

# SmolDocling imports
import torch
from docling_core.types.doc import DoclingDocument
from docling_core.types.doc.document import DocTagsDocument
from transformers import AutoProcessor, AutoModelForVision2Seq
from pdf2image import convert_from_path
from PIL import Image
import tempfile

logger = logging.getLogger(__name__)

class PDFProcessor:
    def __init__(self):
        self.encoding = tiktoken.encoding_for_model("gpt-3.5-turbo")
        self.max_tokens = int(os.getenv("MAX_TOKENS", "180000"))
        self.chunk_size = int(os.getenv("CHUNK_SIZE", "8000"))
        
        # Initialize SmolDocling model
        self.device = "cuda" if torch.cuda.is_available() else "cpu"
        self.model_path = "ds4sd/SmolDocling-256M-preview"
        
        # Load SmolDocling model and processor
        try:
            logger.info(f"Loading SmolDocling model on {self.device}")
            self.processor = AutoProcessor.from_pretrained(self.model_path)
            self.model = AutoModelForVision2Seq.from_pretrained(
                self.model_path,
                torch_dtype=torch.bfloat16 if self.device == "cuda" else torch.float32,
                _attn_implementation="flash_attention_2" if self.device == "cuda" else "eager",
            ).to(self.device)
            logger.info("SmolDocling model loaded successfully")
        except Exception as e:
            logger.error(f"Failed to load SmolDocling model: {str(e)}")
            logger.info("Falling back to PyMuPDF for extraction")
            self.model = None
            self.processor = None
    
    def extract_text(self, pdf_path: Path) -> str:
        """Extract text from PDF using SmolDocling (with PyMuPDF fallback)"""
        try:
            if self.model is None or self.processor is None:
                logger.info("Using PyMuPDF fallback for text extraction")
                return self._extract_text_pymupdf(pdf_path)
            
            logger.info(f"Extracting text from PDF using SmolDocling: {pdf_path}")
            
            # Convert PDF to images
            images = convert_from_path(str(pdf_path))
            
            all_text = ""
            
            for page_num, image in enumerate(images):
                logger.info(f"Processing page {page_num + 1}/{len(images)}")
                
                # Create input messages for SmolDocling
                messages = [
                    {
                        "role": "user",
                        "content": [
                            {"type": "image"},
                            {"type": "text", "text": "Convert this page to docling."}
                        ]
                    },
                ]
                
                # Prepare inputs
                prompt = self.processor.apply_chat_template(messages, add_generation_prompt=True)
                inputs = self.processor(text=prompt, images=[image], return_tensors="pt")
                inputs = inputs.to(self.device)
                
                # Generate outputs
                with torch.no_grad():
                    generated_ids = self.model.generate(**inputs, max_new_tokens=8192)
                    prompt_length = inputs.input_ids.shape[1]
                    trimmed_generated_ids = generated_ids[:, prompt_length:]
                    doctags = self.processor.batch_decode(
                        trimmed_generated_ids,
                        skip_special_tokens=False,
                    )[0].lstrip()
                
                # Convert DocTags to text
                try:
                    doctags_doc = DocTagsDocument.from_doctags_and_image_pairs([doctags], [image])
                    doc = DoclingDocument(name=f"Page_{page_num + 1}")
                    doc.load_from_doctags(doctags_doc)
                    
                    # Export as markdown and extract text
                    page_text = doc.export_to_markdown()
                    
                    # Add page separator and content
                    all_text += f"\n--- Page {page_num + 1} ---\n{page_text}\n"
                    
                except Exception as e:
                    logger.warning(f"Failed to convert DocTags for page {page_num + 1}: {str(e)}")
                    # Fallback: extract text directly from DocTags
                    page_text = self._extract_text_from_doctags(doctags)
                    all_text += f"\n--- Page {page_num + 1} ---\n{page_text}\n"
            
            logger.info(f"Successfully extracted text from {len(images)} pages using SmolDocling")
            return all_text.strip()
            
        except Exception as e:
            logger.error(f"Error extracting text with SmolDocling: {str(e)}")
            logger.info("Falling back to PyMuPDF")
            return self._extract_text_pymupdf(pdf_path)
    
    def _extract_text_pymupdf(self, pdf_path: Path) -> str:
        """Fallback method using PyMuPDF"""
        try:
            doc = fitz.open(pdf_path)
            text = ""
            
            for page_num in range(len(doc)):
                page = doc.load_page(page_num)
                page_text = page.get_text()
                text += f"\n--- Page {page_num + 1} ---\n{page_text}\n"
            
            doc.close()
            return text.strip()
            
        except Exception as e:
            logger.error(f"Error extracting text from PDF with PyMuPDF: {str(e)}")
            raise Exception(f"Failed to extract text from PDF: {str(e)}")
    
    def _extract_text_from_doctags(self, doctags: str) -> str:
        """Extract plain text from DocTags as fallback"""
        try:
            # Simple text extraction from DocTags
            import re
            
            # Remove XML-like tags and extract text content
            text = re.sub(r'<[^>]+>', '', doctags)
            text = re.sub(r'\s+', ' ', text)  # Normalize whitespace
            return text.strip()
            
        except Exception as e:
            logger.warning(f"Failed to extract text from DocTags: {str(e)}")
            return "Failed to extract text from this page"
    
    def count_tokens(self, text: str) -> int:
        """Count tokens in text"""
        return len(self.encoding.encode(text))
    
    def chunk_text(self, text: str) -> List[Dict[str, Any]]:
        """Split text into chunks based on token limits with proper management"""
        chunks = []
        
        # Calculate total tokens for the entire document
        total_tokens = self.count_tokens(text)
        logger.info(f"Total document tokens: {total_tokens}")
        
        # If text is within token limit, return as single chunk
        if total_tokens <= self.chunk_size:
            return [{
                "chunk_id": 0,
                "text": text,
                "tokens": total_tokens,
                "page_range": "all",
                "original_length": len(text)
            }]
        
        # Split by pages first
        pages = text.split("--- Page ")
        current_chunk = ""
        current_tokens = 0
        chunk_id = 0
        start_page = 1
        total_processed_tokens = 0
        
        logger.info(f"Processing {len(pages)-1} pages into chunks")
        
        for i, page in enumerate(pages):
            if i == 0:  # Skip empty first split
                continue
                
            page_text = f"--- Page {page}"
            page_tokens = self.count_tokens(page_text)
            
            # If single page exceeds chunk size, split it further
            if page_tokens > self.chunk_size:
                logger.info(f"Page {i} has {page_tokens} tokens, splitting further")
                
                # Save current chunk if it has content
                if current_chunk:
                    chunks.append({
                        "chunk_id": chunk_id,
                        "text": current_chunk,
                        "tokens": current_tokens,
                        "page_range": f"{start_page}-{i-1}",
                        "original_length": len(current_chunk)
                    })
                    total_processed_tokens += current_tokens
                    chunk_id += 1
                
                # Split large page into smaller chunks
                page_chunks = self._split_large_page(page_text, page_tokens, chunk_id, i)
                chunks.extend(page_chunks)
                chunk_id += len(page_chunks)
                total_processed_tokens += page_tokens
                
                # Reset for next chunk
                current_chunk = ""
                current_tokens = 0
                start_page = i + 1
                
            # If adding this page would exceed chunk size, save current chunk
            elif current_tokens + page_tokens > self.chunk_size:
                if current_chunk:
                    chunks.append({
                        "chunk_id": chunk_id,
                        "text": current_chunk,
                        "tokens": current_tokens,
                        "page_range": f"{start_page}-{i-1}",
                        "original_length": len(current_chunk)
                    })
                    total_processed_tokens += current_tokens
                    chunk_id += 1
                
                # Start new chunk with current page
                current_chunk = page_text
                current_tokens = page_tokens
                start_page = i
                
            else:
                # Add page to current chunk
                if current_chunk:
                    current_chunk += "\n" + page_text
                else:
                    current_chunk = page_text
                current_tokens += page_tokens
        
        # Add final chunk if it has content
        if current_chunk:
            chunks.append({
                "chunk_id": chunk_id,
                "text": current_chunk,
                "tokens": current_tokens,
                "page_range": f"{start_page}-{len(pages)-1}",
                "original_length": len(current_chunk)
            })
            total_processed_tokens += current_tokens
        
        logger.info(f"Created {len(chunks)} chunks, total processed tokens: {total_processed_tokens}")
        
        # Verify we didn't lose content
        if abs(total_processed_tokens - total_tokens) > 100:  # Allow small variance
            logger.warning(f"Token count mismatch: original={total_tokens}, processed={total_processed_tokens}")
        
        return chunks
    
    def _split_large_page(self, page_text: str, page_tokens: int, start_chunk_id: int, page_num: int) -> List[Dict[str, Any]]:
        """Split a large page into smaller chunks"""
        chunks = []
        lines = page_text.split('\n')
        current_chunk = ""
        current_tokens = 0
        chunk_id = start_chunk_id
        
        logger.info(f"Splitting page {page_num} with {page_tokens} tokens into smaller chunks")
        
        for line in lines:
            line_tokens = self.count_tokens(line)
            
            if current_tokens + line_tokens > self.chunk_size:
                if current_chunk:
                    chunks.append({
                        "chunk_id": chunk_id,
                        "text": current_chunk,
                        "tokens": current_tokens,
                        "page_range": f"page-{page_num}-part-{chunk_id-start_chunk_id+1}",
                        "original_length": len(current_chunk)
                    })
                    chunk_id += 1
                
                current_chunk = line
                current_tokens = line_tokens
            else:
                current_chunk += "\n" + line if current_chunk else line
                current_tokens += line_tokens
        
        # Add final chunk
        if current_chunk:
            chunks.append({
                "chunk_id": chunk_id,
                "text": current_chunk,
                "tokens": current_tokens,
                "page_range": f"page-{page_num}-part-{chunk_id-start_chunk_id+1}",
                "original_length": len(current_chunk)
            })
        
        logger.info(f"Split page {page_num} into {len(chunks)} chunks")
        return chunks
    
    def get_text_preview(self, text: str, max_chars: int = 500) -> str:
        """Get a preview of the text"""
        if len(text) <= max_chars:
            return text
        return text[:max_chars] + "..."