File size: 16,113 Bytes
4244db2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
"""

Optimized text-based chunking without GPU dependencies.

Fast, memory-efficient, and reliable chunking for document processing.

"""

import re
import math
from typing import List, Dict, Optional, Any
from dataclasses import dataclass
import hashlib
import time

@dataclass
class OptimizedChunkConfig:
    """Configuration for optimized text chunking."""
    chunk_size: int = 800       # Target characters per chunk
    chunk_overlap: int = 250       # Overlap between chunks in characters
    min_chunk_size: int = 200      # Minimum chunk size
    max_chunk_size: int = 1500    # Maximum chunk size
    split_on_sentences: bool = True # Try to split on sentence boundaries
    preserve_paragraphs: bool = True # Try to keep paragraphs intact
    separate_tables: bool = True    # Create separate chunks for tables


class OptimizedTextChunker:
    """

    Fast, CPU-only text chunker optimized for performance and simplicity.

    

    Features:

    - Pure text-based chunking (no embeddings/GPU)

    - Sentence and paragraph boundary awareness

    - Configurable overlap and sizing

    - Memory efficient processing

    - Fast execution for large documents

    """
    
    def __init__(self, config: Optional[OptimizedChunkConfig] = None):
        self.config = config or OptimizedChunkConfig()
        print(f"βœ… OptimizedTextChunker initialized (CPU-only, {self.config.chunk_size} chars/chunk)")
        if self.config.separate_tables:
            print(f"πŸ“Š Table separation enabled - tables will get dedicated chunks")
    
    def _extract_tables_and_text(self, content: str) -> tuple:
        """

        Extract tables and remaining text separately.

        Returns: (tables_list, text_without_tables)

        """
        tables = []
        remaining_text = content
        
        # Find all table sections
        table_pattern = r'TABLE:\n(.*?)(?=\n\n|\n(?=[A-Z][A-Za-z\s]*:)|$)'
        table_matches = re.finditer(table_pattern, content, re.DOTALL)
        
        # Extract tables from end to beginning to preserve indices
        table_positions = []
        for match in table_matches:
            table_content = match.group(1).strip()
            if table_content:  # Only if table has content
                tables.append({
                    'content': f"TABLE:\n{table_content}",
                    'start': match.start(),
                    'end': match.end(),
                    'type': 'table'
                })
                table_positions.append((match.start(), match.end()))
        
        # Remove tables from text (process from end to beginning)
        table_positions.sort(reverse=True)
        for start, end in table_positions:
            remaining_text = remaining_text[:start] + remaining_text[end:]
        
        # Clean up remaining text
        remaining_text = re.sub(r'\n{3,}', '\n\n', remaining_text).strip()
        
        return tables, remaining_text
    
    def _split_into_sentences(self, text: str) -> List[str]:
        """Split text into sentences using regex patterns."""
        # Enhanced sentence splitting pattern
        sentence_pattern = r'(?<=[.!?])\s+(?=[A-Z])'
        sentences = re.split(sentence_pattern, text)
        
        # Clean and filter sentences
        cleaned_sentences = []
        for sentence in sentences:
            sentence = sentence.strip()
            if sentence and len(sentence) > 10:  # Filter very short fragments
                cleaned_sentences.append(sentence)
        
        return cleaned_sentences
    
    def _split_into_paragraphs(self, text: str) -> List[str]:
        """Split text into paragraphs."""
        # Split on double newlines or more
        paragraphs = re.split(r'\n\s*\n', text)
        
        # Clean and filter paragraphs
        cleaned_paragraphs = []
        for paragraph in paragraphs:
            paragraph = paragraph.strip()
            if paragraph and len(paragraph) > 20:  # Filter very short paragraphs
                cleaned_paragraphs.append(paragraph)
        
        return cleaned_paragraphs
    
    def _create_overlapping_chunks(self, text_units: List[str], unit_type: str = "sentence") -> List[str]:
        """Create overlapping chunks from text units."""
        chunks = []
        current_chunk = ""
        current_length = 0
        overlap_buffer = []
        
        for i, unit in enumerate(text_units):
            unit_length = len(unit)
            
            # Check if adding this unit would exceed max chunk size
            if current_length + unit_length > self.config.max_chunk_size and current_chunk:
                # Finalize current chunk if it meets minimum size
                if current_length >= self.config.min_chunk_size:
                    chunks.append(current_chunk.strip())
                    
                    # Create overlap for next chunk
                    overlap_text = ""
                    overlap_length = 0
                    
                    # Add units to overlap buffer (from end of current chunk)
                    for j in range(len(overlap_buffer) - 1, -1, -1):
                        if overlap_length + len(overlap_buffer[j]) <= self.config.chunk_overlap:
                            overlap_text = overlap_buffer[j] + " " + overlap_text
                            overlap_length += len(overlap_buffer[j])
                        else:
                            break
                    
                    # Start new chunk with overlap
                    current_chunk = overlap_text.strip()
                    current_length = len(current_chunk)
                    overlap_buffer = []
                else:
                    # Current chunk too small, continue building
                    pass
            
            # Add current unit to chunk
            if current_chunk:
                current_chunk += " " + unit
            else:
                current_chunk = unit
            
            current_length += unit_length + 1  # +1 for space
            overlap_buffer.append(unit)
            
            # Keep overlap buffer reasonable size
            if len(overlap_buffer) > 10:
                overlap_buffer.pop(0)
        
        # Add final chunk if it has content
        if current_chunk.strip() and len(current_chunk) >= self.config.min_chunk_size:
            chunks.append(current_chunk.strip())
        
        return chunks
    
    def _simple_chunk_by_size(self, text: str) -> List[str]:
        """Simple fallback chunking by character count."""
        chunks = []
        start = 0
        text_length = len(text)
        
        while start < text_length:
            # Calculate end position
            end = start + self.config.chunk_size
            
            # If this is not the last chunk, try to end at a good break point
            if end < text_length:
                # Look for sentence endings within reasonable distance
                search_start = max(start + self.config.chunk_size - 100, start)
                search_end = min(end + 100, text_length)
                
                # Find the best break point (sentence ending)
                best_break = -1
                for i in range(search_end - 1, search_start - 1, -1):
                    if text[i] in '.!?':
                        # Make sure next character is whitespace or end
                        if i + 1 >= text_length or text[i + 1].isspace():
                            best_break = i + 1
                            break
                
                if best_break > start:
                    end = best_break
            
            # Extract chunk
            chunk = text[start:end].strip()
            
            if chunk and len(chunk) >= self.config.min_chunk_size:
                chunks.append(chunk)
            
            # Move start position (with overlap)
            if end >= text_length:
                break
            
            start = max(end - self.config.chunk_overlap, start + 1)
        
        return chunks
    
    def chunk_text(self, text: str, document_name: str = "unknown") -> List[Dict[str, Any]]:
        """

        Chunk a single text document into optimized chunks with separate table handling.

        

        Args:

            text: Input text to chunk

            document_name: Name of the document for metadata

            

        Returns:

            List of chunk dictionaries

        """
        if not text or not text.strip():
            return []
        
        start_time = time.time()
        text = text.strip()
        
        print(f"πŸ“„ Chunking {document_name}: {len(text):,} characters")
        
        structured_chunks = []
        chunk_index = 0
        
        # Handle tables separately if enabled
        if self.config.separate_tables and "TABLE:" in text:
            tables, remaining_text = self._extract_tables_and_text(text)
            
            print(f"πŸ“Š Found {len(tables)} tables to chunk separately")
            
            # Create separate chunks for each table
            for table in tables:
                table_content = table['content']
                chunk_id = f"{document_name}_table_{chunk_index}_{hashlib.md5(table_content.encode()).hexdigest()[:8]}"
                
                structured_chunks.append({
                    'chunk_id': chunk_id,
                    'document_name': document_name,
                    'content': table_content,
                    'chunk_index': chunk_index,
                    'char_count': len(table_content),
                    'metadata': {
                        'chunking_method': 'table_dedicated',
                        'content_type': 'table',
                        'source_document': document_name,
                        'chunk_size_config': self.config.chunk_size,
                        'overlap_config': self.config.chunk_overlap
                    }
                })
                chunk_index += 1
            
            # Use remaining text for regular chunking
            text = remaining_text
            print(f"πŸ“ Processing remaining text: {len(text):,} characters")
        
        # Process regular text content
        if text and text.strip():
            # Try different chunking strategies based on text characteristics
            chunks = []
            
            if self.config.preserve_paragraphs and '\n\n' in text:
                # Strategy 1: Paragraph-aware chunking
                paragraphs = self._split_into_paragraphs(text)
                
                if paragraphs:
                    print(f"πŸ“ Using paragraph-based chunking ({len(paragraphs)} paragraphs)")
                    chunks = self._create_overlapping_chunks(paragraphs, "paragraph")
            
            # If paragraph chunking didn't work well, try sentence-based
            if not chunks and self.config.split_on_sentences:
                sentences = self._split_into_sentences(text)
                
                if len(sentences) > 5:  # Only if we have enough sentences
                    print(f"πŸ“ Using sentence-based chunking ({len(sentences)} sentences)")
                    chunks = self._create_overlapping_chunks(sentences, "sentence")
            
            # Fallback to simple character-based chunking
            if not chunks:
                print(f"πŸ“ Using character-based chunking (fallback)")
                chunks = self._simple_chunk_by_size(text)
            
            # Convert text chunks to structured format
            for chunk_text in chunks:
                chunk_id = f"{document_name}_{chunk_index}_{hashlib.md5(chunk_text.encode()).hexdigest()[:8]}"
                
                structured_chunks.append({
                    'chunk_id': chunk_id,
                    'document_name': document_name,
                    'content': chunk_text,
                    'chunk_index': chunk_index,
                    'char_count': len(chunk_text),
                    'metadata': {
                        'chunking_method': 'optimized_text',
                        'content_type': 'text',
                        'source_document': document_name,
                        'chunk_size_config': self.config.chunk_size,
                        'overlap_config': self.config.chunk_overlap
                    }
                })
                chunk_index += 1
        
        processing_time = time.time() - start_time
        table_count = sum(1 for chunk in structured_chunks if chunk['metadata'].get('content_type') == 'table')
        text_count = len(structured_chunks) - table_count
        
        print(f"βœ… Created {len(structured_chunks)} chunks in {processing_time:.2f}s")
        print(f"πŸ“Š Tables: {table_count} dedicated chunks")
        print(f"πŸ“ Text: {text_count} regular chunks")
        
        return structured_chunks


def chunk_documents_optimized(parsed_content: List[Dict[str, Any]], 

                            chunk_size: int = 500,

                            chunk_overlap: int = 150,

                            save_parsed_text: bool = False,

                            output_dir: str = "results") -> List[Dict[str, Any]]:
    """

    Optimized document chunking function - CPU only, fast and reliable.

    

    Args:

        parsed_content: List of parsed document dictionaries

        chunk_size: Target chunk size in characters

        chunk_overlap: Overlap between chunks in characters

        save_parsed_text: Whether to save text files (unused in optimized version)

        output_dir: Output directory (unused in optimized version)

        

    Returns:

        List of chunk dictionaries

    """
    if not parsed_content:
        return []
    
    print(f"πŸš€ Starting Optimized Text Chunking...")
    start_time = time.time()
    
    # Create optimized chunker with table separation enabled
    config = OptimizedChunkConfig(
        chunk_size=chunk_size,
        chunk_overlap=chunk_overlap,
        min_chunk_size=max(50, chunk_size // 8),  # More reasonable minimum
        max_chunk_size=chunk_size * 2,
        split_on_sentences=True,
        preserve_paragraphs=True,
        separate_tables=True  # Enable table separation
    )
    
    chunker = OptimizedTextChunker(config)
    all_chunks = []
    
    for doc_data in parsed_content:
        doc_name = doc_data.get('document_name', 'unknown')
        content = doc_data.get('content', '')
        
        if not content or not content.strip():
            print(f"⚠️ Skipping empty document: {doc_name}")
            continue
        
        # Chunk the document
        doc_chunks = chunker.chunk_text(content, doc_name)
        all_chunks.extend(doc_chunks)
    
    total_time = time.time() - start_time
    total_chars = sum(len(doc.get('content', '')) for doc in parsed_content)
    avg_chunk_size = sum(len(chunk['content']) for chunk in all_chunks) / len(all_chunks) if all_chunks else 0
    
    # Count table vs text chunks
    table_chunks = sum(1 for chunk in all_chunks if chunk['metadata'].get('content_type') == 'table')
    text_chunks = len(all_chunks) - table_chunks
    
    print(f"βœ… Optimized Text Chunking complete!")
    print(f"πŸ“Š Generated {len(all_chunks)} total chunks")
    print(f"πŸ“Š Table chunks: {table_chunks} (dedicated)")
    print(f"πŸ“ Text chunks: {text_chunks} (regular)")
    if total_time > 0:
        print(f"⚑ Processing speed: {total_chars / total_time:.0f} chars/sec")
    else:
        print(f"⚑ Processing speed: >1M chars/sec (instantaneous)")
    print(f"πŸ“ Average chunk size: {avg_chunk_size:.0f} characters")
    print(f"⏱️ Total time: {total_time:.3f}s")
    
    return all_chunks