rag-backend / src /processing /chunker.py
Harshilforworks's picture
Upload 99 files
4244db2 verified
Raw History Blame Contribute Delete
16.1 kB
"""
Optimized text-based chunking without GPU dependencies.
Fast, memory-efficient, and reliable chunking for document processing.
"""
import re
import math
from typing import List, Dict, Optional, Any
from dataclasses import dataclass
import hashlib
import time
@dataclass
class OptimizedChunkConfig:
"""Configuration for optimized text chunking."""
chunk_size: int = 800 # Target characters per chunk
chunk_overlap: int = 250 # Overlap between chunks in characters
min_chunk_size: int = 200 # Minimum chunk size
max_chunk_size: int = 1500 # Maximum chunk size
split_on_sentences: bool = True # Try to split on sentence boundaries
preserve_paragraphs: bool = True # Try to keep paragraphs intact
separate_tables: bool = True # Create separate chunks for tables
class OptimizedTextChunker:
"""
Fast, CPU-only text chunker optimized for performance and simplicity.
Features:
- Pure text-based chunking (no embeddings/GPU)
- Sentence and paragraph boundary awareness
- Configurable overlap and sizing
- Memory efficient processing
- Fast execution for large documents
"""
def __init__(self, config: Optional[OptimizedChunkConfig] = None):
self.config = config or OptimizedChunkConfig()
print(f"βœ… OptimizedTextChunker initialized (CPU-only, {self.config.chunk_size} chars/chunk)")
if self.config.separate_tables:
print(f"πŸ“Š Table separation enabled - tables will get dedicated chunks")
def _extract_tables_and_text(self, content: str) -> tuple:
"""
Extract tables and remaining text separately.
Returns: (tables_list, text_without_tables)
"""
tables = []
remaining_text = content
# Find all table sections
table_pattern = r'TABLE:\n(.*?)(?=\n\n|\n(?=[A-Z][A-Za-z\s]*:)|$)'
table_matches = re.finditer(table_pattern, content, re.DOTALL)
# Extract tables from end to beginning to preserve indices
table_positions = []
for match in table_matches:
table_content = match.group(1).strip()
if table_content: # Only if table has content
tables.append({
'content': f"TABLE:\n{table_content}",
'start': match.start(),
'end': match.end(),
'type': 'table'
})
table_positions.append((match.start(), match.end()))
# Remove tables from text (process from end to beginning)
table_positions.sort(reverse=True)
for start, end in table_positions:
remaining_text = remaining_text[:start] + remaining_text[end:]
# Clean up remaining text
remaining_text = re.sub(r'\n{3,}', '\n\n', remaining_text).strip()
return tables, remaining_text
def _split_into_sentences(self, text: str) -> List[str]:
"""Split text into sentences using regex patterns."""
# Enhanced sentence splitting pattern
sentence_pattern = r'(?<=[.!?])\s+(?=[A-Z])'
sentences = re.split(sentence_pattern, text)
# Clean and filter sentences
cleaned_sentences = []
for sentence in sentences:
sentence = sentence.strip()
if sentence and len(sentence) > 10: # Filter very short fragments
cleaned_sentences.append(sentence)
return cleaned_sentences
def _split_into_paragraphs(self, text: str) -> List[str]:
"""Split text into paragraphs."""
# Split on double newlines or more
paragraphs = re.split(r'\n\s*\n', text)
# Clean and filter paragraphs
cleaned_paragraphs = []
for paragraph in paragraphs:
paragraph = paragraph.strip()
if paragraph and len(paragraph) > 20: # Filter very short paragraphs
cleaned_paragraphs.append(paragraph)
return cleaned_paragraphs
def _create_overlapping_chunks(self, text_units: List[str], unit_type: str = "sentence") -> List[str]:
"""Create overlapping chunks from text units."""
chunks = []
current_chunk = ""
current_length = 0
overlap_buffer = []
for i, unit in enumerate(text_units):
unit_length = len(unit)
# Check if adding this unit would exceed max chunk size
if current_length + unit_length > self.config.max_chunk_size and current_chunk:
# Finalize current chunk if it meets minimum size
if current_length >= self.config.min_chunk_size:
chunks.append(current_chunk.strip())
# Create overlap for next chunk
overlap_text = ""
overlap_length = 0
# Add units to overlap buffer (from end of current chunk)
for j in range(len(overlap_buffer) - 1, -1, -1):
if overlap_length + len(overlap_buffer[j]) <= self.config.chunk_overlap:
overlap_text = overlap_buffer[j] + " " + overlap_text
overlap_length += len(overlap_buffer[j])
else:
break
# Start new chunk with overlap
current_chunk = overlap_text.strip()
current_length = len(current_chunk)
overlap_buffer = []
else:
# Current chunk too small, continue building
pass
# Add current unit to chunk
if current_chunk:
current_chunk += " " + unit
else:
current_chunk = unit
current_length += unit_length + 1 # +1 for space
overlap_buffer.append(unit)
# Keep overlap buffer reasonable size
if len(overlap_buffer) > 10:
overlap_buffer.pop(0)
# Add final chunk if it has content
if current_chunk.strip() and len(current_chunk) >= self.config.min_chunk_size:
chunks.append(current_chunk.strip())
return chunks
def _simple_chunk_by_size(self, text: str) -> List[str]:
"""Simple fallback chunking by character count."""
chunks = []
start = 0
text_length = len(text)
while start < text_length:
# Calculate end position
end = start + self.config.chunk_size
# If this is not the last chunk, try to end at a good break point
if end < text_length:
# Look for sentence endings within reasonable distance
search_start = max(start + self.config.chunk_size - 100, start)
search_end = min(end + 100, text_length)
# Find the best break point (sentence ending)
best_break = -1
for i in range(search_end - 1, search_start - 1, -1):
if text[i] in '.!?':
# Make sure next character is whitespace or end
if i + 1 >= text_length or text[i + 1].isspace():
best_break = i + 1
break
if best_break > start:
end = best_break
# Extract chunk
chunk = text[start:end].strip()
if chunk and len(chunk) >= self.config.min_chunk_size:
chunks.append(chunk)
# Move start position (with overlap)
if end >= text_length:
break
start = max(end - self.config.chunk_overlap, start + 1)
return chunks
def chunk_text(self, text: str, document_name: str = "unknown") -> List[Dict[str, Any]]:
"""
Chunk a single text document into optimized chunks with separate table handling.
Args:
text: Input text to chunk
document_name: Name of the document for metadata
Returns:
List of chunk dictionaries
"""
if not text or not text.strip():
return []
start_time = time.time()
text = text.strip()
print(f"πŸ“„ Chunking {document_name}: {len(text):,} characters")
structured_chunks = []
chunk_index = 0
# Handle tables separately if enabled
if self.config.separate_tables and "TABLE:" in text:
tables, remaining_text = self._extract_tables_and_text(text)
print(f"πŸ“Š Found {len(tables)} tables to chunk separately")
# Create separate chunks for each table
for table in tables:
table_content = table['content']
chunk_id = f"{document_name}_table_{chunk_index}_{hashlib.md5(table_content.encode()).hexdigest()[:8]}"
structured_chunks.append({
'chunk_id': chunk_id,
'document_name': document_name,
'content': table_content,
'chunk_index': chunk_index,
'char_count': len(table_content),
'metadata': {
'chunking_method': 'table_dedicated',
'content_type': 'table',
'source_document': document_name,
'chunk_size_config': self.config.chunk_size,
'overlap_config': self.config.chunk_overlap
}
})
chunk_index += 1
# Use remaining text for regular chunking
text = remaining_text
print(f"πŸ“ Processing remaining text: {len(text):,} characters")
# Process regular text content
if text and text.strip():
# Try different chunking strategies based on text characteristics
chunks = []
if self.config.preserve_paragraphs and '\n\n' in text:
# Strategy 1: Paragraph-aware chunking
paragraphs = self._split_into_paragraphs(text)
if paragraphs:
print(f"πŸ“ Using paragraph-based chunking ({len(paragraphs)} paragraphs)")
chunks = self._create_overlapping_chunks(paragraphs, "paragraph")
# If paragraph chunking didn't work well, try sentence-based
if not chunks and self.config.split_on_sentences:
sentences = self._split_into_sentences(text)
if len(sentences) > 5: # Only if we have enough sentences
print(f"πŸ“ Using sentence-based chunking ({len(sentences)} sentences)")
chunks = self._create_overlapping_chunks(sentences, "sentence")
# Fallback to simple character-based chunking
if not chunks:
print(f"πŸ“ Using character-based chunking (fallback)")
chunks = self._simple_chunk_by_size(text)
# Convert text chunks to structured format
for chunk_text in chunks:
chunk_id = f"{document_name}_{chunk_index}_{hashlib.md5(chunk_text.encode()).hexdigest()[:8]}"
structured_chunks.append({
'chunk_id': chunk_id,
'document_name': document_name,
'content': chunk_text,
'chunk_index': chunk_index,
'char_count': len(chunk_text),
'metadata': {
'chunking_method': 'optimized_text',
'content_type': 'text',
'source_document': document_name,
'chunk_size_config': self.config.chunk_size,
'overlap_config': self.config.chunk_overlap
}
})
chunk_index += 1
processing_time = time.time() - start_time
table_count = sum(1 for chunk in structured_chunks if chunk['metadata'].get('content_type') == 'table')
text_count = len(structured_chunks) - table_count
print(f"βœ… Created {len(structured_chunks)} chunks in {processing_time:.2f}s")
print(f"πŸ“Š Tables: {table_count} dedicated chunks")
print(f"πŸ“ Text: {text_count} regular chunks")
return structured_chunks
def chunk_documents_optimized(parsed_content: List[Dict[str, Any]],
chunk_size: int = 500,
chunk_overlap: int = 150,
save_parsed_text: bool = False,
output_dir: str = "results") -> List[Dict[str, Any]]:
"""
Optimized document chunking function - CPU only, fast and reliable.
Args:
parsed_content: List of parsed document dictionaries
chunk_size: Target chunk size in characters
chunk_overlap: Overlap between chunks in characters
save_parsed_text: Whether to save text files (unused in optimized version)
output_dir: Output directory (unused in optimized version)
Returns:
List of chunk dictionaries
"""
if not parsed_content:
return []
print(f"πŸš€ Starting Optimized Text Chunking...")
start_time = time.time()
# Create optimized chunker with table separation enabled
config = OptimizedChunkConfig(
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
min_chunk_size=max(50, chunk_size // 8), # More reasonable minimum
max_chunk_size=chunk_size * 2,
split_on_sentences=True,
preserve_paragraphs=True,
separate_tables=True # Enable table separation
)
chunker = OptimizedTextChunker(config)
all_chunks = []
for doc_data in parsed_content:
doc_name = doc_data.get('document_name', 'unknown')
content = doc_data.get('content', '')
if not content or not content.strip():
print(f"⚠️ Skipping empty document: {doc_name}")
continue
# Chunk the document
doc_chunks = chunker.chunk_text(content, doc_name)
all_chunks.extend(doc_chunks)
total_time = time.time() - start_time
total_chars = sum(len(doc.get('content', '')) for doc in parsed_content)
avg_chunk_size = sum(len(chunk['content']) for chunk in all_chunks) / len(all_chunks) if all_chunks else 0
# Count table vs text chunks
table_chunks = sum(1 for chunk in all_chunks if chunk['metadata'].get('content_type') == 'table')
text_chunks = len(all_chunks) - table_chunks
print(f"βœ… Optimized Text Chunking complete!")
print(f"πŸ“Š Generated {len(all_chunks)} total chunks")
print(f"πŸ“Š Table chunks: {table_chunks} (dedicated)")
print(f"πŸ“ Text chunks: {text_chunks} (regular)")
if total_time > 0:
print(f"⚑ Processing speed: {total_chars / total_time:.0f} chars/sec")
else:
print(f"⚑ Processing speed: >1M chars/sec (instantaneous)")
print(f"πŸ“ Average chunk size: {avg_chunk_size:.0f} characters")
print(f"⏱️ Total time: {total_time:.3f}s")
return all_chunks