Spaces:
Runtime error
Runtime error
Download src/processing/chunker.py from Harshilforworks/rag-backend: direct link, hf CLI and curl.
- Browser
- Download file 16.1 kB
-
https://huggingface.co/spaces/Harshilforworks/rag-backend/resolve/main/src/processing/chunker.py
- Command line
-
hf download hf://spaces/Harshilforworks/rag-backend/src/processing/chunker.py
-
curl -L -o chunker.py https://huggingface.co/spaces/Harshilforworks/rag-backend/resolve/main/src/processing/chunker.py
16.1 kB
| """ | |
| Optimized text-based chunking without GPU dependencies. | |
| Fast, memory-efficient, and reliable chunking for document processing. | |
| """ | |
| import re | |
| import math | |
| from typing import List, Dict, Optional, Any | |
| from dataclasses import dataclass | |
| import hashlib | |
| import time | |
| class OptimizedChunkConfig: | |
| """Configuration for optimized text chunking.""" | |
| chunk_size: int = 800 # Target characters per chunk | |
| chunk_overlap: int = 250 # Overlap between chunks in characters | |
| min_chunk_size: int = 200 # Minimum chunk size | |
| max_chunk_size: int = 1500 # Maximum chunk size | |
| split_on_sentences: bool = True # Try to split on sentence boundaries | |
| preserve_paragraphs: bool = True # Try to keep paragraphs intact | |
| separate_tables: bool = True # Create separate chunks for tables | |
| class OptimizedTextChunker: | |
| """ | |
| Fast, CPU-only text chunker optimized for performance and simplicity. | |
| Features: | |
| - Pure text-based chunking (no embeddings/GPU) | |
| - Sentence and paragraph boundary awareness | |
| - Configurable overlap and sizing | |
| - Memory efficient processing | |
| - Fast execution for large documents | |
| """ | |
| def __init__(self, config: Optional[OptimizedChunkConfig] = None): | |
| self.config = config or OptimizedChunkConfig() | |
| print(f"β OptimizedTextChunker initialized (CPU-only, {self.config.chunk_size} chars/chunk)") | |
| if self.config.separate_tables: | |
| print(f"π Table separation enabled - tables will get dedicated chunks") | |
| def _extract_tables_and_text(self, content: str) -> tuple: | |
| """ | |
| Extract tables and remaining text separately. | |
| Returns: (tables_list, text_without_tables) | |
| """ | |
| tables = [] | |
| remaining_text = content | |
| # Find all table sections | |
| table_pattern = r'TABLE:\n(.*?)(?=\n\n|\n(?=[A-Z][A-Za-z\s]*:)|$)' | |
| table_matches = re.finditer(table_pattern, content, re.DOTALL) | |
| # Extract tables from end to beginning to preserve indices | |
| table_positions = [] | |
| for match in table_matches: | |
| table_content = match.group(1).strip() | |
| if table_content: # Only if table has content | |
| tables.append({ | |
| 'content': f"TABLE:\n{table_content}", | |
| 'start': match.start(), | |
| 'end': match.end(), | |
| 'type': 'table' | |
| }) | |
| table_positions.append((match.start(), match.end())) | |
| # Remove tables from text (process from end to beginning) | |
| table_positions.sort(reverse=True) | |
| for start, end in table_positions: | |
| remaining_text = remaining_text[:start] + remaining_text[end:] | |
| # Clean up remaining text | |
| remaining_text = re.sub(r'\n{3,}', '\n\n', remaining_text).strip() | |
| return tables, remaining_text | |
| def _split_into_sentences(self, text: str) -> List[str]: | |
| """Split text into sentences using regex patterns.""" | |
| # Enhanced sentence splitting pattern | |
| sentence_pattern = r'(?<=[.!?])\s+(?=[A-Z])' | |
| sentences = re.split(sentence_pattern, text) | |
| # Clean and filter sentences | |
| cleaned_sentences = [] | |
| for sentence in sentences: | |
| sentence = sentence.strip() | |
| if sentence and len(sentence) > 10: # Filter very short fragments | |
| cleaned_sentences.append(sentence) | |
| return cleaned_sentences | |
| def _split_into_paragraphs(self, text: str) -> List[str]: | |
| """Split text into paragraphs.""" | |
| # Split on double newlines or more | |
| paragraphs = re.split(r'\n\s*\n', text) | |
| # Clean and filter paragraphs | |
| cleaned_paragraphs = [] | |
| for paragraph in paragraphs: | |
| paragraph = paragraph.strip() | |
| if paragraph and len(paragraph) > 20: # Filter very short paragraphs | |
| cleaned_paragraphs.append(paragraph) | |
| return cleaned_paragraphs | |
| def _create_overlapping_chunks(self, text_units: List[str], unit_type: str = "sentence") -> List[str]: | |
| """Create overlapping chunks from text units.""" | |
| chunks = [] | |
| current_chunk = "" | |
| current_length = 0 | |
| overlap_buffer = [] | |
| for i, unit in enumerate(text_units): | |
| unit_length = len(unit) | |
| # Check if adding this unit would exceed max chunk size | |
| if current_length + unit_length > self.config.max_chunk_size and current_chunk: | |
| # Finalize current chunk if it meets minimum size | |
| if current_length >= self.config.min_chunk_size: | |
| chunks.append(current_chunk.strip()) | |
| # Create overlap for next chunk | |
| overlap_text = "" | |
| overlap_length = 0 | |
| # Add units to overlap buffer (from end of current chunk) | |
| for j in range(len(overlap_buffer) - 1, -1, -1): | |
| if overlap_length + len(overlap_buffer[j]) <= self.config.chunk_overlap: | |
| overlap_text = overlap_buffer[j] + " " + overlap_text | |
| overlap_length += len(overlap_buffer[j]) | |
| else: | |
| break | |
| # Start new chunk with overlap | |
| current_chunk = overlap_text.strip() | |
| current_length = len(current_chunk) | |
| overlap_buffer = [] | |
| else: | |
| # Current chunk too small, continue building | |
| pass | |
| # Add current unit to chunk | |
| if current_chunk: | |
| current_chunk += " " + unit | |
| else: | |
| current_chunk = unit | |
| current_length += unit_length + 1 # +1 for space | |
| overlap_buffer.append(unit) | |
| # Keep overlap buffer reasonable size | |
| if len(overlap_buffer) > 10: | |
| overlap_buffer.pop(0) | |
| # Add final chunk if it has content | |
| if current_chunk.strip() and len(current_chunk) >= self.config.min_chunk_size: | |
| chunks.append(current_chunk.strip()) | |
| return chunks | |
| def _simple_chunk_by_size(self, text: str) -> List[str]: | |
| """Simple fallback chunking by character count.""" | |
| chunks = [] | |
| start = 0 | |
| text_length = len(text) | |
| while start < text_length: | |
| # Calculate end position | |
| end = start + self.config.chunk_size | |
| # If this is not the last chunk, try to end at a good break point | |
| if end < text_length: | |
| # Look for sentence endings within reasonable distance | |
| search_start = max(start + self.config.chunk_size - 100, start) | |
| search_end = min(end + 100, text_length) | |
| # Find the best break point (sentence ending) | |
| best_break = -1 | |
| for i in range(search_end - 1, search_start - 1, -1): | |
| if text[i] in '.!?': | |
| # Make sure next character is whitespace or end | |
| if i + 1 >= text_length or text[i + 1].isspace(): | |
| best_break = i + 1 | |
| break | |
| if best_break > start: | |
| end = best_break | |
| # Extract chunk | |
| chunk = text[start:end].strip() | |
| if chunk and len(chunk) >= self.config.min_chunk_size: | |
| chunks.append(chunk) | |
| # Move start position (with overlap) | |
| if end >= text_length: | |
| break | |
| start = max(end - self.config.chunk_overlap, start + 1) | |
| return chunks | |
| def chunk_text(self, text: str, document_name: str = "unknown") -> List[Dict[str, Any]]: | |
| """ | |
| Chunk a single text document into optimized chunks with separate table handling. | |
| Args: | |
| text: Input text to chunk | |
| document_name: Name of the document for metadata | |
| Returns: | |
| List of chunk dictionaries | |
| """ | |
| if not text or not text.strip(): | |
| return [] | |
| start_time = time.time() | |
| text = text.strip() | |
| print(f"π Chunking {document_name}: {len(text):,} characters") | |
| structured_chunks = [] | |
| chunk_index = 0 | |
| # Handle tables separately if enabled | |
| if self.config.separate_tables and "TABLE:" in text: | |
| tables, remaining_text = self._extract_tables_and_text(text) | |
| print(f"π Found {len(tables)} tables to chunk separately") | |
| # Create separate chunks for each table | |
| for table in tables: | |
| table_content = table['content'] | |
| chunk_id = f"{document_name}_table_{chunk_index}_{hashlib.md5(table_content.encode()).hexdigest()[:8]}" | |
| structured_chunks.append({ | |
| 'chunk_id': chunk_id, | |
| 'document_name': document_name, | |
| 'content': table_content, | |
| 'chunk_index': chunk_index, | |
| 'char_count': len(table_content), | |
| 'metadata': { | |
| 'chunking_method': 'table_dedicated', | |
| 'content_type': 'table', | |
| 'source_document': document_name, | |
| 'chunk_size_config': self.config.chunk_size, | |
| 'overlap_config': self.config.chunk_overlap | |
| } | |
| }) | |
| chunk_index += 1 | |
| # Use remaining text for regular chunking | |
| text = remaining_text | |
| print(f"π Processing remaining text: {len(text):,} characters") | |
| # Process regular text content | |
| if text and text.strip(): | |
| # Try different chunking strategies based on text characteristics | |
| chunks = [] | |
| if self.config.preserve_paragraphs and '\n\n' in text: | |
| # Strategy 1: Paragraph-aware chunking | |
| paragraphs = self._split_into_paragraphs(text) | |
| if paragraphs: | |
| print(f"π Using paragraph-based chunking ({len(paragraphs)} paragraphs)") | |
| chunks = self._create_overlapping_chunks(paragraphs, "paragraph") | |
| # If paragraph chunking didn't work well, try sentence-based | |
| if not chunks and self.config.split_on_sentences: | |
| sentences = self._split_into_sentences(text) | |
| if len(sentences) > 5: # Only if we have enough sentences | |
| print(f"π Using sentence-based chunking ({len(sentences)} sentences)") | |
| chunks = self._create_overlapping_chunks(sentences, "sentence") | |
| # Fallback to simple character-based chunking | |
| if not chunks: | |
| print(f"π Using character-based chunking (fallback)") | |
| chunks = self._simple_chunk_by_size(text) | |
| # Convert text chunks to structured format | |
| for chunk_text in chunks: | |
| chunk_id = f"{document_name}_{chunk_index}_{hashlib.md5(chunk_text.encode()).hexdigest()[:8]}" | |
| structured_chunks.append({ | |
| 'chunk_id': chunk_id, | |
| 'document_name': document_name, | |
| 'content': chunk_text, | |
| 'chunk_index': chunk_index, | |
| 'char_count': len(chunk_text), | |
| 'metadata': { | |
| 'chunking_method': 'optimized_text', | |
| 'content_type': 'text', | |
| 'source_document': document_name, | |
| 'chunk_size_config': self.config.chunk_size, | |
| 'overlap_config': self.config.chunk_overlap | |
| } | |
| }) | |
| chunk_index += 1 | |
| processing_time = time.time() - start_time | |
| table_count = sum(1 for chunk in structured_chunks if chunk['metadata'].get('content_type') == 'table') | |
| text_count = len(structured_chunks) - table_count | |
| print(f"β Created {len(structured_chunks)} chunks in {processing_time:.2f}s") | |
| print(f"π Tables: {table_count} dedicated chunks") | |
| print(f"π Text: {text_count} regular chunks") | |
| return structured_chunks | |
| def chunk_documents_optimized(parsed_content: List[Dict[str, Any]], | |
| chunk_size: int = 500, | |
| chunk_overlap: int = 150, | |
| save_parsed_text: bool = False, | |
| output_dir: str = "results") -> List[Dict[str, Any]]: | |
| """ | |
| Optimized document chunking function - CPU only, fast and reliable. | |
| Args: | |
| parsed_content: List of parsed document dictionaries | |
| chunk_size: Target chunk size in characters | |
| chunk_overlap: Overlap between chunks in characters | |
| save_parsed_text: Whether to save text files (unused in optimized version) | |
| output_dir: Output directory (unused in optimized version) | |
| Returns: | |
| List of chunk dictionaries | |
| """ | |
| if not parsed_content: | |
| return [] | |
| print(f"π Starting Optimized Text Chunking...") | |
| start_time = time.time() | |
| # Create optimized chunker with table separation enabled | |
| config = OptimizedChunkConfig( | |
| chunk_size=chunk_size, | |
| chunk_overlap=chunk_overlap, | |
| min_chunk_size=max(50, chunk_size // 8), # More reasonable minimum | |
| max_chunk_size=chunk_size * 2, | |
| split_on_sentences=True, | |
| preserve_paragraphs=True, | |
| separate_tables=True # Enable table separation | |
| ) | |
| chunker = OptimizedTextChunker(config) | |
| all_chunks = [] | |
| for doc_data in parsed_content: | |
| doc_name = doc_data.get('document_name', 'unknown') | |
| content = doc_data.get('content', '') | |
| if not content or not content.strip(): | |
| print(f"β οΈ Skipping empty document: {doc_name}") | |
| continue | |
| # Chunk the document | |
| doc_chunks = chunker.chunk_text(content, doc_name) | |
| all_chunks.extend(doc_chunks) | |
| total_time = time.time() - start_time | |
| total_chars = sum(len(doc.get('content', '')) for doc in parsed_content) | |
| avg_chunk_size = sum(len(chunk['content']) for chunk in all_chunks) / len(all_chunks) if all_chunks else 0 | |
| # Count table vs text chunks | |
| table_chunks = sum(1 for chunk in all_chunks if chunk['metadata'].get('content_type') == 'table') | |
| text_chunks = len(all_chunks) - table_chunks | |
| print(f"β Optimized Text Chunking complete!") | |
| print(f"π Generated {len(all_chunks)} total chunks") | |
| print(f"π Table chunks: {table_chunks} (dedicated)") | |
| print(f"π Text chunks: {text_chunks} (regular)") | |
| if total_time > 0: | |
| print(f"β‘ Processing speed: {total_chars / total_time:.0f} chars/sec") | |
| else: | |
| print(f"β‘ Processing speed: >1M chars/sec (instantaneous)") | |
| print(f"π Average chunk size: {avg_chunk_size:.0f} characters") | |
| print(f"β±οΈ Total time: {total_time:.3f}s") | |
| return all_chunks | |