Spaces:
Sleeping
Sleeping
File size: 5,539 Bytes
f729bac 4cc8f6d f729bac 4cc8f6d f729bac 4cc8f6d f729bac 4cc8f6d f729bac 4cc8f6d f729bac 4cc8f6d f729bac | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 | """
Document processor module for handling various document types and chunking methods.
Integrates PDF loading and chunking functionality for the Gradio interface.
"""
import logging
from typing import Dict, List, Optional
import os
from pathlib import Path
# Import existing modules
from pdf_loader import PDFLoader
from chunking_methods import ChunkingManager, FixedChunking, SentenceChunking, AdaptiveChunking
import settings
logger = logging.getLogger(__name__)
class DocumentProcessor:
"""
Handles document loading and processing with multiple chunking methods.
"""
def __init__(self):
"""Initialize the document processor with available chunking methods."""
self.chunking_manager = ChunkingManager()
self.pdf_loader = PDFLoader()
# Register available chunking methods
self.chunking_strategies = {
'fixed': FixedChunking(),
'sentence': SentenceChunking(),
'adaptive': AdaptiveChunking()
}
logger.info(f"DocumentProcessor initialized with {len(self.chunking_strategies)} chunking strategies")
def load_document(self, file_path: str) -> Optional[str]:
"""
Load document content based on file type.
Args:
file_path: Path to the document file
Returns:
Document text content or None if loading fails
"""
try:
if not os.path.exists(file_path):
logger.error(f"File not found: {file_path}")
return None
file_extension = Path(file_path).suffix.lower()
if file_extension == '.pdf':
# Use PDFLoader to load the file directly
try:
content = self.pdf_loader.load_pdf(Path(file_path))
return content
except Exception as e:
logger.error(f"Error loading PDF {file_path}: {str(e)}")
return None
elif file_extension == '.txt':
with open(file_path, 'r', encoding='utf-8') as f:
return f.read()
elif file_extension in ['.doc', '.docx']:
try:
from docx import Document
doc = Document(file_path)
content = []
for paragraph in doc.paragraphs:
content.append(paragraph.text)
return '\n'.join(content)
except ImportError:
logger.error("python-docx not installed. Cannot process DOCX files.")
return None
else:
logger.error(f"Unsupported file type: {file_extension}")
return None
except Exception as e:
logger.error(f"Error loading document {file_path}: {str(e)}")
return None
def process_document(self, file_path: str, chunking_methods: List[str]) -> Dict[str, List[str]]:
"""
Process a document with specified chunking methods.
Args:
file_path: Path to the document file
chunking_methods: List of chunking method names to apply
Returns:
Dictionary mapping method names to lists of chunks
"""
try:
# Load document content
content = self.load_document(file_path)
if not content:
logger.error(f"Failed to load content from {file_path}")
return {}
logger.info(f"Loaded document with {len(content)} characters")
# Process with each requested chunking method
chunks_by_method = {}
for method_name in chunking_methods:
if method_name not in self.chunking_strategies:
logger.warning(f"Unknown chunking method: {method_name}")
continue
try:
strategy = self.chunking_strategies[method_name]
chunks = strategy.create_chunks(content)
chunks_by_method[method_name] = chunks
logger.info(f"Generated {len(chunks)} chunks using {method_name} method")
except Exception as e:
logger.error(f"Error processing with {method_name} method: {str(e)}")
continue
return chunks_by_method
except Exception as e:
logger.error(f"Error processing document {file_path}: {str(e)}")
return {}
def get_supported_extensions(self) -> List[str]:
"""
Get list of supported file extensions.
Returns:
List of supported file extensions
"""
return ['.pdf', '.txt', '.docx']
def validate_file(self, file_path: str) -> bool:
"""
Validate if a file can be processed.
Args:
file_path: Path to the file to validate
Returns:
True if file can be processed, False otherwise
"""
if not os.path.exists(file_path):
return False
extension = Path(file_path).suffix.lower()
return extension in self.get_supported_extensions() |