""" Enhanced document processor with multiple format support """ from langchain_community.document_loaders import TextLoader from langchain.text_splitter import CharacterTextSplitter, RecursiveCharacterTextSplitter from langchain.schema import Document from typing import List, Optional import logging logger = logging.getLogger(__name__) class DocumentProcessor: """Enhanced document processor""" def __init__(self, file_path: Optional[str] = None, chunk_size: int = 1000, chunk_overlap: int = 200): self.file_path = file_path self.chunk_size = chunk_size self.chunk_overlap = chunk_overlap def load_and_split(self) -> List[Document]: """Load document from file and split into chunks""" if not self.file_path: raise ValueError("file_path must be provided") try: loader = TextLoader(self.file_path, encoding='utf-8') documents = loader.load() # Use recursive splitter for better chunking splitter = RecursiveCharacterTextSplitter( chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap, separators=["\n\n", "\n", ".", "!", "?", ",", " ", ""], length_function=len ) texts = splitter.split_documents(documents) logger.info(f"Processed {len(texts)} chunks from {self.file_path}") return texts except Exception as e: logger.error(f"Error processing document: {e}") raise def split_text(self, text: str, metadata: Optional[dict] = None) -> List[Document]: """Split raw text into chunks""" try: splitter = RecursiveCharacterTextSplitter( chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap, separators=["\n\n", "\n", ".", "!", "?", ",", " ", ""], length_function=len ) # Create document doc = Document(page_content=text, metadata=metadata or {}) texts = splitter.split_documents([doc]) logger.info(f"Split text into {len(texts)} chunks") return texts except Exception as e: logger.error(f"Error splitting text: {e}") raise def create_documents_from_texts( self, texts: List[str], metadatas: Optional[List[dict]] = None ) -> List[Document]: """Create Document objects from list of texts""" documents = [] for i, text in enumerate(texts): metadata = metadatas[i] if metadatas and i < len(metadatas) else {} doc = Document(page_content=text, metadata=metadata) documents.append(doc) return documents if __name__ == "__main__": processor = DocumentProcessor("ecommerce_faq.txt") texts = processor.load_and_split() print(f"Processed {len(texts)} text chunks") # Print first chunk if texts: print(f"\nFirst chunk:\n{texts[0].page_content[:200]}...")