ThinkOV2 / documentProcessor.py
Ayush-003's picture
Upload 28 files
9be2851 verified
Raw History Blame Contribute Delete
3.27 kB
"""
Enhanced document processor with multiple format support
"""
from langchain_community.document_loaders import TextLoader
from langchain.text_splitter import CharacterTextSplitter, RecursiveCharacterTextSplitter
from langchain.schema import Document
from typing import List, Optional
import logging
logger = logging.getLogger(__name__)
class DocumentProcessor:
"""Enhanced document processor"""
def __init__(self, file_path: Optional[str] = None, chunk_size: int = 1000, chunk_overlap: int = 200):
self.file_path = file_path
self.chunk_size = chunk_size
self.chunk_overlap = chunk_overlap
def load_and_split(self) -> List[Document]:
"""Load document from file and split into chunks"""
if not self.file_path:
raise ValueError("file_path must be provided")
try:
loader = TextLoader(self.file_path, encoding='utf-8')
documents = loader.load()
# Use recursive splitter for better chunking
splitter = RecursiveCharacterTextSplitter(
chunk_size=self.chunk_size,
chunk_overlap=self.chunk_overlap,
separators=["\n\n", "\n", ".", "!", "?", ",", " ", ""],
length_function=len
)
texts = splitter.split_documents(documents)
logger.info(f"Processed {len(texts)} chunks from {self.file_path}")
return texts
except Exception as e:
logger.error(f"Error processing document: {e}")
raise
def split_text(self, text: str, metadata: Optional[dict] = None) -> List[Document]:
"""Split raw text into chunks"""
try:
splitter = RecursiveCharacterTextSplitter(
chunk_size=self.chunk_size,
chunk_overlap=self.chunk_overlap,
separators=["\n\n", "\n", ".", "!", "?", ",", " ", ""],
length_function=len
)
# Create document
doc = Document(page_content=text, metadata=metadata or {})
texts = splitter.split_documents([doc])
logger.info(f"Split text into {len(texts)} chunks")
return texts
except Exception as e:
logger.error(f"Error splitting text: {e}")
raise
def create_documents_from_texts(
self,
texts: List[str],
metadatas: Optional[List[dict]] = None
) -> List[Document]:
"""Create Document objects from list of texts"""
documents = []
for i, text in enumerate(texts):
metadata = metadatas[i] if metadatas and i < len(metadatas) else {}
doc = Document(page_content=text, metadata=metadata)
documents.append(doc)
return documents
if __name__ == "__main__":
processor = DocumentProcessor("ecommerce_faq.txt")
texts = processor.load_and_split()
print(f"Processed {len(texts)} text chunks")
# Print first chunk
if texts:
print(f"\nFirst chunk:\n{texts[0].page_content[:200]}...")