File size: 3,270 Bytes
9be2851
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
"""

Enhanced document processor with multiple format support

"""
from langchain_community.document_loaders import TextLoader
from langchain.text_splitter import CharacterTextSplitter, RecursiveCharacterTextSplitter
from langchain.schema import Document
from typing import List, Optional
import logging

logger = logging.getLogger(__name__)


class DocumentProcessor:
    """Enhanced document processor"""
    
    def __init__(self, file_path: Optional[str] = None, chunk_size: int = 1000, chunk_overlap: int = 200):
        self.file_path = file_path
        self.chunk_size = chunk_size
        self.chunk_overlap = chunk_overlap
    
    def load_and_split(self) -> List[Document]:
        """Load document from file and split into chunks"""
        if not self.file_path:
            raise ValueError("file_path must be provided")
        
        try:
            loader = TextLoader(self.file_path, encoding='utf-8')
            documents = loader.load()
            
            # Use recursive splitter for better chunking
            splitter = RecursiveCharacterTextSplitter(
                chunk_size=self.chunk_size,
                chunk_overlap=self.chunk_overlap,
                separators=["\n\n", "\n", ".", "!", "?", ",", " ", ""],
                length_function=len
            )
            texts = splitter.split_documents(documents)
            
            logger.info(f"Processed {len(texts)} chunks from {self.file_path}")
            return texts
            
        except Exception as e:
            logger.error(f"Error processing document: {e}")
            raise
    
    def split_text(self, text: str, metadata: Optional[dict] = None) -> List[Document]:
        """Split raw text into chunks"""
        try:
            splitter = RecursiveCharacterTextSplitter(
                chunk_size=self.chunk_size,
                chunk_overlap=self.chunk_overlap,
                separators=["\n\n", "\n", ".", "!", "?", ",", " ", ""],
                length_function=len
            )
            
            # Create document
            doc = Document(page_content=text, metadata=metadata or {})
            texts = splitter.split_documents([doc])
            
            logger.info(f"Split text into {len(texts)} chunks")
            return texts
            
        except Exception as e:
            logger.error(f"Error splitting text: {e}")
            raise
    
    def create_documents_from_texts(

        self,

        texts: List[str],

        metadatas: Optional[List[dict]] = None

    ) -> List[Document]:
        """Create Document objects from list of texts"""
        documents = []
        
        for i, text in enumerate(texts):
            metadata = metadatas[i] if metadatas and i < len(metadatas) else {}
            doc = Document(page_content=text, metadata=metadata)
            documents.append(doc)
        
        return documents


if __name__ == "__main__":
    processor = DocumentProcessor("ecommerce_faq.txt")
    texts = processor.load_and_split()
    print(f"Processed {len(texts)} text chunks")
    
    # Print first chunk
    if texts:
        print(f"\nFirst chunk:\n{texts[0].page_content[:200]}...")