grocifychat / documentProcessor.py
Ayush-003's picture
Upload 7 files
743e367 verified
Raw History Blame Contribute Delete
665 Bytes
from langchain_community.document_loaders.text import TextLoader
from langchain.text_splitter import CharacterTextSplitter
class DocumentProcessor:
def __init__(self, file_path):
self.file_path = file_path
def load_and_split(self):
loader = TextLoader(self.file_path)
documents = loader.load()
splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
texts = splitter.split_documents(documents)
return texts
if __name__ == "__main__":
processor = DocumentProcessor("ecommerce_faq.txt")
texts = processor.load_and_split()
print(f"Processed {len(texts)} text chunks")