Spaces:
Sleeping
Sleeping
File size: 665 Bytes
743e367 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 | from langchain_community.document_loaders.text import TextLoader
from langchain.text_splitter import CharacterTextSplitter
class DocumentProcessor:
def __init__(self, file_path):
self.file_path = file_path
def load_and_split(self):
loader = TextLoader(self.file_path)
documents = loader.load()
splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
texts = splitter.split_documents(documents)
return texts
if __name__ == "__main__":
processor = DocumentProcessor("ecommerce_faq.txt")
texts = processor.load_and_split()
print(f"Processed {len(texts)} text chunks") |