Spaces:
Sleeping
Sleeping
Download documentProcessor.py from Ayush-003/grocifychat: direct link, hf CLI and curl.
- Browser
- Download file 665 Bytes
-
https://huggingface.co/spaces/Ayush-003/grocifychat/resolve/main/documentProcessor.py
- Command line
-
hf download hf://spaces/Ayush-003/grocifychat/documentProcessor.py
-
curl -L -o documentProcessor.py https://huggingface.co/spaces/Ayush-003/grocifychat/resolve/main/documentProcessor.py
665 Bytes
| from langchain_community.document_loaders.text import TextLoader | |
| from langchain.text_splitter import CharacterTextSplitter | |
| class DocumentProcessor: | |
| def __init__(self, file_path): | |
| self.file_path = file_path | |
| def load_and_split(self): | |
| loader = TextLoader(self.file_path) | |
| documents = loader.load() | |
| splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=200) | |
| texts = splitter.split_documents(documents) | |
| return texts | |
| if __name__ == "__main__": | |
| processor = DocumentProcessor("ecommerce_faq.txt") | |
| texts = processor.load_and_split() | |
| print(f"Processed {len(texts)} text chunks") |