Spaces:
Runtime error
Runtime error
Download utils/loader.py from Resaofd/resao-chat: direct link, hf CLI and curl.
- Browser
- Download file 1.79 kB
-
https://huggingface.co/spaces/Resaofd/resao-chat/resolve/main/utils/loader.py
- Command line
-
hf download hf://spaces/Resaofd/resao-chat/utils/loader.py
-
curl -L -o loader.py https://huggingface.co/spaces/Resaofd/resao-chat/resolve/main/utils/loader.py
1.79 kB
| from langchain_community.document_loaders import TextLoader | |
| from langchain.text_splitter import RecursiveCharacterTextSplitter | |
| from langchain.schema import Document | |
| def load_documents(file_path): | |
| loader = TextLoader(file_path) | |
| docs = loader.load() | |
| text_splitter = RecursiveCharacterTextSplitter() | |
| print("tonga eto **************************") | |
| print(type(docs)) | |
| print(len(docs)) | |
| print(docs) | |
| print("tonga eto **************************") | |
| return text_splitter.split_documents(docs) | |
| # def load_documents(file_path, chunk_size=1000, chunk_overlap=200): | |
| # """ | |
| # Load and split documents into chunks. | |
| # Args: | |
| # file_path (str): Path to the document file. | |
| # chunk_size (int): Maximum characters per chunk. | |
| # chunk_overlap (int): Overlap between chunks. | |
| # Returns: | |
| # list: List of chunked document contents. | |
| # """ | |
| # loader = TextLoader(file_path) | |
| # docs = loader.load() # This should load as a list of strings or Document objects | |
| # # Ensure docs is a list of Document objects | |
| # print("tonga eto **************************") | |
| # print(type(docs)) | |
| # if isinstance(docs, list) and isinstance(docs[0], str): | |
| # docs = [Document(page_content=text) for text in docs] | |
| # print("tonga eto **************************") | |
| # # Configure the text splitter | |
| # text_splitter = RecursiveCharacterTextSplitter( | |
| # chunk_size=chunk_size, | |
| # chunk_overlap=chunk_overlap | |
| # ) | |
| # # Split the documents into chunks and extract the content | |
| # split_docs = text_splitter.split_documents(docs) | |
| # return [doc.page_content for doc in split_docs] # Extract only the text content from Document objects | |