resao-chat / utils /loader.py
Voadinajaina's picture
Upload 9 files
3265f46 verified
Raw History Blame Contribute Delete
1.79 kB
from langchain_community.document_loaders import TextLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.schema import Document
def load_documents(file_path):
loader = TextLoader(file_path)
docs = loader.load()
text_splitter = RecursiveCharacterTextSplitter()
print("tonga eto **************************")
print(type(docs))
print(len(docs))
print(docs)
print("tonga eto **************************")
return text_splitter.split_documents(docs)
# def load_documents(file_path, chunk_size=1000, chunk_overlap=200):
# """
# Load and split documents into chunks.
# Args:
# file_path (str): Path to the document file.
# chunk_size (int): Maximum characters per chunk.
# chunk_overlap (int): Overlap between chunks.
# Returns:
# list: List of chunked document contents.
# """
# loader = TextLoader(file_path)
# docs = loader.load() # This should load as a list of strings or Document objects
# # Ensure docs is a list of Document objects
# print("tonga eto **************************")
# print(type(docs))
# if isinstance(docs, list) and isinstance(docs[0], str):
# docs = [Document(page_content=text) for text in docs]
# print("tonga eto **************************")
# # Configure the text splitter
# text_splitter = RecursiveCharacterTextSplitter(
# chunk_size=chunk_size,
# chunk_overlap=chunk_overlap
# )
# # Split the documents into chunks and extract the content
# split_docs = text_splitter.split_documents(docs)
# print
# return [doc.page_content for doc in split_docs] # Extract only the text content from Document objects