from langchain_community.document_loaders import TextLoader from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.schema import Document def load_documents(file_path): loader = TextLoader(file_path) docs = loader.load() text_splitter = RecursiveCharacterTextSplitter() print("tonga eto **************************") print(type(docs)) print(len(docs)) print(docs) print("tonga eto **************************") return text_splitter.split_documents(docs) # def load_documents(file_path, chunk_size=1000, chunk_overlap=200): # """ # Load and split documents into chunks. # Args: # file_path (str): Path to the document file. # chunk_size (int): Maximum characters per chunk. # chunk_overlap (int): Overlap between chunks. # Returns: # list: List of chunked document contents. # """ # loader = TextLoader(file_path) # docs = loader.load() # This should load as a list of strings or Document objects # # Ensure docs is a list of Document objects # print("tonga eto **************************") # print(type(docs)) # if isinstance(docs, list) and isinstance(docs[0], str): # docs = [Document(page_content=text) for text in docs] # print("tonga eto **************************") # # Configure the text splitter # text_splitter = RecursiveCharacterTextSplitter( # chunk_size=chunk_size, # chunk_overlap=chunk_overlap # ) # # Split the documents into chunks and extract the content # split_docs = text_splitter.split_documents(docs) # print # return [doc.page_content for doc in split_docs] # Extract only the text content from Document objects