Spaces:
Runtime error
Runtime error
File size: 1,791 Bytes
3265f46 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 | from langchain_community.document_loaders import TextLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.schema import Document
def load_documents(file_path):
loader = TextLoader(file_path)
docs = loader.load()
text_splitter = RecursiveCharacterTextSplitter()
print("tonga eto **************************")
print(type(docs))
print(len(docs))
print(docs)
print("tonga eto **************************")
return text_splitter.split_documents(docs)
# def load_documents(file_path, chunk_size=1000, chunk_overlap=200):
# """
# Load and split documents into chunks.
# Args:
# file_path (str): Path to the document file.
# chunk_size (int): Maximum characters per chunk.
# chunk_overlap (int): Overlap between chunks.
# Returns:
# list: List of chunked document contents.
# """
# loader = TextLoader(file_path)
# docs = loader.load() # This should load as a list of strings or Document objects
# # Ensure docs is a list of Document objects
# print("tonga eto **************************")
# print(type(docs))
# if isinstance(docs, list) and isinstance(docs[0], str):
# docs = [Document(page_content=text) for text in docs]
# print("tonga eto **************************")
# # Configure the text splitter
# text_splitter = RecursiveCharacterTextSplitter(
# chunk_size=chunk_size,
# chunk_overlap=chunk_overlap
# )
# # Split the documents into chunks and extract the content
# split_docs = text_splitter.split_documents(docs)
# print
# return [doc.page_content for doc in split_docs] # Extract only the text content from Document objects
|