File size: 1,791 Bytes
3265f46
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
from langchain_community.document_loaders import TextLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.schema import Document


def load_documents(file_path):
    loader = TextLoader(file_path)
    docs = loader.load()
    text_splitter = RecursiveCharacterTextSplitter()
    print("tonga eto **************************")
    print(type(docs))
    print(len(docs))
    print(docs)
    print("tonga eto **************************")
    return text_splitter.split_documents(docs)

# def load_documents(file_path, chunk_size=1000, chunk_overlap=200):
#     """
#     Load and split documents into chunks.

#     Args:
#         file_path (str): Path to the document file.
#         chunk_size (int): Maximum characters per chunk.
#         chunk_overlap (int): Overlap between chunks.

#     Returns:
#         list: List of chunked document contents.
#     """
#     loader = TextLoader(file_path)
#     docs = loader.load()  # This should load as a list of strings or Document objects

#     # Ensure docs is a list of Document objects
#     print("tonga eto **************************")
#     print(type(docs))
#     if isinstance(docs, list) and isinstance(docs[0], str):
#         docs = [Document(page_content=text) for text in docs]
#     print("tonga eto **************************")

#     # Configure the text splitter
#     text_splitter = RecursiveCharacterTextSplitter(
#         chunk_size=chunk_size,
#         chunk_overlap=chunk_overlap
#     )

#     # Split the documents into chunks and extract the content
#     split_docs = text_splitter.split_documents(docs)

#     print

#     return [doc.page_content for doc in split_docs]  # Extract only the text content from Document objects