File size: 559 Bytes
a6e314f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
from langchain_text_splitters import RecursiveCharacterTextSplitter
from typing import List


class TextSplitter:
    def __init__(self,
                seperators: List[str] = ['\n\n', '\n', ' ', ''],
                chunk_size: int = 512,
                chunk_overlap: int = 50) -> None:
        self.splitter = RecursiveCharacterTextSplitter(
            separators=seperators,
            chunk_size=chunk_size,
            chunk_overlap=chunk_overlap
        )

    def __call__(self, documents):
        return self.splitter.split_documents(documents)