lllm2 / src /rag /utils.py
lotran12's picture
Upload 9 files
b8228d1 verified
Raw History Blame Contribute Delete
533 Bytes
from langchain_text_splitters import RecursiveCharacterTextSplitter
class TextSplitter:
def __init__(self,
seperators: List[str] = ['\n\n', '\n', ' ', ''],
chunk_size: int = 300,
chunk_overlap: int = 0) -> None:
self.splitter = RecursiveCharacterTextSplitter(
separators=seperators,
chunk_size=chunk_size,
chunk_overlap=chunk_overlap
)
def __call__(self, documents):
return self.splitter.split_documents(documents)