llm / src /rag /utils.py
lotran12's picture
Upload 8 files
a6e314f verified
Raw History Blame Contribute Delete
559 Bytes
from langchain_text_splitters import RecursiveCharacterTextSplitter
from typing import List
class TextSplitter:
def __init__(self,
seperators: List[str] = ['\n\n', '\n', ' ', ''],
chunk_size: int = 512,
chunk_overlap: int = 50) -> None:
self.splitter = RecursiveCharacterTextSplitter(
separators=seperators,
chunk_size=chunk_size,
chunk_overlap=chunk_overlap
)
def __call__(self, documents):
return self.splitter.split_documents(documents)