FastAPIExperience2 / core /text_processor.py
utkarsh134's picture
Upload 14 files
a88785b verified
Raw History Blame
392 Bytes
from langchain.text_splitter import RecursiveCharacterTextSplitter
def chunk_text(text, chunk_size=1000, chunk_overlap=200):
text_splitter = RecursiveCharacterTextSplitter(
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
length_function=len,
is_separator_regex=False,
)
chunks = text_splitter.split_text(text)
return chunks