Download src/rag/utils.py from lotran12/llm: direct link, hf CLI and curl.
- Browser
- Download file 559 Bytes
-
https://huggingface.co/spaces/lotran12/llm/resolve/main/src/rag/utils.py
- Command line
-
hf download hf://spaces/lotran12/llm/src/rag/utils.py
-
curl -L -o utils.py https://huggingface.co/spaces/lotran12/llm/resolve/main/src/rag/utils.py
559 Bytes
| from langchain_text_splitters import RecursiveCharacterTextSplitter | |
| from typing import List | |
| class TextSplitter: | |
| def __init__(self, | |
| seperators: List[str] = ['\n\n', '\n', ' ', ''], | |
| chunk_size: int = 512, | |
| chunk_overlap: int = 50) -> None: | |
| self.splitter = RecursiveCharacterTextSplitter( | |
| separators=seperators, | |
| chunk_size=chunk_size, | |
| chunk_overlap=chunk_overlap | |
| ) | |
| def __call__(self, documents): | |
| return self.splitter.split_documents(documents) | |