Spaces:
Running
Running
Update app/core/text_splitter.py
Browse files
app/core/text_splitter.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
# text_splitter.py
|
| 2 |
|
| 3 |
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
| 4 |
|
|
@@ -12,7 +12,7 @@ def split_text(text):
|
|
| 12 |
|
| 13 |
chunks = splitter.split_text(text)
|
| 14 |
|
| 15 |
-
#
|
| 16 |
cleaned_chunks = []
|
| 17 |
for chunk in chunks:
|
| 18 |
chunk = chunk.strip()
|
|
|
|
| 1 |
+
# app/core/text_splitter.py
|
| 2 |
|
| 3 |
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
| 4 |
|
|
|
|
| 12 |
|
| 13 |
chunks = splitter.split_text(text)
|
| 14 |
|
| 15 |
+
# CLEANING STEP (VERY IMPORTANT)
|
| 16 |
cleaned_chunks = []
|
| 17 |
for chunk in chunks:
|
| 18 |
chunk = chunk.strip()
|