File size: 880 Bytes
215f97f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | """
Shared text chunking logic for training and inference.
"""
def chunk_text(text: str, chunk_size: int = 60, overlap: int = 30) -> list[str]:
"""Chunk text into sliding word windows while preserving line breaks.
Args:
text: Raw document text input.
chunk_size: Maximum words per chunk (default: 60).
overlap: Word overlap between consecutive chunks (default: 30).
Returns:
List of text chunk strings.
"""
lines = [line.strip() for line in text.split("\n") if line.strip()]
chunks = []
for line in lines:
words = line.split()
if len(words) <= chunk_size:
chunks.append(line)
else:
i = 0
while i < len(words):
c = " ".join(words[i:i + chunk_size])
chunks.append(c)
i += chunk_size - overlap
return chunks
|