Spaces:
Running
Running
File size: 1,305 Bytes
a55ea57 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 | def chunk_text(text: str, max_chars: int = 6000) -> list[str]:
cleaned = (text or "").strip()
if not cleaned or len(cleaned) <= max_chars:
return [cleaned] if cleaned else [""]
# Prefer blank-line paragraphs, then single newlines (dialogue turns),
# else hard character slices.
if "\n\n" in cleaned:
parts = [p.strip() for p in cleaned.split("\n\n") if p.strip()]
joiner = "\n\n"
elif "\n" in cleaned:
parts = [p.strip() for p in cleaned.split("\n") if p.strip()]
joiner = "\n"
else:
parts = [cleaned]
joiner = "\n"
if not parts:
parts = [cleaned]
chunks: list[str] = []
current = ""
for part in parts:
candidate = f"{current}{joiner}{part}".strip() if current else part
if len(candidate) <= max_chars:
current = candidate
continue
if current:
chunks.append(current)
current = ""
if len(part) <= max_chars:
current = part
continue
for i in range(0, len(part), max_chars):
piece = part[i : i + max_chars].strip()
if piece:
chunks.append(piece)
current = ""
if current:
chunks.append(current)
return chunks or [cleaned]
|