Spaces:
Running on Zero
Running on Zero
File size: 1,983 Bytes
88a619b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | """Whitespace/punctuation-based text segmentation.
There is no subtitle timing here — segments are just chunks of source text
sized for one translation call each. Splits on blank-line paragraph breaks
first, then on Tibetan sentence-ending shad marks (། ༎) for any paragraph
still longer than max_chars, then falls back to plain whitespace chunking
for any single "sentence" that is still too long on its own.
"""
import re
_SHAD_SPLIT_RE = re.compile(r"(?<=[།༎])\s*")
def _chunk_by_words(text: str, max_chars: int) -> list[str]:
words = text.split(" ")
chunks, current = [], ""
for word in words:
if current and len(current) + 1 + len(word) > max_chars:
chunks.append(current)
current = word
else:
current = f"{current} {word}".strip()
if current:
chunks.append(current)
return chunks
def _chunk_paragraph(paragraph: str, max_chars: int) -> list[str]:
if len(paragraph) <= max_chars:
return [paragraph]
pieces = [p.strip() for p in _SHAD_SPLIT_RE.split(paragraph) if p.strip()]
chunks, current = [], ""
for piece in pieces:
if len(piece) > max_chars:
if current:
chunks.append(current)
current = ""
chunks.extend(_chunk_by_words(piece, max_chars))
elif current and len(current) + 1 + len(piece) > max_chars:
chunks.append(current)
current = piece
else:
current = f"{current} {piece}".strip()
if current:
chunks.append(current)
return chunks
def split_into_segments(text: str, max_chars: int = 400) -> list[str]:
"""Split raw text into a list of segment strings."""
paragraphs = re.split(r"\n\s*\n", text.strip())
segments = []
for para in paragraphs:
para = " ".join(para.split())
if not para:
continue
segments.extend(_chunk_paragraph(para, max_chars))
return segments
|