Spaces:
Runtime error
Runtime error
File size: 1,941 Bytes
0a19322 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | """Clean page text and split it into sentence-aware, overlapping chunks."""
import re
_SENT_SPLIT = re.compile(r"(?<=[.!?\u0964])\s+") # \u0964 = Hindi danda
def paragraphs(text: str) -> list[str]:
text = text.replace("\r", "")
text = re.sub(r"-\n(?=[a-z])", "", text) # join words hyphenated across lines
text = re.sub(r"[ \t\u00a0]+", " ", text)
paras = re.split(r"\n\s*\n", text)
paras = [re.sub(r"\s*\n\s*", " ", p).strip() for p in paras]
return [p for p in paras if p]
def _split_units(par: str, size: int) -> list[str]:
units: list[str] = []
for sent in _SENT_SPLIT.split(par):
sent = sent.strip()
while len(sent) > size: # one huge "sentence" (tables, code): cut at a space
cut = sent.rfind(" ", 0, size)
cut = cut if cut > size * 0.5 else size
units.append(sent[:cut].strip())
sent = sent[cut:].strip()
if sent:
units.append(sent)
return units
def chunk_text(text: str, size: int, overlap: int) -> list[str]:
units: list[str] = []
for par in paragraphs(text):
units.extend(_split_units(par, size))
chunks: list[str] = []
cur: list[str] = []
cur_len = 0
for u in units:
add = len(u) + (1 if cur else 0)
if cur and cur_len + add > size:
chunks.append(" ".join(cur))
keep: list[str] = [] # carry the last sentences over as overlap
keep_len = 0
for prev in reversed(cur):
if keep_len + len(prev) + 1 > overlap:
break
keep.insert(0, prev)
keep_len += len(prev) + 1
cur = keep
cur_len = sum(len(x) for x in cur) + max(0, len(cur) - 1)
add = len(u) + (1 if cur else 0)
cur.append(u)
cur_len += add
if cur:
chunks.append(" ".join(cur))
return [c for c in chunks if c.strip()]
|