Spaces:
Sleeping
Sleeping
File size: 1,081 Bytes
8d7b4b1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 | import re
from typing import List
class TextPreprocessor:
"""Text preprocessing and chunking for transcript inputs."""
def normalize(self, text: str) -> str:
text = text.strip()
text = re.sub(r"\s+", " ", text)
return text
def split_sentences(self, text: str) -> List[str]:
sentences = re.split(r"(?<=[.!?])\s+", text)
return [sentence.strip() for sentence in sentences if sentence.strip()]
def chunk_text(self, text: str, max_tokens: int = 256) -> List[str]:
sentences = self.split_sentences(self.normalize(text))
chunks: List[str] = []
current: List[str] = []
for sentence in sentences:
current.append(sentence)
if len(" ".join(current).split()) >= max_tokens:
chunks.append(" ".join(current))
current = []
if current:
chunks.append(" ".join(current))
return chunks
def preprocess(self, text: str) -> List[str]:
normalized = self.normalize(text)
return self.chunk_text(normalized)
|