File size: 1,081 Bytes
8d7b4b1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
import re
from typing import List


class TextPreprocessor:
    """Text preprocessing and chunking for transcript inputs."""

    def normalize(self, text: str) -> str:
        text = text.strip()
        text = re.sub(r"\s+", " ", text)
        return text

    def split_sentences(self, text: str) -> List[str]:
        sentences = re.split(r"(?<=[.!?])\s+", text)
        return [sentence.strip() for sentence in sentences if sentence.strip()]

    def chunk_text(self, text: str, max_tokens: int = 256) -> List[str]:
        sentences = self.split_sentences(self.normalize(text))
        chunks: List[str] = []
        current: List[str] = []
        for sentence in sentences:
            current.append(sentence)
            if len(" ".join(current).split()) >= max_tokens:
                chunks.append(" ".join(current))
                current = []
        if current:
            chunks.append(" ".join(current))
        return chunks

    def preprocess(self, text: str) -> List[str]:
        normalized = self.normalize(text)
        return self.chunk_text(normalized)