Spaces:
Sleeping
Sleeping
Download src/preprocessing/text_preprocessor.py from koushik2109/mindscope-Model: direct link, hf CLI and curl.
- Browser
- Download file 1.08 kB
-
https://huggingface.co/spaces/koushik2109/mindscope-Model/resolve/main/src/preprocessing/text_preprocessor.py
- Command line
-
hf download hf://spaces/koushik2109/mindscope-Model/src/preprocessing/text_preprocessor.py
-
curl -L -o text_preprocessor.py https://huggingface.co/spaces/koushik2109/mindscope-Model/resolve/main/src/preprocessing/text_preprocessor.py
1.08 kB
| import re | |
| from typing import List | |
| class TextPreprocessor: | |
| """Text preprocessing and chunking for transcript inputs.""" | |
| def normalize(self, text: str) -> str: | |
| text = text.strip() | |
| text = re.sub(r"\s+", " ", text) | |
| return text | |
| def split_sentences(self, text: str) -> List[str]: | |
| sentences = re.split(r"(?<=[.!?])\s+", text) | |
| return [sentence.strip() for sentence in sentences if sentence.strip()] | |
| def chunk_text(self, text: str, max_tokens: int = 256) -> List[str]: | |
| sentences = self.split_sentences(self.normalize(text)) | |
| chunks: List[str] = [] | |
| current: List[str] = [] | |
| for sentence in sentences: | |
| current.append(sentence) | |
| if len(" ".join(current).split()) >= max_tokens: | |
| chunks.append(" ".join(current)) | |
| current = [] | |
| if current: | |
| chunks.append(" ".join(current)) | |
| return chunks | |
| def preprocess(self, text: str) -> List[str]: | |
| normalized = self.normalize(text) | |
| return self.chunk_text(normalized) | |