| import re |
|
|
| |
| def clean_text(text): |
| text = re.sub(r'\[[^\]]*\]', '', text) |
| text = re.sub(r'<[^>]*>', '', text) |
| text = re.sub(r'\d+', '', text) |
| text = re.sub(r'\s+', ' ', text) |
| return text.strip().lower() |
|
|
| |
| def split_sentences(text): |
| text = clean_text(text) |
| sentences = re.split(r'[.!?;:]+', text) |
| return [s.strip() for s in sentences if s.strip()] |
|
|
| |
| def split_words(sentence): |
| words = sentence.split() |
| cleaned = [] |
| for w in words: |
| w = w.strip('.,!?;:()"\'—') |
| if w: |
| cleaned.append(w) |
| return cleaned |
|
|
| |
| def split_words_from_text(text): |
| text = clean_text(text) |
| words = text.split() |
| cleaned = [] |
| for w in words: |
| w = w.strip('.,!?;:()"\'—') |
| if w: |
| cleaned.append(w) |
| return cleaned |
