processing/texts.py

import re


def clean_text(text: str) -> str:
    """
    Clean the text by removing unwanted characters while preserving document structure.
    """
    text = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]', '', text)  # remove control chars (keep \n \t \r)
    text = re.sub(r'\r\n?', '\n', text)           # normalize line endings
    text = re.sub(r'\n{3,}', '\n\n', text)         # collapse excessive newlines
    text = re.sub(r'[^\S\n]{3,}', ' ', text)       # collapse excessive spaces (not newlines)
    return text.strip()