acpcchatbot_testing_first / app /utils /text_cleaning.py
acpcchatbotadmin's picture
Upload 40 files
481dfe6 verified
Raw History Blame Contribute Delete
457 Bytes
import re
def clean_pdf_text(text: str) -> str:
"""Normalize text extracted from PDF chunks."""
# Collapse multiple whitespace into single space
text = re.sub(r"[ \t]+", " ", text)
# Collapse more than 2 consecutive newlines
text = re.sub(r"\n{3,}", "\n\n", text)
# Remove null bytes and control characters (except newlines/tabs)
text = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]", "", text)
return text.strip()