File size: 3,418 Bytes
bde4309 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 | # ============================================
# utils/text_preprocessing.py
# Text cleaning and preprocessing utilities
# ============================================
import re
import string
def normalize_whitespace(text: str) -> str:
"""
Normalize whitespace in text:
- Replace multiple spaces with single space
- Replace multiple newlines with double newline (paragraph break)
- Strip leading/trailing whitespace
"""
# Replace multiple newlines with paragraph separator
text = re.sub(r'\n\s*\n', '\n\n', text)
# Replace single newlines with space (within paragraphs)
text = re.sub(r'(?<!\n)\n(?!\n)', ' ', text)
# Replace multiple spaces with single space
text = re.sub(r' +', ' ', text)
return text.strip()
def remove_special_chars(text: str, keep_punctuation: bool = True) -> str:
"""
Remove special characters from text while preserving readability.
Args:
text: Input text to clean
keep_punctuation: If True, keep basic punctuation marks
Returns:
Cleaned text string
"""
if keep_punctuation:
# Keep letters, numbers, basic punctuation, and Arabic characters
pattern = r'[^\w\s\u0600-\u06FF.,;:?!\'"()\-/]'
else:
# Keep only letters, numbers, spaces, and Arabic characters
pattern = r'[^\w\s\u0600-\u06FF]'
text = re.sub(pattern, '', text)
return text
def remove_headers_footers(text: str) -> str:
"""
Remove common PDF headers/footers like page numbers,
repeated headers, etc.
"""
lines = text.split('\n')
cleaned_lines = []
for line in lines:
stripped = line.strip()
# Skip empty lines
if not stripped:
cleaned_lines.append(line)
continue
# Skip lines that are just page numbers
if re.match(r'^\d{1,4}$', stripped):
continue
# Skip lines like "Page X of Y"
if re.match(r'^[Pp]age\s+\d+\s+(of|/)\s+\d+$', stripped):
continue
# Skip very short lines that look like headers (all caps, < 5 words)
# but keep them if they might be section titles
cleaned_lines.append(line)
return '\n'.join(cleaned_lines)
def clean_text(text: str) -> str:
"""
Main text cleaning pipeline. Applies all preprocessing steps
in the correct order.
Args:
text: Raw text extracted from PDF
Returns:
Cleaned and preprocessed text
"""
if not text or not text.strip():
return ""
# Step 1: Remove headers/footers
text = remove_headers_footers(text)
# Step 2: Remove special characters
text = remove_special_chars(text, keep_punctuation=True)
# Step 3: Normalize whitespace
text = normalize_whitespace(text)
# Step 4: Final strip
text = text.strip()
return text
def get_sentences(text: str) -> list:
"""
Split text into sentences using regex-based sentence boundary detection.
Handles both English and Arabic sentence endings.
Args:
text: Input text
Returns:
List of sentence strings
"""
# Split on sentence-ending punctuation followed by space or end of string
sentences = re.split(r'(?<=[.!?؟])\s+', text)
# Filter out empty sentences
sentences = [s.strip() for s in sentences if s.strip()]
return sentences
|