Hugging Face's logo

Spaces:
Amr-Yasserr
/
ara-route-parser
Sleeping

App Files Community
ara-route-parser
File size: 3,738 Bytes
bdc3fd6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
"""
Arabic Text Normalizer for Route Extraction Pipeline.

Handles:
- Hamza/Alif normalization
- Ta Marbuta/Ha equivalence
- Egyptian filler word removal
- Arabic prefix handling with landmark protection
- Whitespace normalization
"""

import re

# ============================================================
# Protected words that start with common prefix letters
# ============================================================
PROTECTED_PREFIXES = [
    "لبنان", "لوران", "لوتس", "لوكاندة",
    "بورسعيد", "بورفؤاد", "بني سويف", "بولاق", "بنها",
]

# ============================================================
# Egyptian noise/filler words to strip
# ============================================================
NOISE_PHRASES = [
    "لو سمحت", "من فضلك", "يا ريس", "يا باشا", "يا حج",
    "يا كابتن", "يا معلم", "يا عم", "الله يخليك",
    "عايز اروح", "عايزة اروح", "محتاج اوصل",
    "بسرعة", "دلوقتي", "ممكن", "خدني", "وصلني",
    "عايزة", "عايز", "يعني", "كده", "بقى", "بالراحة",
    "طب", "يلا", "بقولك", "اسمع", "شوف",
]


def normalize_arabic(text: str) -> str:
    """
    Full Arabic text normalization pipeline.
    
    Normalizes hamza variants, ta marbuta, diacritics, and whitespace.
    Does NOT remove noise words (that's a separate step).
    """
    if not text:
        return ""

    t = text.strip()

    # Remove diacritics (tashkeel)
    t = re.sub(r'[\u064B-\u065F\u0670]', '', t)

    # Normalize Alif variants
    t = t.replace('أ', 'ا').replace('إ', 'ا').replace('آ', 'ا')

    # Normalize Alif Maqsura to Ya
    t = t.replace('ى', 'ي')

    # Normalize Ta Marbuta to Ha (for matching purposes)
    # Note: We keep the original in the output, only normalize for matching
    # t = t.replace('ة', 'ه')  # Only used in matching, not preprocessing

    # Normalize whitespace
    t = re.sub(r'\s+', ' ', t).strip()

    return t


def remove_noise(text: str) -> str:
    """Remove Egyptian filler/noise words from text."""
    cleaned = text
    for noise in NOISE_PHRASES:
        cleaned = cleaned.replace(noise, '')

    # Clean up extra whitespace
    cleaned = re.sub(r'\s+', ' ', cleaned).strip()
    return cleaned


def strip_prefixes(text: str) -> str:
    """
    Strip Arabic definite article and preposition prefixes.
    Protects known landmark names that naturally start with these letters.
    """
    if not text:
        return ""

    cleaned = text.strip()

    # Check if word is protected
    for protected in PROTECTED_PREFIXES:
        if cleaned.startswith(protected):
            return cleaned

    # Strip leading prepositions/articles: لل, لـ, ل, ب, ال
    cleaned = re.sub(r'^(?:لل|لـ|ل|بـ|ب)(?![\s])', '', cleaned)

    return cleaned.strip()


def normalize_for_matching(text: str) -> str:
    """
    Aggressive normalization specifically for fuzzy matching.
    Strips articles, normalizes ta marbuta, etc.
    """
    if not text:
        return ""

    t = normalize_arabic(text)

    # Strip definite article ال
    t = re.sub(r'^ال', '', t)

    # Normalize Ta Marbuta → Ha
    t = t.replace('ة', 'ه')

    # Remove common prefixes (شارع, ميدان, محطة)
    t = re.sub(r'^(?:شارع|ميدان|محطة|موقف)\s*', '', t)

    return t.strip()


def full_preprocess(text: str) -> str:
    """
    Full preprocessing pipeline for incoming transcripts.
    
    1. Normalize Arabic characters
    2. Remove noise/filler words
    3. Normalize whitespace
    """
    t = normalize_arabic(text)
    t = remove_noise(t)
    return t