| """Deterministic Persian text normalization before G2P. |
| |
| The G2P model should predict pronunciation, not perform arithmetic. This module |
| expands the number forms that previously caused structural decoder failures while |
| leaving punctuation available to the pacing layer. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import re |
|
|
|
|
| _DIGIT_TRANSLATION = str.maketrans("۰۱۲۳۴۵۶۷۸۹٠١٢٣٤٥٦٧٨٩", "01234567890123456789") |
| _ONES = ("صفر", "یک", "دو", "سه", "چهار", "پنج", "شش", "هفت", "هشت", "نه") |
| _TEENS = { |
| 10: "ده", 11: "یازده", 12: "دوازده", 13: "سیزده", 14: "چهارده", |
| 15: "پانزده", 16: "شانزده", 17: "هفده", 18: "هجده", 19: "نوزده", |
| } |
| _TENS = {20: "بیست", 30: "سی", 40: "چهل", 50: "پنجاه", 60: "شصت", 70: "هفتاد", 80: "هشتاد", 90: "نود"} |
| _HUNDREDS = {100: "صد", 200: "دویست", 300: "سیصد", 400: "چهارصد", 500: "پانصد", 600: "ششصد", 700: "هفتصد", 800: "هشتصد", 900: "نهصد"} |
| _SCALES = ((10**12, "تریلیون"), (10**9, "میلیارد"), (10**6, "میلیون"), (10**3, "هزار")) |
| _NUMBER = re.compile( |
| r"(?<![\w])(?P<sign>[+\-−]?)(?P<int>[0-9]+(?:[٬,][0-9]{3})*)(?:(?P<decimal>[٫.])(?P<fraction>[0-9]+))?(?P<percent>[٪%])?(?![\w])" |
| ) |
|
|
|
|
| def integer_to_persian(number: int) -> str: |
| if number < 0: |
| return "منفی " + integer_to_persian(-number) |
| if number < 10: |
| return _ONES[number] |
| if number < 20: |
| return _TEENS[number] |
| if number < 100: |
| tens, rest = divmod(number, 10) |
| parts = [_TENS[tens * 10]] |
| if rest: |
| parts.append(_ONES[rest]) |
| return " و ".join(parts) |
| if number < 1000: |
| hundreds, rest = divmod(number, 100) |
| parts = [_HUNDREDS[hundreds * 100]] |
| if rest: |
| parts.append(integer_to_persian(rest)) |
| return " و ".join(parts) |
| for scale, label in _SCALES: |
| if number >= scale: |
| count, rest = divmod(number, scale) |
| parts = [f"{integer_to_persian(count)} {label}"] |
| if rest: |
| parts.append(integer_to_persian(rest)) |
| return " و ".join(parts) |
| raise ValueError(f"number is outside the supported range: {number}") |
|
|
|
|
| def _expand_number(match: re.Match[str]) -> str: |
| integer_digits = match.group("int").replace("٬", "").replace(",", "") |
| parts: list[str] = [] |
| sign = match.group("sign") |
| if sign in {"-", "−"}: |
| parts.append("منفی") |
| elif sign == "+": |
| parts.append("مثبت") |
| parts.append(integer_to_persian(int(integer_digits))) |
| fraction = match.group("fraction") |
| if fraction is not None: |
| parts.append("ممیز") |
| parts.append(" ".join(_ONES[int(digit)] for digit in fraction)) |
| if match.group("percent"): |
| parts.append("درصد") |
| return " ".join(parts) |
|
|
|
|
| def normalize_persian_for_g2p(text: str) -> str: |
| """Expand Persian/Arabic/ASCII numerals and retain authored punctuation.""" |
| ascii_digits = str(text).translate(_DIGIT_TRANSLATION) |
| return _NUMBER.sub(_expand_number, ascii_digits) |
|
|
