import random import re from num2fawords import words from num2words import num2words from rules.base import BaseProcessor from utils import NO_LETTERS, detect_digit_language # ASCII, Persian (۰-۹) and Arabic-Indic (٠-٩) digits. DIGIT = "0-9۰-۹٠-٩" TO_ASCII = str.maketrans("۰۱۲۳۴۵۶۷۸۹٠١٢٣٤٥٦٧٨٩", "01234567890123456789") THOUSANDS_SEPARATORS = ",٬" DECIMAL_SEPARATORS = ".٫" # `1,000,000`, `12`, `۱٬۰۰۰`, `3.5`, `۳٫۵`, `1,000.25`. NUMBER = ( rf"(?:[{DIGIT}]{{1,3}}(?:[{THOUSANDS_SEPARATORS}][{DIGIT}]{{3}})+|[{DIGIT}]+)" rf"(?:[{DECIMAL_SEPARATORS}][{DIGIT}]+)?" ) # A number must stand alone. It is left alone when it is glued to a letter, a # digit or an underscore on either side (`3D`, `mp3`, `H2O`, `A4`, `5G`, # `covid19`), or sits inside a time, date, version, range, URL, e-mail, # hashtag or `1,5` style number (`10:30`, `2024/01/05`, `1.2.3`, `10-12`, # `#2024`, `a@3`). NUMBER_PATTERN = re.compile( rf"(?{NUMBER})" rf"(?![\w‌@]|[.,٫٬:/\\][{DIGIT}]|[-–][{DIGIT}]|[-–][A-Za-z]\b)" ) MAX_INTEGER_DIGITS = 15 MAX_FLOAT_FRACTION_DIGITS = 9 ENGLISH_YEAR_WORDS = r"(?:in|since|year|until|till|circa|c\.|AD|BC)" ENGLISH_YEAR_PATTERN = re.compile(rf"(?:^|\s|[(\[\"'“‘]){ENGLISH_YEAR_WORDS}\s+$", re.IGNORECASE) PERSIAN_YEAR_PATTERN = re.compile(r"(?:^|\s|[(\[\"'«])سال\s+$") class NumberProcessor(BaseProcessor): """Reads any remaining standalone number as words, in Persian or English. English: `I have 20 apples` -> `I have twenty apples` `1,000` -> `one thousand`, `3.5` -> `three point five` Persian: `من ۲۰ سیب دارم` -> `من بیست سیب دارم` `۱٬۰۰۰` -> `یک هزار`, `۳٫۵` -> `سه و پنج دهم` It is the last step of the pipeline, so every more specific processor (units, currency, dates, times, phone numbers, ranges, fractions, ...) gets the number first and this only reads what is left. Handled: * integers in ASCII, Persian or Arabic-Indic digits, with thousands separators (`,` or `٬`); * decimals (`.` or `٫`). English reads every digit after the point, so `3.50` -> `three point five zero` and `0.05` -> `zero point zero five`. Persian reads the decimal as a fraction (`۰٫۰۵` -> `پنج صدم`), and a decimal with more than nine digits after the point as `ممیز` followed by the digits; * negative numbers are read by `NegativeSignProcessor` before this runs. Language rules: * English years are read the way English speakers say them, but only after a word such as `in`, `since`, `year`, `until`, `till`, `circa`, `AD`, `BC`: `in 1999` -> `in nineteen ninety-nine`, `since 1905` -> `since nineteen oh five`, `in 2024` -> `in twenty twenty-four`, `in 2005` -> `in two thousand and five`. * Persian years after `سال` drop the leading `یک`: `سال ۱۴۰۳` -> `سال هزار و چهارصد و سه`. * An integer with a leading zero (`007`, `۰۰۷`) is read digit by digit, as an identifier: `zero zero seven` / `صفر صفر هفت`. Left alone: numbers glued to letters or digits (`3D`, `mp3`, `H2O`, `A4`, `5G`), times, dates, versions (`1.2.3`), `1,5` style numbers, numbers in URLs, e-mails and hashtags, `+5`, integers longer than 15 digits, and text in "mixed" or any other language. With no letters in the text, the digits pick the language: Persian digits -> Persian, ASCII digits -> English. """ def __init__(self, epsilon: float = 1.0): super(NumberProcessor, self).__init__(epsilon=epsilon) def process(self, text: str, language: str = None, *args, **kwargs) -> str: if random.random() > self.epsilon: return text if re.search(rf"[{DIGIT}]", text) is None: return text if language in (None, NO_LETTERS): language = detect_digit_language(text) if language not in ("fa", "en"): return text return NUMBER_PATTERN.sub(lambda match: self._convert(match, language), text) def _convert(self, match: re.Match, language: str) -> str: raw = match.group("number") clean = raw.translate(TO_ASCII) has_thousands_separator = any(separator in clean for separator in THOUSANDS_SEPARATORS) for separator in THOUSANDS_SEPARATORS: clean = clean.replace(separator, "") clean = clean.replace("٫", ".") whole, _, fraction = clean.partition(".") has_fraction = "." in clean if has_fraction: if len(whole) > 1 and whole.startswith("0"): return raw return self._decimal_to_words(whole, fraction, language) if len(whole) > MAX_INTEGER_DIGITS: return raw if len(whole) > 1 and whole.startswith("0") and not has_thousands_separator: return self._digits_to_words(whole, language) before = match.string[:match.start()] return self._integer_to_words(int(whole), language, before) @staticmethod def _integer_to_words(value: int, language: str, before: str = "") -> str: if language == "fa": text = words(value) if 1000 <= value <= 1999 and PERSIAN_YEAR_PATTERN.search(before): text = text.removeprefix("یک ") return text if 1100 <= value <= 2099 and not 2000 <= value <= 2009 \ and ENGLISH_YEAR_PATTERN.search(before): return NumberProcessor._english_year(value) return num2words(value) @staticmethod def _english_year(value: int) -> str: """1999 -> `nineteen ninety-nine`, 1905 -> `nineteen oh five`, 1900 -> `nineteen hundred`.""" high, low = divmod(value, 100) if low == 0: return f"{num2words(high)} hundred" if low < 10: return f"{num2words(high)} oh {num2words(low)}" return f"{num2words(high)} {num2words(low)}" @staticmethod def _decimal_to_words(whole: str, fraction: str, language: str) -> str: if language == "fa": if len(fraction) > MAX_FLOAT_FRACTION_DIGITS or len(whole) > MAX_INTEGER_DIGITS: spoken_fraction = " ".join(words(int(digit)) for digit in fraction) return f"{words(int(whole))} ممیز {spoken_fraction}" return words(float(f"{whole}.{fraction}")) spoken_fraction = " ".join(num2words(int(digit)) for digit in fraction) return f"{num2words(int(whole))} point {spoken_fraction}" @staticmethod def _digits_to_words(digits: str, language: str) -> str: spell = words if language == "fa" else num2words return " ".join(spell(int(digit)) for digit in digits)