Spaces:
Sleeping
Sleeping
| import random | |
| import re | |
| from num2fawords import words | |
| from num2words import num2words | |
| from rules.base import BaseProcessor | |
| from utils import NO_LETTERS, detect_digit_language | |
| # ASCII, Persian (۰-۹) and Arabic-Indic (٠-٩) digits. | |
| DIGIT = "0-9۰-۹٠-٩" | |
| TO_ASCII = str.maketrans("۰۱۲۳۴۵۶۷۸۹٠١٢٣٤٥٦٧٨٩", "01234567890123456789") | |
| THOUSANDS_SEPARATORS = ",٬" | |
| DECIMAL_SEPARATORS = ".٫" | |
| # `1,000,000`, `12`, `۱٬۰۰۰`, `3.5`, `۳٫۵`, `1,000.25`. | |
| NUMBER = ( | |
| rf"(?:[{DIGIT}]{{1,3}}(?:[{THOUSANDS_SEPARATORS}][{DIGIT}]{{3}})+|[{DIGIT}]+)" | |
| rf"(?:[{DECIMAL_SEPARATORS}][{DIGIT}]+)?" | |
| ) | |
| # A number must stand alone. It is left alone when it is glued to a letter, a | |
| # digit or an underscore on either side (`3D`, `mp3`, `H2O`, `A4`, `5G`, | |
| # `covid19`), or sits inside a time, date, version, range, URL, e-mail, | |
| # hashtag or `1,5` style number (`10:30`, `2024/01/05`, `1.2.3`, `10-12`, | |
| # `#2024`, `a@3`). | |
| NUMBER_PATTERN = re.compile( | |
| rf"(?<![\w.,٫٬:/\\\-–+#@$€£])" | |
| rf"(?P<number>{NUMBER})" | |
| rf"(?![\w@]|[.,٫٬:/\\][{DIGIT}]|[-–][{DIGIT}]|[-–][A-Za-z]\b)" | |
| ) | |
| MAX_INTEGER_DIGITS = 15 | |
| MAX_FLOAT_FRACTION_DIGITS = 9 | |
| ENGLISH_YEAR_WORDS = r"(?:in|since|year|until|till|circa|c\.|AD|BC)" | |
| ENGLISH_YEAR_PATTERN = re.compile(rf"(?:^|\s|[(\[\"'“‘]){ENGLISH_YEAR_WORDS}\s+$", re.IGNORECASE) | |
| PERSIAN_YEAR_PATTERN = re.compile(r"(?:^|\s|[(\[\"'«])سال\s+$") | |
| class NumberProcessor(BaseProcessor): | |
| """Reads any remaining standalone number as words, in Persian or English. | |
| English: `I have 20 apples` -> `I have twenty apples` | |
| `1,000` -> `one thousand`, `3.5` -> `three point five` | |
| Persian: `من ۲۰ سیب دارم` -> `من بیست سیب دارم` | |
| `۱٬۰۰۰` -> `یک هزار`, `۳٫۵` -> `سه و پنج دهم` | |
| It is the last step of the pipeline, so every more specific processor | |
| (units, currency, dates, times, phone numbers, ranges, fractions, ...) gets | |
| the number first and this only reads what is left. | |
| Handled: | |
| * integers in ASCII, Persian or Arabic-Indic digits, with thousands | |
| separators (`,` or `٬`); | |
| * decimals (`.` or `٫`). English reads every digit after the point, so | |
| `3.50` -> `three point five zero` and `0.05` -> `zero point zero five`. | |
| Persian reads the decimal as a fraction (`۰٫۰۵` -> `پنج صدم`), and a | |
| decimal with more than nine digits after the point as `ممیز` followed by | |
| the digits; | |
| * negative numbers are read by `NegativeSignProcessor` before this runs. | |
| Language rules: | |
| * English years are read the way English speakers say them, but only after | |
| a word such as `in`, `since`, `year`, `until`, `till`, `circa`, `AD`, | |
| `BC`: `in 1999` -> `in nineteen ninety-nine`, `since 1905` -> | |
| `since nineteen oh five`, `in 2024` -> `in twenty twenty-four`, | |
| `in 2005` -> `in two thousand and five`. | |
| * Persian years after `سال` drop the leading `یک`: `سال ۱۴۰۳` -> | |
| `سال هزار و چهارصد و سه`. | |
| * An integer with a leading zero (`007`, `۰۰۷`) is read digit by digit, | |
| as an identifier: `zero zero seven` / `صفر صفر هفت`. | |
| Left alone: numbers glued to letters or digits (`3D`, `mp3`, `H2O`, `A4`, | |
| `5G`), times, dates, versions (`1.2.3`), `1,5` style numbers, numbers in | |
| URLs, e-mails and hashtags, `+5`, integers longer than 15 digits, and text | |
| in "mixed" or any other language. | |
| With no letters in the text, the digits pick the language: Persian digits -> | |
| Persian, ASCII digits -> English. | |
| """ | |
| def __init__(self, epsilon: float = 1.0): | |
| super(NumberProcessor, self).__init__(epsilon=epsilon) | |
| def process(self, text: str, language: str = None, *args, **kwargs) -> str: | |
| if random.random() > self.epsilon: | |
| return text | |
| if re.search(rf"[{DIGIT}]", text) is None: | |
| return text | |
| if language in (None, NO_LETTERS): | |
| language = detect_digit_language(text) | |
| if language not in ("fa", "en"): | |
| return text | |
| return NUMBER_PATTERN.sub(lambda match: self._convert(match, language), text) | |
| def _convert(self, match: re.Match, language: str) -> str: | |
| raw = match.group("number") | |
| clean = raw.translate(TO_ASCII) | |
| has_thousands_separator = any(separator in clean for separator in THOUSANDS_SEPARATORS) | |
| for separator in THOUSANDS_SEPARATORS: | |
| clean = clean.replace(separator, "") | |
| clean = clean.replace("٫", ".") | |
| whole, _, fraction = clean.partition(".") | |
| has_fraction = "." in clean | |
| if has_fraction: | |
| if len(whole) > 1 and whole.startswith("0"): | |
| return raw | |
| return self._decimal_to_words(whole, fraction, language) | |
| if len(whole) > MAX_INTEGER_DIGITS: | |
| return raw | |
| if len(whole) > 1 and whole.startswith("0") and not has_thousands_separator: | |
| return self._digits_to_words(whole, language) | |
| before = match.string[:match.start()] | |
| return self._integer_to_words(int(whole), language, before) | |
| def _integer_to_words(value: int, language: str, before: str = "") -> str: | |
| if language == "fa": | |
| text = words(value) | |
| if 1000 <= value <= 1999 and PERSIAN_YEAR_PATTERN.search(before): | |
| text = text.removeprefix("یک ") | |
| return text | |
| if 1100 <= value <= 2099 and not 2000 <= value <= 2009 \ | |
| and ENGLISH_YEAR_PATTERN.search(before): | |
| return NumberProcessor._english_year(value) | |
| return num2words(value) | |
| def _english_year(value: int) -> str: | |
| """1999 -> `nineteen ninety-nine`, 1905 -> `nineteen oh five`, 1900 -> `nineteen hundred`.""" | |
| high, low = divmod(value, 100) | |
| if low == 0: | |
| return f"{num2words(high)} hundred" | |
| if low < 10: | |
| return f"{num2words(high)} oh {num2words(low)}" | |
| return f"{num2words(high)} {num2words(low)}" | |
| def _decimal_to_words(whole: str, fraction: str, language: str) -> str: | |
| if language == "fa": | |
| if len(fraction) > MAX_FLOAT_FRACTION_DIGITS or len(whole) > MAX_INTEGER_DIGITS: | |
| spoken_fraction = " ".join(words(int(digit)) for digit in fraction) | |
| return f"{words(int(whole))} ممیز {spoken_fraction}" | |
| return words(float(f"{whole}.{fraction}")) | |
| spoken_fraction = " ".join(num2words(int(digit)) for digit in fraction) | |
| return f"{num2words(int(whole))} point {spoken_fraction}" | |
| def _digits_to_words(digits: str, language: str) -> str: | |
| spell = words if language == "fa" else num2words | |
| return " ".join(spell(int(digit)) for digit in digits) | |