Upload 13 files

rules/base.py

@@ -0,0 +1,9 @@
+from abc import ABC
+
+
+class BaseProcessor(ABC):
+    def __init__(self, epsilon:float=1.0):
+        self.epsilon = epsilon
+
+    def process(self, *args, **kwargs):
+        raise NotImplementedError

rules/cards.py

@@ -0,0 +1,102 @@
+import random
+import re
+
+from num2words import num2words
+
+from rules.base import BaseProcessor
+from rules.phone import PhoneNumberProcessor
+from utils import NO_LETTERS, detect_digit_language
+
+
+# ASCII, Persian (۰-۹) and Arabic-Indic (٠-٩) digits.
+DIGIT = "0-9۰-۹٠-٩"
+TO_ASCII = str.maketrans("۰۱۲۳۴۵۶۷۸۹٠١٢٣٤٥٦٧٨٩", "01234567890123456789")
+
+GROUP = rf"[{DIGIT}]{{4}}"
+SEPARATOR = r"(?:[ \t]*[-–][ \t]*|[ \t]+)"
+
+# Four groups of four digits (`6219 - 8765 - 2345 - 9031`, `6219-8765-2345-9031`,
+# `6219 8765 2345 9031`) or sixteen digits in a row. It must not be glued to a
+# word, a longer number, a decimal or a URL path.
+CARD_PATTERN = re.compile(
+    rf"(?<![\w.,/+\-])"
+    rf"(?<![{DIGIT}][ \t])(?<![{DIGIT}][-–])(?<![{DIGIT}][-–][ \t])(?<![{DIGIT}][ \t][-–])(?<![{DIGIT}][ \t][-–][ \t])"
+    rf"(?:{GROUP}(?:{SEPARATOR}{GROUP}){{3}}|[{DIGIT}]{{16}})"
+    rf"(?![\w/]|[-–.,][{DIGIT}]|{SEPARATOR}[{DIGIT}])"
+)
+
+# English reads a pair that starts with 0 as "oh five" and `00` as "double zero".
+ENGLISH_ZERO = "oh"
+ENGLISH_DOUBLE_ZERO = "double zero"
+
+GROUP_JOINER = " - "
+PAIR_JOINER = {"fa": "، ", "en": ", "}
+
+
+class CardNumberProcessor(BaseProcessor):
+    """Reads 16-digit bank card numbers in pairs of digits.
+
+    Persian: `۶۲۱۹ - ۸۷۶۵ - ۲۳۴۵ - ۹۰۳۱`
+             -> `شصت و دو، نوزده - هشتاد و هفت، شصت و پنج - بیست و سه، چهل و پنج - نود، سی و یک`
+    English: `4111 1111 1111 1111`
+             -> `forty-one, eleven - eleven, eleven - eleven, eleven - eleven, eleven`
+             `5105-1051-0510-5100`
+             -> `fifty-one, oh five - ten, fifty-one - oh five, ten - fifty-one, double zero`
+
+    Each group of four digits is said as two numbers separated by a pause
+    (comma), and the groups are separated by ` - `. A pair that starts with `0`
+    keeps its zero: Persian `05` -> `صفر پنج`, `00` -> `دو صفر`; English `05` ->
+    `oh five`, `00` -> `double zero`.
+
+    Recognised: four groups of four digits separated by spaces or hyphens (or
+    sixteen digits in a row), written with ASCII, Persian or Arabic-Indic digits,
+    with any first digit. Left alone: other lengths (15-digit Amex, 19-digit),
+    masked numbers (`6219-****-****-9031`), numbers glued to words, and decimals.
+
+    With no letters in the text, the digits pick the language: Persian digits ->
+    Persian, ASCII digits -> English. This processor must run before
+    `PhoneNumberProcessor`, because a word like `شماره` ("number") before a card
+    would otherwise make the phone processor read it.
+    """
+
+    def __init__(self, epsilon: float = 1.0):
+        super(CardNumberProcessor, self).__init__(epsilon=epsilon)
+
+    def process(self, text: str, language: str = None, *args, **kwargs) -> str:
+        if random.random() > self.epsilon:
+            return text
+        if re.search(rf"[{DIGIT}]", text) is None:
+            return text
+
+        if language in (None, NO_LETTERS):
+            language = detect_digit_language(text)
+        if language not in ("fa", "en"):
+            return text
+
+        return CARD_PATTERN.sub(lambda match: self._convert(match, language), text)
+
+    def _convert(self, match: re.Match, language: str) -> str:
+        digits = re.sub(rf"[^{DIGIT}]", "", match.group(0)).translate(TO_ASCII)
+        if len(digits) != 16:
+            return match.group(0)
+
+        groups = [digits[index:index + 4] for index in range(0, 16, 4)]
+        return GROUP_JOINER.join(
+            PAIR_JOINER[language].join(
+                self._pair_to_words(group[start:start + 2], language) for start in (0, 2)
+            )
+            for group in groups
+        )
+
+    @staticmethod
+    def _pair_to_words(pair: str, language: str) -> str:
+        if language == "fa":
+            # Same reading as inside a phone number: `62` -> `شصت و دو`,
+            # `05` -> `صفر پنج`, `00` -> `دو صفر`.
+            return PhoneNumberProcessor._persian_number(pair)
+
+        if pair == "00":
+            return ENGLISH_DOUBLE_ZERO
+        if pair.startswith("0"):
+            return f"{ENGLISH_ZERO} {num2words(int(pair))}"
+        return num2words(int(pair))

rules/currency.py

@@ -0,0 +1,164 @@
+import random
+import re
+from dataclasses import dataclass
+from decimal import Decimal
+
+from num2fawords import words
+from num2words import num2words
+from persiantools import digits
+
+from rules.base import BaseProcessor
+from utils import NO_LETTERS, detect_digit_language
+
+
+@dataclass(frozen=True)
+class Currency:
+    symbol: str
+    fa_unit: str
+    fa_subunit: str
+    en_unit: str
+    en_units: str
+    en_subunit: str
+    en_subunits: str
+
+
+CURRENCIES = {
+    "$": Currency("$", "دلار", "سنت", "dollar", "dollars", "cent", "cents"),
+    "€": Currency("€", "یورو", "سنت", "euro", "euros", "cent", "cents"),
+    "£": Currency("£", "پوند", "پنس", "pound", "pounds", "penny", "pence"),
+}
+
+SYMBOLS = "".join(re.escape(symbol) for symbol in CURRENCIES)
+AMOUNT = r"\d+(?:[.,٫٬]\d+)*"
+
+# Matches a single token such as `3$`, `$3`, `۳€`, `£12.50`, `1,000$`,
+# optionally wrapped in punctuation, e.g. `(3$)` or `£12.50.`.
+CURRENCY_PATTERN = re.compile(
+    rf"""
+    ^
+    (?P<lead>[^\w{SYMBOLS}]*)
+    (?:
+        (?P<prefix_symbol>[{SYMBOLS}])(?P<prefixed>{AMOUNT})
+        |
+        (?P<suffixed>{AMOUNT})(?P<suffix_symbol>[{SYMBOLS}])
+    )
+    (?P<trail>[^\w{SYMBOLS}]*)
+    $
+    """,
+    re.VERBOSE,
+)
+
+
+class CurrencyProcessor(BaseProcessor):
+    """Converts amounts such as `3$`, `€3` or `£12.50` into spoken text.
+
+    Supported symbols: `$` (dollar), `€` (euro), `£` (pound).
+
+    Persian: `۳$`     -> `سه دلار`
+             `€12.50` -> `دوازده یورو و پنجاه سنت`
+             `£12.50` -> `دوازده پوند و پنجاه پنس`
+    English: `3$`     -> `three dollars`
+             `$12.50` -> `twelve dollars and fifty cents`
+             `£12.50` -> `twelve pounds and fifty pence`
+
+    Amounts with more than two decimal places are read as a plain decimal
+    number (e.g. `3.141$` -> `three point one four one dollars`).
+
+    If the text has no letters at all, the digits pick the language:
+             `۱۰۰۰$` -> `یک هزار دلار`
+             `1000$` -> `one thousand dollars`
+    """
+
+    def __init__(self, epsilon: float = 1.0):
+        super(CurrencyProcessor, self).__init__(epsilon=epsilon)
+
+    def process(self, token: str, language: str = "en", *args, **kwargs) -> str:
+        if random.random() > self.epsilon:
+            return token
+
+        match = CURRENCY_PATTERN.fullmatch(token)
+        if match is None:
+            return token
+
+        # A text with no letters at all (e.g. just `۱۰۰۰$`) has no language, so
+        # the digits decide: Persian digits -> Persian, ASCII digits -> English.
+        if language == NO_LETTERS:
+            language = detect_digit_language(token)
+
+        amount = digits.fa_to_en(match.group("prefixed") or match.group("suffixed"))
+        symbol = match.group("prefix_symbol") or match.group("suffix_symbol")
+        currency = CURRENCIES[symbol]
+
+        if language == "fa":
+            text = self._amount_to_persian(amount, currency)
+        elif language == "en":
+            text = self._amount_to_english(amount, currency)
+        else:
+            return token
+
+        return f"{match.group('lead')}{text}{match.group('trail')}"
+
+    @staticmethod
+    def is_currency(token: str) -> bool:
+        return CURRENCY_PATTERN.fullmatch(token) is not None
+
+    @staticmethod
+    def _amount_to_persian(amount: str, currency: Currency) -> str:
+        clean = CurrencyProcessor._clean_amount(amount)
+        parts = CurrencyProcessor._split_major_and_minor(clean)
+
+        if parts is None:
+            return f"{words(float(clean))} {currency.fa_unit}"
+
+        major, minor = parts
+        return CurrencyProcessor._compose(
+            major=major,
+            minor=minor,
+            spell=words,
+            major_unit=currency.fa_unit,
+            minor_unit=currency.fa_subunit,
+            joiner=" و ",
+        )
+
+    @staticmethod
+    def _amount_to_english(amount: str, currency: Currency) -> str:
+        clean = CurrencyProcessor._clean_amount(amount)
+        parts = CurrencyProcessor._split_major_and_minor(clean)
+
+        if parts is None:
+            return f"{num2words(Decimal(clean))} {currency.en_units}"
+
+        major, minor = parts
+        return CurrencyProcessor._compose(
+            major=major,
+            minor=minor,
+            spell=num2words,
+            major_unit=currency.en_unit if major == 1 else currency.en_units,
+            minor_unit=currency.en_subunit if minor == 1 else currency.en_subunits,
+            joiner=" and ",
+        )
+
+    @staticmethod
+    def _compose(major, minor, spell, major_unit, minor_unit, joiner) -> str:
+        pieces = []
+        # Skip "zero dollars" when there are cents (`0.50$` -> "fifty cents").
+        if major or not minor:
+            pieces.append(f"{spell(major)} {major_unit}")
+        if minor:
+            pieces.append(f"{spell(minor)} {minor_unit}")
+        return joiner.join(pieces)
+
+    @staticmethod
+    def _split_major_and_minor(clean: str):
+        """Returns (major, minor), or None if there are more than 2 decimals."""
+        if "." not in clean:
+            return int(clean), 0
+        whole, fraction = clean.split(".")
+        if len(fraction) > 2:
+            return None
+        return int(whole), int(fraction.ljust(2, "0"))
+
+    @staticmethod
+    def _clean_amount(amount: str) -> str:
+        # Thousands separators are dropped; ٫ is the Persian decimal mark.
+        return amount.replace(",", "").replace("٬", "").replace("٫", ".")

rules/dates.py

@@ -0,0 +1,154 @@
+import re
+from datetime import datetime
+from typing import Union
+
+from dateutil import parser as gregorian_parser
+from num2fawords import words
+from num2words import num2words
+from persiantools import digits
+from persiantools.jdatetime import JalaliDate
+
+from rules.base import BaseProcessor
+import random
+
+
+Date = Union[datetime, JalaliDate]
+
+GREGORIAN_MONTHS = (
+    "January",
+    "February",
+    "March",
+    "April",
+    "May",
+    "June",
+    "July",
+    "August",
+    "September",
+    "October",
+    "November",
+    "December",
+)
+
+DATE_PATTERN = re.compile(
+    r"""
+    (?:
+        (?P<year>\d{4})[^\d]+(?P<month>\d{1,2})[^\d]+(?P<day>\d{1,2})
+        |
+        (?P<day_first>\d{1,2})[^\d]+(?P<month_first>\d{1,2})[^\d]+(?P<year_first>\d{4})
+    )
+    """,
+    re.VERBOSE,
+)
+
+
+class DateProcessor(BaseProcessor):
+    def __init__(self, epsilon:float=1.0):
+        super(DateProcessor, self).__init__(epsilon=epsilon)
+
+    def process(self, date_string: str, language: str, *args, **kwargs) -> str:
+        if random.random() > self.epsilon:
+            return date_string
+        if not self.is_date(date_string):
+            return date_string
+
+        try:
+            date = self._parse_date(date_string)
+        except ValueError:
+            return date_string
+
+        if date is None:
+            raise date_string
+
+        return self._date_to_text(date, language)
+
+    @staticmethod
+    def is_date(date_string: str) -> bool:
+        date_string = digits.fa_to_en(date_string)
+        return DATE_PATTERN.fullmatch(date_string) is not None
+
+    def _parse_date(self, date_string: str) -> Date | None:
+        clean_string = digits.fa_to_en(date_string)
+
+        match = DATE_PATTERN.search(clean_string)
+
+        if match:
+            year, month, day = self._extract_date_parts(match)
+
+            # Years less than 1600 is considered as Jalali
+            if year < 1600:
+                try:
+                    return JalaliDate(year, month, day)
+                except ValueError:
+                    pass
+
+        # Try Gregorian parsing.
+        try:
+            date = gregorian_parser.parse(clean_string)
+
+            if date.year >= 1600:
+                return date
+
+        except (ValueError, OverflowError):
+            pass
+
+        # If regex matched, give Jalali parsing one final attempt so that
+        # invalid Jalali dates produce a useful error.
+        if match:
+            year, month, day = self._extract_date_parts(match)
+
+            try:
+                return JalaliDate(year, month, day)
+            except ValueError as exc:
+                raise ValueError(
+                    f"Invalid Jalali date: {date_string!r}"
+                ) from exc
+
+        return None
+
+    @staticmethod
+    def _extract_date_parts(
+        match: re.Match[str],
+    ) -> tuple[int, int, int]:
+        if match.group("year"):
+            return (
+                int(match.group("year")),
+                int(match.group("month")),
+                int(match.group("day")),
+            )
+
+        return (
+            int(match.group("year_first")),
+            int(match.group("month_first")),
+            int(match.group("day_first")),
+        )
+
+    def _date_to_text(self, date: Date, language: str) -> str:
+        if language == "fa":
+            return self._date_to_persian(date)
+
+        if language == "en":
+            return self._date_to_english(date)
+
+        raise ValueError(f"Unsupported language: {language!r}")
+
+    @staticmethod
+    def _date_to_persian(date: Date) -> str:
+        return f"{words(date.day)} {words(date.month)} {words(date.year)}"
+
+    @staticmethod
+    def _date_to_english(date: Date) -> str:
+        year = date.year
+
+        if "00" in str(year) or len(str(year)) < 4:
+            year_text = num2words(year)
+        else:
+            year_text = (
+                f"{num2words(year // 100)} "
+                f"{num2words(year % 100)}"
+            )
+
+        return (
+            f"{num2words(date.day, ordinal=True)} "
+            f"of {GREGORIAN_MONTHS[date.month - 1]}, "
+            f"{year_text}"
+        )

rules/fraction.py

@@ -0,0 +1,164 @@
+import random
+import re
+
+from num2fawords import ordinal_words, words
+from num2words import num2words
+from persiantools import digits
+
+from rules.base import BaseProcessor
+from utils import NO_LETTERS, detect_digit_language
+
+
+VULGAR_FRACTIONS = {
+    "½": (1, 2),
+    "⅓": (1, 3),
+    "⅔": (2, 3),
+    "¼": (1, 4),
+    "¾": (3, 4),
+    "⅕": (1, 5),
+    "⅖": (2, 5),
+    "⅗": (3, 5),
+    "⅘": (4, 5),
+    "⅙": (1, 6),
+    "⅚": (5, 6),
+    "⅛": (1, 8),
+    "⅜": (3, 8),
+    "⅝": (5, 8),
+    "⅞": (7, 8),
+}
+
+VULGAR = "".join(VULGAR_FRACTIONS)
+
+# A whole number, optionally with thousands separators: `12`, `1,000`, `۱٬۰۰۰`.
+NUMBER = r"(?:\d{1,3}(?:[,٬]\d{3})+|\d+)"
+
+# The fraction itself: `1/3`, `۱/۳`, `1,000/2`, `2 1/2` (mixed number), `½`, `1½`.
+# Shared with `UnitProcessor`, which reads `1/2 kg`.
+FRACTION_BODY = rf"""
+    (?:
+        (?:(?P<whole>{NUMBER})[ ])?(?P<numerator>{NUMBER})/(?P<denominator>{NUMBER})
+        |
+        (?:(?P<vulgar_whole>{NUMBER})[ ]?)?(?P<vulgar>[{VULGAR}])
+    )
+"""
+
+# Does not match parts of dates (`1402/05/10`), decimals (`3.1/2`), or fractions
+# glued to a word or currency/percent sign (`and/or`, `1/2$`).
+FRACTION_PATTERN = re.compile(
+    rf"""
+    (?<![\w/])(?<!\d[.,٫٬])
+    {FRACTION_BODY}
+    (?![\w/$€£%°])(?![.,٫٬]\d)
+    """,
+    re.VERBOSE,
+)
+
+
+class FractionProcessor(BaseProcessor):
+    """Converts fractions into spoken text.
+
+    Persian: `۱/۳`   -> `یک سوم`
+             `۳/۴`   -> `سه چهارم`
+             `۲ ۱/۲` -> `دو و یک دوم`
+    English: `1/3`   -> `one third`
+             `3/4`   -> `three quarters`
+             `2 1/2` -> `two and one half`
+             `½`     -> `one half`
+             `1,000/2` -> `one thousand halves`
+
+    Works on the whole text (not word by word) because a mixed number such as
+    `2 1/2` spans two words. If the text has no letters at all, the digits pick
+    the language: Persian digits -> Persian, ASCII digits -> English.
+
+    A fraction followed by a unit (`1/2 kg`) is handled by `UnitProcessor`.
+    """
+
+    def __init__(self, epsilon: float = 1.0):
+        super(FractionProcessor, self).__init__(epsilon=epsilon)
+
+    def process(self, text: str, language: str = None, *args, **kwargs) -> str:
+        if random.random() > self.epsilon:
+            return text
+
+        if language in (None, NO_LETTERS):
+            language = detect_digit_language(text)
+        if language not in ("fa", "en"):
+            return text
+
+        return FRACTION_PATTERN.sub(lambda match: self._convert(match, language), text)
+
+    @staticmethod
+    def _convert(match: re.Match, language: str) -> str:
+        parsed = FractionProcessor.parse(match)
+        if parsed is None:
+            return match.group(0)
+        return FractionProcessor.to_text(*parsed, language=language)
+
+    @staticmethod
+    def parse(match: re.Match):
+        """Returns (whole, numerator, denominator) from a match of `FRACTION_BODY`.
+
+        `whole` is None when there is no mixed-number part. Returns None for
+        `1/0` (not a fraction) and `3/1` (just a ratio), which are left alone.
+        """
+        if match.group("vulgar"):
+            numerator, denominator = VULGAR_FRACTIONS[match.group("vulgar")]
+            whole = match.group("vulgar_whole")
+        else:
+            numerator = FractionProcessor._to_int(match.group("numerator"))
+            denominator = FractionProcessor._to_int(match.group("denominator"))
+            whole = match.group("whole")
+
+        if denominator in (0, 1):
+            return None
+        return (
+            None if whole is None else FractionProcessor._to_int(whole),
+            numerator,
+            denominator,
+        )
+
+    @staticmethod
+    def to_text(whole, numerator: int, denominator: int, language: str) -> str:
+        if language == "fa":
+            fraction = FractionProcessor._persian_fraction(numerator, denominator)
+            joiner, spell = " و ", words
+        else:
+            fraction = FractionProcessor._english_fraction(numerator, denominator)
+            joiner, spell = " and ", num2words
+
+        if whole is None:
+            return fraction
+        return f"{spell(whole)}{joiner}{fraction}"
+
+    @staticmethod
+    def _to_int(number: str) -> int:
+        clean = digits.fa_to_en(number).replace(",", "").replace("٬", "")
+        return int(clean)
+
+    @staticmethod
+    def _persian_fraction(numerator: int, denominator: int) -> str:
+        ordinal = ordinal_words(denominator)
+        # The library writes 30 as `سیم`; the correct form is `سی‌ام`.
+        if ordinal.endswith("سیم"):
+            ordinal = ordinal[: -len("سیم")] + "سی‌ام"
+        # `یک صدم` / `یک هزارم`, not `یک یکصدم` / `یک یک هزارم`.
+        if ordinal == "یکصدم":
+            ordinal = "صدم"
+        elif ordinal.startswith("یک "):
+            ordinal = ordinal[len("یک "):]
+        return f"{words(numerator)} {ordinal}"
+
+    @staticmethod
+    def _english_fraction(numerator: int, denominator: int) -> str:
+        singular = numerator == 1
+        if denominator == 2:
+            unit = "half" if singular else "halves"
+        elif denominator == 4:
+            unit = "quarter" if singular else "quarters"
+        else:
+            ordinal = num2words(denominator, to="ordinal")
+            # `one hundredth`, not `one one hundredth`.
+            if ordinal.startswith("one "):
+                ordinal = ordinal[len("one "):]
+            unit = ordinal if singular else f"{ordinal}s"
+        return f"{num2words(numerator)} {unit}"

rules/numbers.py

@@ -0,0 +1,161 @@
+import random
+import re
+
+from num2fawords import words
+from num2words import num2words
+
+from rules.base import BaseProcessor
+from utils import NO_LETTERS, detect_digit_language
+
+# ASCII, Persian (۰-۹) and Arabic-Indic (٠-٩) digits.
+DIGIT = "0-9۰-۹٠-٩"
+TO_ASCII = str.maketrans("۰۱۲۳۴۵۶۷۸۹٠١٢٣٤٥٦٧٨٩", "01234567890123456789")
+
+THOUSANDS_SEPARATORS = ",٬"
+DECIMAL_SEPARATORS = ".٫"
+
+# `1,000,000`, `12`, `۱٬۰۰۰`, `3.5`, `۳٫۵`, `1,000.25`.
+NUMBER = (
+    rf"(?:[{DIGIT}]{{1,3}}(?:[{THOUSANDS_SEPARATORS}][{DIGIT}]{{3}})+|[{DIGIT}]+)"
+    rf"(?:[{DECIMAL_SEPARATORS}][{DIGIT}]+)?"
+)
+
+# A number must stand alone. It is left alone when it is glued to a letter, a
+# digit or an underscore on either side (`3D`, `mp3`, `H2O`, `A4`, `5G`,
+# `covid19`), or sits inside a time, date, version, range, URL, e-mail,
+# hashtag or `1,5` style number (`10:30`, `2024/01/05`, `1.2.3`, `10-12`,
+# `#2024`, `a@3`).
+NUMBER_PATTERN = re.compile(
+    rf"(?<![\w‌.,٫٬:/\\\-–+#@$€£])"
+    rf"(?P<number>{NUMBER})"
+    rf"(?![\w‌@]|[.,٫٬:/\\][{DIGIT}]|[-–][{DIGIT}]|[-–][A-Za-z]\b)"
+)
+
+MAX_INTEGER_DIGITS = 15
+MAX_FLOAT_FRACTION_DIGITS = 9
+
+ENGLISH_YEAR_WORDS = r"(?:in|since|year|until|till|circa|c\.|AD|BC)"
+ENGLISH_YEAR_PATTERN = re.compile(rf"(?:^|\s|[(\[\"'“‘]){ENGLISH_YEAR_WORDS}\s+$", re.IGNORECASE)
+PERSIAN_YEAR_PATTERN = re.compile(r"(?:^|\s|[(\[\"'«])سال\s+$")
+
+
+class NumberProcessor(BaseProcessor):
+    """Reads any remaining standalone number as words, in Persian or English.
+
+    English: `I have 20 apples` -> `I have twenty apples`
+             `1,000` -> `one thousand`, `3.5` -> `three point five`
+    Persian: `من ۲۰ سیب دارم` -> `من بیست سیب دارم`
+             `۱٬۰۰۰` -> `یک هزار`, `۳٫۵` -> `سه و پنج دهم`
+
+    It is the last step of the pipeline, so every more specific processor
+    (units, currency, dates, times, phone numbers, ranges, fractions, ...) gets
+    the number first and this only reads what is left.
+
+    Handled:
+      * integers in ASCII, Persian or Arabic-Indic digits, with thousands
+        separators (`,` or `٬`);
+      * decimals (`.` or `٫`). English reads every digit after the point, so
+        `3.50` -> `three point five zero` and `0.05` -> `zero point zero five`.
+        Persian reads the decimal as a fraction (`۰٫۰۵` -> `پنج صدم`), and a
+        decimal with more than nine digits after the point as `ممیز` followed by
+        the digits;
+      * negative numbers are read by `NegativeSignProcessor` before this runs.
+
+    Language rules:
+      * English years are read the way English speakers say them, but only after
+        a word such as `in`, `since`, `year`, `until`, `till`, `circa`, `AD`,
+        `BC`: `in 1999` -> `in nineteen ninety-nine`, `since 1905` ->
+        `since nineteen oh five`, `in 2024` -> `in twenty twenty-four`,
+        `in 2005` -> `in two thousand and five`.
+      * Persian years after `سال` drop the leading `یک`: `سال ۱۴۰۳` ->
+        `سال هزار و چهارصد و سه`.
+      * An integer with a leading zero (`007`, `۰۰۷`) is read digit by digit,
+        as an identifier: `zero zero seven` / `صفر صفر هفت`.
+
+    Left alone: numbers glued to letters or digits (`3D`, `mp3`, `H2O`, `A4`,
+    `5G`), times, dates, versions (`1.2.3`), `1,5` style numbers, numbers in
+    URLs, e-mails and hashtags, `+5`, integers longer than 15 digits, and text
+    in "mixed" or any other language.
+
+    With no letters in the text, the digits pick the language: Persian digits ->
+    Persian, ASCII digits -> English.
+    """
+
+    def __init__(self, epsilon: float = 1.0):
+        super(NumberProcessor, self).__init__(epsilon=epsilon)
+
+    def process(self, text: str, language: str = None, *args, **kwargs) -> str:
+        if random.random() > self.epsilon:
+            return text
+        if re.search(rf"[{DIGIT}]", text) is None:
+            return text
+
+        if language in (None, NO_LETTERS):
+            language = detect_digit_language(text)
+        if language not in ("fa", "en"):
+            return text
+
+        return NUMBER_PATTERN.sub(lambda match: self._convert(match, language), text)
+
+    def _convert(self, match: re.Match, language: str) -> str:
+        raw = match.group("number")
+        clean = raw.translate(TO_ASCII)
+        has_thousands_separator = any(separator in clean for separator in THOUSANDS_SEPARATORS)
+        for separator in THOUSANDS_SEPARATORS:
+            clean = clean.replace(separator, "")
+        clean = clean.replace("٫", ".")
+
+        whole, _, fraction = clean.partition(".")
+        has_fraction = "." in clean
+
+        if has_fraction:
+            if len(whole) > 1 and whole.startswith("0"):
+                return raw
+            return self._decimal_to_words(whole, fraction, language)
+
+        if len(whole) > MAX_INTEGER_DIGITS:
+            return raw
+        if len(whole) > 1 and whole.startswith("0") and not has_thousands_separator:
+            return self._digits_to_words(whole, language)
+
+        before = match.string[:match.start()]
+        return self._integer_to_words(int(whole), language, before)
+
+    @staticmethod
+    def _integer_to_words(value: int, language: str, before: str = "") -> str:
+        if language == "fa":
+            text = words(value)
+            if 1000 <= value <= 1999 and PERSIAN_YEAR_PATTERN.search(before):
+                text = text.removeprefix("یک ")
+            return text
+
+        if 1100 <= value <= 2099 and not 2000 <= value <= 2009 \
+                and ENGLISH_YEAR_PATTERN.search(before):
+            return NumberProcessor._english_year(value)
+        return num2words(value)
+
+    @staticmethod
+    def _english_year(value: int) -> str:
+        """1999 -> `nineteen ninety-nine`, 1905 -> `nineteen oh five`, 1900 -> `nineteen hundred`."""
+        high, low = divmod(value, 100)
+        if low == 0:
+            return f"{num2words(high)} hundred"
+        if low < 10:
+            return f"{num2words(high)} oh {num2words(low)}"
+        return f"{num2words(high)} {num2words(low)}"
+
+    @staticmethod
+    def _decimal_to_words(whole: str, fraction: str, language: str) -> str:
+        if language == "fa":
+            if len(fraction) > MAX_FLOAT_FRACTION_DIGITS or len(whole) > MAX_INTEGER_DIGITS:
+                spoken_fraction = " ".join(words(int(digit)) for digit in fraction)
+                return f"{words(int(whole))} ممیز {spoken_fraction}"
+            return words(float(f"{whole}.{fraction}"))
+
+        spoken_fraction = " ".join(num2words(int(digit)) for digit in fraction)
+        return f"{num2words(int(whole))} point {spoken_fraction}"
+
+    @staticmethod
+    def _digits_to_words(digits: str, language: str) -> str:
+        spell = words if language == "fa" else num2words
+        return " ".join(spell(int(digit)) for digit in digits)

rules/phone.py

@@ -0,0 +1,312 @@
+import random
+import re
+
+from num2fawords import words
+
+from rules.base import BaseProcessor
+from utils import NO_LETTERS, detect_digit_language
+
+
+DIGIT_WORDS = {
+    "en": ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine"],
+    "fa": ["صفر", "یک", "دو", "سه", "چهار", "پنج", "شش", "هفت", "هشت", "نه"],
+}
+
+PLUS_WORDS = {"en": "plus", "fa": "مثبت"}
+
+# ASCII, Persian (۰-۹) and Arabic-Indic (٠-٩) digits.
+DIGIT = "0-9۰-۹٠-٩"
+TO_ASCII = str.maketrans("۰۱۲۳۴۵۶۷۸۹٠١٢٣٤٥٦٧٨٩", "01234567890123456789")
+
+# One group of digits, optionally wrapped: `912`, `(021)`, `(+98)`, `+98`.
+FIRST_GROUP = rf"(?:\([ \t]*)?(?:\+[ \t]*)?[{DIGIT}]+(?:[ \t]*\))?"
+NEXT_GROUP = rf"(?:\([ \t]*)?[{DIGIT}]+(?:[ \t]*\))?"
+# Between groups: spaces, a hyphen or dot, or nothing at all right after a
+# closing parenthesis (`(0915)6486320`).
+SEPARATOR = r"(?:[ \t]*[-.–][ \t]*|[ \t]+|(?<=\)))"
+
+# A run of digit groups: `+98-912-4338135`, `021 88069895`, `(021) 88069895`.
+# It must not be glued to a word, a decimal, a URL path or a currency sign.
+RUN_PATTERN = re.compile(
+    rf"(?<![\w.,/\-+$€£]){FIRST_GROUP}(?:{SEPARATOR}{NEXT_GROUP})*"
+)
+
+GROUP_PATTERN = re.compile(
+    rf"(?P<open>\()?[ \t]*(?P<plus>\+)?[ \t]*(?P<digits>[{DIGIT}]+)(?P<close>[ \t]*\))?"
+)
+
+# Words after which a looser number (7-15 digits, any start) is a phone number:
+# `tel: 88069895`, `تلفن ۸۸۰۶۹۸۹۵`.
+TRIGGER_PATTERN = re.compile(
+    r"(?<![^\W\d_])"
+    r"(?:phone|tel|telephone|mobile|cell|cellphone|fax|call|whatsapp|contact|"
+    r"تلفن|موبایل|همراه|شماره|فکس|تماس|واتس[‌ ]?اپ|تلگرام)"
+    r"(?:[ \t]*(?:number|no\.?|num|#|:|،|-|–))*[ \t]*$",
+    re.IGNORECASE,
+)
+
+
+class PhoneNumberProcessor(BaseProcessor):
+    """Reads phone numbers digit by digit.
+
+    English: `09124338135`      -> `zero nine one two, four three three, eight one, three five`
+             `+98-912-4338135`  -> `plus nine eight, nine one two, four three three, eight one, three five`
+             `021-88069895`     -> `zero two one, eight eight, zero six, nine eight, nine five`
+    Persian: `۰۹۱۲۴۳۳۸۱۳۵`        -> `صفر نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج`
+             `02188037395`      -> `صفر بیست و یک، هشتاد و هشت، صفر سه، هفتاد و سه، نود و پنج`
+             `+989124338135`    -> `مثبت نود و هشت، نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج`
+
+    An Iranian number is said in groups: mobile `0|912|433|81|35`, landline
+    `0|21|88|03|73|95`, and an 8-digit local number after a phone word in pairs.
+    Persian says each group as a number (a leading zero inside a group is said:
+    `03` -> `صفر سه`, and several are counted: `001` -> `دو صفر یک`), English reads each group digit by digit with a pause (comma)
+    between groups. Other numbers (`+44 ...`, US style) are read digit by digit:
+    English with a comma between the groups as written (`555-123-4567` ->
+    `five five five, one two three, four five six seven`), Persian without commas.
+
+    Recognised (digits may be ASCII, Persian or Arabic-Indic, and may be
+    separated by spaces, hyphens or dots, with the area or country code in
+    parentheses):
+      - international, with `+`: 8-15 digits (`+98 912 433 8135`, `+1 (555) 123-4567`);
+      - national, starting with `0`: 10-11 digits (`0912 433 8135`, `021-88069895`,
+        `(021) 88069895`); starting with `00`: up to 17 digits (`0098 912 4338135`);
+      - US style `555-123-4567` / `(555) 123-4567` / `1-555-123-4567`;
+      - any 7-15 digit number right after a word like `tel`, `phone`, `call`,
+        `تلفن`, `موبایل`, `شماره` (`tel: 88069895`).
+
+    Left alone: numbers without a leading `0` or `+` (`4338135`, `2024-05-10`),
+    decimals (`0.123456789`), numbers glued to words, URL paths, longer
+    numbers (card numbers) and anything that is not 10-11 digits after a `0`.
+
+    With no letters in the text, the digits pick the language: Persian digits ->
+    Persian, ASCII digits -> English. Parentheses around a group are kept:
+    `(021) 88069895` -> `(zero two one) eight eight ...`.
+    """
+
+    def __init__(self, epsilon: float = 1.0):
+        super(PhoneNumberProcessor, self).__init__(epsilon=epsilon)
+
+    def process(self, text: str, language: str = None, *args, **kwargs) -> str:
+        if random.random() > self.epsilon:
+            return text
+        if re.search(rf"[{DIGIT}]", text) is None:
+            return text
+
+        if language in (None, NO_LETTERS):
+            language = detect_digit_language(text)
+        if language not in ("fa", "en"):
+            return text
+
+        return RUN_PATTERN.sub(lambda match: self._convert_run(match, language), text)
+
+    def _convert_run(self, match: re.Match, language: str) -> str:
+        text = match.string
+        groups = list(GROUP_PATTERN.finditer(match.group(0), 0))
+        # Absolute positions of every group in the whole text.
+        base = match.start()
+        spans = [self._group_span(group, base) for group in groups]
+
+        pieces = []
+        cursor = match.start()
+        index = 0
+        while index < len(groups):
+            found = self._find_phone(text, groups, spans, index, match.end())
+            if found is None:
+                break
+            start, end = found
+            pieces.append(text[cursor:spans[start][0]])
+            pieces.append(self._render(groups[start:end], language))
+            cursor = spans[end - 1][1]
+            index = end
+
+        if not pieces:
+            return match.group(0)
+        pieces.append(text[cursor:match.end()])
+        return "".join(pieces)
+
+    @staticmethod
+    def _group_span(group: re.Match, base: int):
+        start = min(
+            group.start(name) for name in ("open", "plus", "digits") if group.group(name)
+        )
+        end = group.end("close") if group.group("close") else group.end("digits")
+        return base + start, base + end
+
+    def _find_phone(self, text, groups, spans, index, run_end):
+        """The first (start, end) group range from `index` on that is a phone number."""
+        for start in range(index, len(groups)):
+            # A number that begins in the middle of a run must follow a plain
+            # space, not a hyphen or a dot (`3.0912433813` is a decimal).
+            if start > index or start > 0:
+                gap = text[spans[start - 1][1]:spans[start][0]]
+                if gap.strip(" \t"):
+                    continue
+            for end in range(len(groups), start, -1):
+                if not self._is_phone(text, groups, spans, start, end):
+                    continue
+                if end < len(groups):
+                    gap = text[spans[end - 1][1]:spans[end][0]]
+                    # `021-88069895-12`, `0912433813.5`: not a clean boundary.
+                    if gap.strip(" \t"):
+                        continue
+                return start, end
+        return None
+
+    def _is_phone(self, text, groups, spans, start, end) -> bool:
+        chosen = groups[start:end]
+        digit_groups = [group.group("digits").translate(TO_ASCII) for group in chosen]
+        total = sum(len(digits) for digits in digit_groups)
+        first = digit_groups[0]
+        has_plus = chosen[0].group("plus") is not None
+
+        after = text[spans[end - 1][1]:spans[end - 1][1] + 1]
+        if after and re.match(r"\w", after):
+            return False
+
+        if has_plus:
+            return 8 <= total <= 15
+        if self._is_us_format(chosen, digit_groups, text, spans, start):
+            return True
+        if TRIGGER_PATTERN.search(text[:spans[start][0]]):
+            return 7 <= total <= 15
+        if len(first) < 2 or not first.startswith("0"):
+            return False
+        # `0912433813.5` is a decimal: every later group needs at least 2 digits.
+        if any(len(digits) < 2 for digits in digit_groups[1:]):
+            return False
+        if first.startswith("00"):
+            return 10 <= total <= 17
+        return 10 <= total <= 11
+
+    @staticmethod
+    def _is_us_format(chosen, digit_groups, text, spans, start) -> bool:
+        """`555-123-4567`, `(555) 123-4567`, `1-555-123-4567` (not plain spaces)."""
+        sizes = tuple(len(digits) for digits in digit_groups)
+        if sizes not in ((3, 3, 4), (1, 3, 3, 4)):
+            return False
+        if sizes[0] == 1 and digit_groups[0] != "1":
+            return False
+        begin, finish = spans[start][0], spans[start + len(chosen) - 1][1]
+        return re.search(r"[-.–(]", text[begin:finish]) is not None
+
+    def _render(self, chosen, language: str) -> str:
+        grouped = self._render_groups(chosen, language)
+        if grouped is not None:
+            return grouped
+
+        # Not an Iranian number: read each written group digit by digit. English
+        # puts a pause (comma) between the written groups, Persian does not.
+        words_for_digits = DIGIT_WORDS[language]
+        rendered = []
+        for group in chosen:
+            spoken = " ".join(
+                words_for_digits[int(digit)] for digit in group.group("digits").translate(TO_ASCII)
+            )
+            if group.group("plus"):
+                spoken = f"{PLUS_WORDS[language]} {spoken}"
+            rendered.append(
+                f"{'(' if group.group('open') else ''}{spoken}{')' if group.group('close') else ''}"
+            )
+        return (", " if language == "en" else " ").join(rendered)
+
+    def _render_groups(self, chosen, language: str):
+        """Reads an Iranian number in its spoken groups, or None.
+
+        Persian: `09124338135` -> `صفر نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج`
+        English: `09124338135` -> `zero nine one two, four three three, eight one, three five`
+        """
+        digits = "".join(group.group("digits").translate(TO_ASCII) for group in chosen)
+        has_plus = chosen[0].group("plus") is not None
+        units = self._iranian_units(digits, has_plus)
+        if units is None:
+            return None
+
+        # Parentheses are kept only where they line up with a spoken unit.
+        opens, closes, offset = set(), set(), 0
+        for group in chosen:
+            if group.group("open"):
+                opens.add(offset)
+            offset += len(group.group("digits"))
+            if group.group("close"):
+                closes.add(offset)
+
+        pieces = []
+        for index, (start, end) in enumerate(units):
+            spoken = self._spoken_number(digits[start:end], language)
+            if index == 0 and has_plus:
+                spoken = f"{PLUS_WORDS[language]} {spoken}"
+            if start in opens:
+                spoken = f"({spoken}"
+            if end in closes:
+                spoken = f"{spoken})"
+            pieces.append(spoken)
+        return ("، " if language == "fa" else ", ").join(pieces)
+
+    @classmethod
+    def _spoken_number(cls, digits: str, language: str) -> str:
+        if language == "fa":
+            return cls._persian_number(digits)
+        return " ".join(DIGIT_WORDS["en"][int(digit)] for digit in digits)
+
+    @staticmethod
+    def _persian_number(digits: str) -> str:
+        """`912` -> `نهصد و دوازده`; leading zeros are said: `03` -> `صفر سه`.
+
+        Two or more leading zeros are counted: `001` -> `دو صفر یک`, `00` -> `دو صفر`.
+        """
+        stripped = digits.lstrip("0")
+        zeros = len(digits) - len(stripped)
+        spoken = []
+        if zeros == 1:
+            spoken.append(DIGIT_WORDS["fa"][0])
+        elif zeros > 1:
+            spoken.append(f"{words(zeros)} {DIGIT_WORDS['fa'][0]}")
+        if stripped:
+            spoken.append(words(int(stripped)))
+        return " ".join(spoken)
+
+    @staticmethod
+    def _iranian_units(digits: str, has_plus: bool):
+        """Splits an Iranian number into spoken units as (start, end) digit offsets
+        (the same units in Persian and English).
+
+        Mobile `0|912|433|81|35`, landline `0|21|88|03|73|95`; the `0` or the
+        country code (`98`, `0098`) is said together with / before the first
+        group. Returns None for anything that is not an Iranian number, which
+        is then read digit by digit in its written groups.
+        """
+        country = False
+        if has_plus:
+            if len(digits) != 12 or not digits.startswith("98"):
+                return None
+            base, country = 2, True
+        elif digits.startswith("0098"):
+            if len(digits) != 14:
+                return None
+            base, country = 4, True
+        elif len(digits) == 11 and digits[0] == "0":
+            base = 1
+        elif len(digits) == 8:
+            # A local number after a phone word: `۸۸۰۶۹۸۹۵` -> 88|06|98|95.
+            return [(index, index + 2) for index in range(0, 8, 2)]
+        else:
+            return None
+
+        national = digits[base:]
+        if national[0] == "9":
+            sizes = (3, 3, 2, 2)
+        elif national[0] in "012345678":
+            sizes = (2, 2, 2, 2, 2)
+        else:
+            return None
+
+        units = []
+        position = base
+        for size in sizes:
+            units.append((position, position + size))
+            position += size
+        if country:
+            units.insert(0, (0, base))
+        else:
+            units[0] = (0, units[0][1])
+        return units

rules/postal_national_id.py

@@ -0,0 +1,99 @@
+import random
+import re
+
+from rules.base import BaseProcessor
+from rules.cards import CardNumberProcessor, DIGIT, TO_ASCII
+from utils import NO_LETTERS, detect_digit_language
+
+# Ten digits in a row, split 5 + 5 with a space or hyphen (`15147-17118`), or
+# split 3 + 6 + 1 with hyphens (`001-234567-9`, a common national ID layout).
+# It must not be glued to a word, a longer number, a decimal or a URL path.
+_D = rf"[{DIGIT}]"
+IDENTIFIER_PATTERN = re.compile(
+    rf"(?<![\w.,/+\-])(?<![{DIGIT}][ \t])(?<![{DIGIT}][-–])"
+    rf"(?:{_D}{{5}}[ \t]?[-–]?[ \t]?{_D}{{5}}|{_D}{{3}}-{_D}{{6}}-{_D})"
+    rf"(?![\w/]|[-–.,][{DIGIT}]|[ \t][{DIGIT}])"
+)
+
+PAIR_JOINER = {"fa": "، ", "en": ", "}
+
+
+class PostalCodeAndNationalIdProcessor(BaseProcessor):
+    """Reads 10-digit Iranian postal codes and national ID numbers in pairs of digits.
+
+    Persian: `۱۵۱۴۷۱۷۱۱۸` -> `پانزده، چهارده، هفتاد و یک، هفتاد و یک، هجده`
+             `۰۰۱۲۳۴۵۶۷۹` -> `دو صفر، دوازده، سی و چهار، پنجاه و شش، هفتاد و نه`
+    English: `1514717118` -> `fifteen, fourteen, seventy-one, seventy-one, eighteen`
+             `0012345679` -> `double zero, twelve, thirty-four, fifty-six, seventy-nine`
+
+    Both kinds of number are ten digits and are read the same way: five pairs
+    separated by a pause (comma). A pair that starts with `0` keeps its zero,
+    exactly as in `CardNumberProcessor`: Persian `05` -> `صفر پنج`, `00` ->
+    `دو صفر`; English `05` -> `oh five`, `00` -> `double zero`. This matters for
+    national IDs, which often start with one or two zeros.
+
+    A ten-digit number is accepted when it is either
+      * a postal code: the first digit is not 0 and the first five digits
+        contain no 0 or 2, or
+      * a national ID: the check digit is correct (the sum of the first nine
+        digits times 10, 9, ..., 2 gives a remainder r; the last digit must be r
+        when r < 2, otherwise 11 - r) and the digits are not all the same.
+    Written in a row (`1514717118`), split 5 + 5 with a hyphen or space
+    (`15147-17118`), or, for national IDs, split 3 + 6 + 1 with hyphens
+    (`001-234567-9`). The 3 + 6 + 1 split is accepted only when the check digit
+    is correct. ASCII, Persian and Arabic-Indic digits are all accepted.
+
+    Left alone: other lengths (such as 5-digit ZIP codes), ten-digit numbers
+    that pass neither test (for example a phone number without its leading zero,
+    `9124338135`), numbers glued to words, and decimals.
+
+    With no letters in the text, the digits pick the language: Persian digits ->
+    Persian, ASCII digits -> English. It runs after `CardNumberProcessor` and
+    before `PhoneNumberProcessor`.
+    """
+
+    def __init__(self, epsilon: float = 1.0):
+        super(PostalCodeAndNationalIdProcessor, self).__init__(epsilon=epsilon)
+
+    def process(self, text: str, language: str = None, *args, **kwargs) -> str:
+        if random.random() > self.epsilon:
+            return text
+        if re.search(rf"[{DIGIT}]", text) is None:
+            return text
+
+        if language in (None, NO_LETTERS):
+            language = detect_digit_language(text)
+        if language not in ("fa", "en"):
+            return text
+
+        return IDENTIFIER_PATTERN.sub(lambda match: self._convert(match, language), text)
+
+    def _convert(self, match: re.Match, language: str) -> str:
+        raw = match.group(0)
+        digits = re.sub(rf"[^{DIGIT}]", "", raw).translate(TO_ASCII)
+        if len(digits) != 10:
+            return raw
+
+        is_national_id_layout = raw.count("-") == 2
+        if is_national_id_layout:
+            if not self._is_national_id(digits):
+                return raw
+        elif not (self._is_postal_code(digits) or self._is_national_id(digits)):
+            return raw
+
+        pairs = [digits[index:index + 2] for index in range(0, 10, 2)]
+        return PAIR_JOINER[language].join(
+            CardNumberProcessor._pair_to_words(pair, language) for pair in pairs
+        )
+
+    @staticmethod
+    def _is_postal_code(digits: str) -> bool:
+        return digits[0] != "0" and not set(digits[:5]) & {"0", "2"}
+
+    @staticmethod
+    def _is_national_id(digits: str) -> bool:
+        if len(set(digits)) == 1:
+            return False
+        remainder = sum(int(digits[i]) * (10 - i) for i in range(9)) % 11
+        check = remainder if remainder < 2 else 11 - remainder
+        return int(digits[9]) == check

rules/spaces.py

@@ -0,0 +1,38 @@
+import random
+import re
+
+from rules.base import BaseProcessor
+from rules.currency import SYMBOLS
+
+
+# A number, whitespace, then a standalone currency symbol
+# (e.g. `1 $`, `۳ €`, `2.5   £`). A symbol followed by a digit or letter
+# (e.g. `1 $5`) is not a currency suffix.
+NUMBER_SPACE_SYMBOL_PATTERN = re.compile(
+    rf"""
+    (?<![\w.,٫٬])
+    (?P<amount>\d+(?:[.,٫٬]\d+)*)
+    [^\S\n]+
+    (?P<symbol>[{SYMBOLS}])
+    (?![\w{SYMBOLS}])
+    """,
+    re.VERBOSE,
+)
+
+
+class CurrencySpaceProcessor(BaseProcessor):
+    """Glues a currency symbol to the number before it: `1 $` -> `1$`.
+
+    Supported symbols: `$`, `€`, `£`.
+
+    This works on the whole text (not word by word) because the number and
+    the symbol are separate words. Run it before `CurrencyProcessor`.
+    """
+
+    def __init__(self, epsilon: float = 1.0):
+        super(CurrencySpaceProcessor, self).__init__(epsilon=epsilon)
+
+    def process(self, text: str, language: str = "en", *args, **kwargs) -> str:
+        if random.random() > self.epsilon:
+            return text
+        return NUMBER_SPACE_SYMBOL_PATTERN.sub(r"\g<amount>\g<symbol>", text)

rules/symbols.py

@@ -0,0 +1,630 @@
+import random
+import re
+from decimal import Decimal
+
+from num2fawords import words
+from num2words import num2words
+from persiantools import digits
+
+from rules.base import BaseProcessor
+from rules.fraction import FRACTION_PATTERN
+from rules.units import UNIT_PATTERN, UnitProcessor
+from rules.wrappers import CLOSERS, OPENERS, WRAPPERS
+from utils import NO_LETTERS, detect_digit_language, detect_language
+
+
+# ---------------------------------------------------------------------------
+# Roman numerals
+# ---------------------------------------------------------------------------
+
+ROMAN_VALUES = {"I": 1, "V": 5, "X": 10, "L": 50, "C": 100, "D": 500, "M": 1000}
+
+# Strictly valid numerals only (1-3999): `IV` yes, `IIII`, `VX`, `IC` no.
+ROMAN_NUMERAL = re.compile(
+    r"(?=[IVXLCDM])M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})"
+)
+
+# A standalone run of uppercase Roman letters (lowercase `iv`, `mix` is never read).
+ROMAN_CANDIDATE = re.compile(r"(?<!\w)[IVXLCDM]+(?!\w)")
+
+# Valid numerals that are far more often something else: `Washington DC`,
+# `MD`, `CD player`, `XL` shirt, `MIX`. Add to this set to skip more.
+AMBIGUOUS_ABBREVIATIONS = {
+    "DC", "MD", "CD", "CV", "MC", "CM", "MM", "CC", "MI", "LI", "XL",
+    "MIX", "DIV", "CIV",
+}
+
+# `I`, `V` and `X` on their own are only numerals right after one of these
+# words (`Chapter IV`, `World War I`); `Vitamin C`, `Appendix D` or the
+# pronoun `I` are not. `L`, `C`, `D` and `M` on their own are never read.
+SINGLE_LETTER_NUMERALS = "IVX"
+
+EN_TRIGGERS = {
+    "chapter", "part", "section", "volume", "vol", "book", "act", "scene",
+    "phase", "stage", "appendix", "annex", "article", "page", "table",
+    "figure", "fig", "war", "episode", "season", "series", "type", "class",
+    "level", "grade", "step", "unit", "module", "tier",
+}
+
+FA_TRIGGERS = {
+    "فصل", "بخش", "قسمت", "جلد", "پرده", "مرحله", "فاز", "پیوست", "ضمیمه",
+    "ماده", "جدول", "شکل", "صفحه", "قرن", "جنگ", "جهانی", "سری", "نوع",
+    "دوره", "کتاب", "گام", "سطح", "درجه", "کلاس", "رده", "گروه",
+}
+
+
+def roman_to_int(token: str):
+    """Returns the value of a strictly valid Roman numeral, else None."""
+    if not ROMAN_NUMERAL.fullmatch(token):
+        return None
+    total = 0
+    previous = 0
+    for letter in reversed(token):
+        value = ROMAN_VALUES[letter]
+        total += -value if value < previous else value
+        previous = max(previous, value)
+    return total
+
+
+class RomanNumeralProcessor(BaseProcessor):
+    """Converts Roman numerals into plain numbers in words.
+
+    English: `Henry VIII`   -> `Henry eight`
+             `Chapter IV`   -> `Chapter four`
+             `World War II` -> `World War two`
+    Persian: `قرن XX`        -> `قرن بیست`
+             `فصل IV`        -> `فصل چهار`
+
+    Only uppercase, strictly valid numerals from 1 to 3999 are read. Several
+    letters (`XIV`) are read anywhere; a single `I`, `V` or `X` only after a word
+    like `Chapter`, `Part` or `فصل`, so the pronoun `I` is safe. Common
+    abbreviations that happen to be valid numerals (`DC`, `MD`, `CD`, ...) are
+    skipped, see `AMBIGUOUS_ABBREVIATIONS`.
+
+    The language is detected ignoring the numerals themselves, so a Persian
+    sentence stays Persian even with a long numeral like `XVIII`.
+    """
+
+    def __init__(self, epsilon: float = 1.0):
+        super(RomanNumeralProcessor, self).__init__(epsilon=epsilon)
+
+    def process(self, text: str, language: str = None, *args, **kwargs) -> str:
+        if random.random() > self.epsilon:
+            return text
+
+        found = []
+        for match in ROMAN_CANDIDATE.finditer(text):
+            value = self._value_to_read(text, match)
+            if value is not None:
+                found.append((match, value))
+        if not found:
+            return text
+
+        language = self._resolve_language(text, found, language)
+        if language is None:
+            return text
+
+        spell = words if language == "fa" else num2words
+        pieces = []
+        last = 0
+        for match, value in found:
+            pieces.append(text[last:match.start()])
+            pieces.append(spell(value))
+            last = match.end()
+        pieces.append(text[last:])
+        return "".join(pieces)
+
+    @staticmethod
+    def _value_to_read(text: str, match: re.Match):
+        token = match.group(0)
+        if token in AMBIGUOUS_ABBREVIATIONS:
+            return None
+
+        value = roman_to_int(token)
+        if value is None:
+            return None
+
+        if len(token) > 1:
+            return value
+
+        if token not in SINGLE_LETTER_NUMERALS:
+            return None
+        previous = re.search(
+            rf"([^\W\d_]+)(?:[^\S\n]+|[^\S\n]*{OPENERS}[^\S\n]*)$", text[: match.start()]
+        )
+        if previous is None:
+            return None
+        word = previous.group(1)
+        if word in FA_TRIGGERS:
+            return value
+        if word.lower() in EN_TRIGGERS:
+            # `Part I` yes, `after the war I went` no.
+            if token == "I" and not word[0].isupper():
+                return None
+            return value
+        return None
+
+    @staticmethod
+    def _resolve_language(text: str, found, language: str):
+        """Detects the language ignoring the numerals that will be replaced."""
+        pieces = []
+        last = 0
+        for match, _ in found:
+            pieces.append(text[last:match.start()])
+            last = match.end()
+        pieces.append(text[last:])
+
+        detected = detect_language(" ".join(pieces))
+        if detected in ("fa", "en"):
+            return detected
+        if detected != NO_LETTERS:
+            return None
+        return language if language in ("fa", "en") else None
+
+
+# ---------------------------------------------------------------------------
+# Ampersand
+# ---------------------------------------------------------------------------
+
+HTML_ENTITY = r"(?:#\d+|#[xX][0-9a-fA-F]+|[A-Za-z][A-Za-z0-9]*)"
+
+# A single `&`: not `&&`, and not the start of an HTML entity like `&amp;`.
+AMPERSAND_SIGN = rf"(?<!&)&(?!&|{HTML_ENTITY};)"
+AMPERSAND = re.compile(AMPERSAND_SIGN)
+
+AMOUNT = r"\d+(?:[.,٫٬]\d+)*"
+
+# A number right before an `&` (`3 & 4` -> the 3), possibly wrapped: `(3) & (4)`,
+# `3 (&) 4`. Not part of a date, time, range, URL or currency
+# (`2024-05-10`, `10:30`, `5-6`, `a=1&b`, `$3`).
+NUMBER_BEFORE_AMPERSAND = re.compile(
+    rf"(?<![\w./,٫٬=:\-$€£])(?P<number>{AMOUNT})"
+    rf"(?=(?:[^\S\n]|{CLOSERS})*(?:{OPENERS}[^\S\n]*)?{AMPERSAND_SIGN})"
+)
+
+# A number right after an `&` (`3 & 4` -> the 4).
+NUMBER_AFTER_AMPERSAND = re.compile(
+    rf"(?P<sign>{AMPERSAND_SIGN}(?:[^\S\n]|{CLOSERS})*(?:{OPENERS}[^\S\n]*)?)"
+    rf"(?P<number>{AMOUNT})(?![\w/:\-$€£%°]|[.,٫٬]\d)"
+)
+
+# `(&)`, `[&]`, `"&"`: the symbol alone inside a wrapper.
+WRAPPED_AMPERSAND = re.compile(rf"(?<!&)(?P<open>{OPENERS})[^\S\n]*&[^\S\n]*(?P<close>{CLOSERS})")
+
+
+def number_to_words(number: str, language: str) -> str:
+    """`3` -> `three` / `سه`; handles Persian digits, `1,000` and decimals."""
+    clean = digits.fa_to_en(number).replace(",", "").replace("٬", "").replace("٫", ".")
+    if language == "fa":
+        return words(float(clean) if "." in clean else int(clean))
+    return num2words(Decimal(clean) if "." in clean else int(clean))
+
+
+class AmpersandProcessor(BaseProcessor):
+    """Converts `&` into `and` (English) or `و` (Persian).
+
+    English: `Tom & Jerry` -> `Tom and Jerry`
+             `R&D`         -> `R and D`
+             `3 & 4`       -> `three and four`
+    Persian: `تام & جری`    -> `تام و جری`
+             `۳ & ۴`        -> `سه و چهار`
+
+    A plain number right next to an `&` is read as words too (`Windows 10 & 11`
+    -> `Windows ten and eleven`, `2.5 & 3.5` -> `two point five and three point
+    five`). Fractions, units, percents and currency beside an `&` are already
+    read by their own processors (`1/2 & 1/3`, `5 km & 3 kg`, `5% & 10%`,
+    `$3 & $4`).
+
+    Left alone: `&&`, HTML entities (`&amp;`) and URLs / query strings
+    (`a=1&b=2`). With no letters in the text, the digits pick the language:
+    `۱ & ۲` -> `یک و دو`, `1 & 2` -> `one and two`.
+    """
+
+    def __init__(self, epsilon: float = 1.0):
+        super(AmpersandProcessor, self).__init__(epsilon=epsilon)
+
+    def process(self, text: str, language: str = None, *args, **kwargs) -> str:
+        if random.random() > self.epsilon:
+            return text
+
+        if language in (None, NO_LETTERS):
+            language = detect_digit_language(text)
+        if language not in ("fa", "en"):
+            return text
+
+        protected = [
+            match.span()
+            for match in re.finditer(r"\S+", text)
+            if "://" in match.group(0) or "=" in match.group(0)
+        ]
+
+        def is_protected(position: int) -> bool:
+            return any(start <= position < end for start, end in protected)
+
+        def read_number(match: re.Match) -> str:
+            if is_protected(match.start()):
+                return match.group(0)
+            return number_to_words(match.group("number"), language)
+
+        def read_number_after(match: re.Match) -> str:
+            if is_protected(match.start()):
+                return match.group(0)
+            return match.group("sign") + number_to_words(match.group("number"), language)
+
+        text = NUMBER_BEFORE_AMPERSAND.sub(read_number, text)
+        text = NUMBER_AFTER_AMPERSAND.sub(read_number_after, text)
+
+        word = "و" if language == "fa" else "and"
+        return re.sub(r"\S+", lambda match: self._convert_token(match.group(0), word), text)
+
+    @staticmethod
+    def _convert_token(token: str, word: str) -> str:
+        if "&" not in token or "://" in token or "=" in token:
+            return token
+        token = WRAPPED_AMPERSAND.sub(rf"\g<open>{word}\g<close>", token)
+        return AMPERSAND.sub(f" {word} ", token).strip()
+
+
+# ---------------------------------------------------------------------------
+# Negative sign
+# ---------------------------------------------------------------------------
+
+# A minus right before a number that is not part of a word, a date or a range:
+# `-2`, `(-2)`, `= -2`, `-$5`. `a-2`, `5-3`, `COVID-19`, `2024-05-10` don't match.
+NEGATIVE_PATTERN = re.compile(
+    r"(?<![\w.,٫٬)\]}/\-−])(?P<sign>[-−])(?P<space>[^\S\n]*)"
+    r"(?P<currency>[$€£])?(?P<number>" + AMOUNT + r")"
+)
+
+NEGATIVE_WORDS = {"en": "negative", "fa": "منفی"}
+
+
+class NegativeSignProcessor(BaseProcessor):
+    """Converts a minus sign before a number into `negative` / `منفی`.
+
+    English: `-2`     -> `negative two`
+             `-2 m`   -> `negative two metres`
+             `-1/2 kg`-> `negative one half kilogram`
+    Persian: `-۲ m`   -> `منفی دو متر`
+             `-۲`     -> `منفی دو`
+
+    A number followed by a unit, fraction, percent or currency is left for
+    those processors, so this must run before `UnitProcessor`. A space after the
+    minus (`- 2 m`) is accepted only with a unit, or right after an operator
+    (`3 + - 2`, `= - 2`), so a list item like `- 2024 report` is not touched.
+    """
+
+    def __init__(self, epsilon: float = 1.0):
+        super(NegativeSignProcessor, self).__init__(epsilon=epsilon)
+
+    def process(self, text: str, language: str = None, *args, **kwargs) -> str:
+        if random.random() > self.epsilon:
+            return text
+        if NEGATIVE_PATTERN.search(text) is None:
+            return text
+
+        language = UnitProcessor._resolve_language(text, language)
+        if language not in ("fa", "en"):
+            return text
+
+        return NEGATIVE_PATTERN.sub(lambda match: self._convert(match, language), text)
+
+    @staticmethod
+    def _convert(match: re.Match, language: str) -> str:
+        text = match.string
+        start = match.start("number")
+        end = match.end("number")
+        word = NEGATIVE_WORDS[language]
+        currency = match.group("currency") or ""
+
+        unit_follows = UNIT_PATTERN.match(text, start) is not None
+        fraction_follows = FRACTION_PATTERN.match(text, start) is not None
+        suffix_follows = text[end:end + 1] in tuple("$€£%٪°")
+        leave_digits = bool(currency) or unit_follows or fraction_follows or suffix_follows
+
+        if match.group("space"):
+            before = text[: match.start()].rstrip()
+            after_operator = before[-1:] in tuple("+×÷=*")
+            # `10 - 20 km` is a range, not a negative number.
+            after_operand = bool(before) and (
+                bool(re.match(r"\w", before[-1])) or before[-1] in tuple("%٪$€£°)]")
+            )
+            if not (after_operator or (unit_follows and not after_operand)):
+                return match.group(0)
+
+        if leave_digits:
+            return f"{word} {currency}{match.group('number')}"
+
+        following = text[end:end + 1]
+        if following and re.match(r"[\w/:\-]", following):
+            return match.group(0)
+        return f"{word} {number_to_words(match.group('number'), language)}"
+
+
+# ---------------------------------------------------------------------------
+# Math operators
+# ---------------------------------------------------------------------------
+
+OPERATOR_WORDS = {
+    "en": {"+": "plus", "×": "times", "*": "times", "÷": "divided by", "=": "equals", "−": "minus", "-": "minus"},
+    "fa": {"+": "بعلاوه", "×": "ضربدر", "*": "ضربدر", "÷": "تقسیم بر", "=": "مساوی است با", "−": "منهای", "-": "منهای"},
+}
+
+OPERATOR_CHARS = re.compile(r"[+×÷=*−-]")
+
+# Wrappers and spaces may sit between an operand and the symbol: `(3) + (4)`, `3 (+) 4`.
+NUMBER_LEFT_OF_OPERATOR = re.compile(
+    r"(?<![\w./,٫٬=:\-$€£])(?P<number>" + AMOUNT + rf")(?:[^\S\n]|{WRAPPERS})*$"
+)
+NUMBER_RIGHT_OF_OPERATOR = re.compile(
+    rf"(?:[^\S\n]|{WRAPPERS})*(?P<number>" + AMOUNT + r")(?![\w/:\-$€£%°]|[.,٫٬]\d)"
+)
+
+
+class MathOperatorProcessor(BaseProcessor):
+    """Converts `+`, `×`, `÷`, `=` (and `*`, `−`) between operands into words.
+
+    English: `3 + 4`     -> `three plus four`
+             `6 × 2`     -> `six times two`
+             `8 ÷ 2`     -> `eight divided by two`
+             `3 + 4 = 7` -> `three plus four equals seven`
+    Persian: `۳ + ۴`     -> `سه بعلاوه چهار`
+             `۶ × ۲`     -> `شش ضربدر دو`
+             `۸ ÷ ۲`     -> `هشت تقسیم بر دو`
+             `۷ = ۷`     -> `هفت مساوی است با هفت`
+
+    A symbol is read only between two operands, spaced on both sides (`3 + 4`)
+    or glued between two digits (`3+4`), so `+98 912 ...`, `C++`, `A+ grade`,
+    `x=5`, `a=1&b=2`, `==` and bullet `* item` are left alone. A plain number
+    next to the symbol is read as words too; numbers with units, fractions or
+    currency are read by their own processors. An ASCII `-` is `minus` only
+    with spaces on both sides in a line that also has an `=` (`5 - 3 = 2`);
+    elsewhere it is a dash or a range.
+    """
+
+    def __init__(self, epsilon: float = 1.0):
+        super(MathOperatorProcessor, self).__init__(epsilon=epsilon)
+
+    def process(self, text: str, language: str = None, *args, **kwargs) -> str:
+        if random.random() > self.epsilon:
+            return text
+        if OPERATOR_CHARS.search(text) is None:
+            return text
+
+        if language in (None, NO_LETTERS):
+            language = detect_digit_language(text)
+        if language not in ("fa", "en"):
+            return text
+
+        protected = [
+            match.span()
+            for match in re.finditer(r"\S+", text)
+            if "://" in match.group(0) or ("?" in match.group(0) and "=" in match.group(0))
+        ]
+        operators = self._find_operators(text, protected)
+        if not operators:
+            return text
+
+        replacements = {}
+        for position, char in operators:
+            end = position + 1
+            left = NUMBER_LEFT_OF_OPERATOR.search(text, 0, position)
+            if left is not None:
+                replacements[left.span("number")] = number_to_words(left.group("number"), language)
+            right = NUMBER_RIGHT_OF_OPERATOR.match(text, end)
+            if right is not None:
+                replacements[right.span("number")] = number_to_words(right.group("number"), language)
+
+            # No extra space next to a wrapper: `3 (+) 4` -> `3 (plus) 4`.
+            before = " " if not re.match(rf"[\s]|{WRAPPERS}", text[position - 1]) else ""
+            after = " " if not re.match(rf"[\s]|{WRAPPERS}", text[end]) else ""
+            replacements[(position, end)] = f"{before}{OPERATOR_WORDS[language][char]}{after}"
+
+        pieces = []
+        last = 0
+        for (start, stop), replacement in sorted(replacements.items()):
+            pieces.append(text[last:start])
+            pieces.append(replacement)
+            last = stop
+        pieces.append(text[last:])
+        return "".join(pieces)
+
+    @staticmethod
+    def _find_operators(text: str, protected):
+        def is_protected(position: int) -> bool:
+            return any(start <= position < end for start, end in protected)
+
+        def is_between_operands(position: int) -> bool:
+            line_start = text.rfind("\n", 0, position) + 1
+            line_end = text.find("\n", position)
+            line_end = len(text) if line_end == -1 else line_end
+            if not text[line_start:position].strip() or not text[position + 1:line_end].strip():
+                return False
+            # Look past spaces and wrappers on both sides: `(3) + (4)`, `3 (+) 4`.
+            before = re.search(rf"(?:[^\S\n]|{WRAPPERS})*$", text[line_start:position]).group(0)
+            after = re.match(rf"(?:[^\S\n]|{WRAPPERS})*", text[position + 1:line_end]).group(0)
+            if before and after:
+                return True
+            return text[position - 1].isdigit() and text[position + 1].isdigit()
+
+        found = []
+        equation_lines = set()
+        for match in OPERATOR_CHARS.finditer(text):
+            position, char = match.start(), match.group(0)
+            if char == "-" or is_protected(position) or position == 0 or position + 1 >= len(text):
+                continue
+            if is_between_operands(position):
+                found.append((position, char))
+                if char == "=":
+                    equation_lines.add(text.rfind("\n", 0, position))
+
+        # An ASCII hyphen is a minus only in a spaced equation: `5 - 3 = 2`.
+        for match in re.finditer(r"-", text):
+            position = match.start()
+            if position == 0 or position + 1 >= len(text) or is_protected(position):
+                continue
+            left, right = text[position - 1], text[position + 1]
+            if left in " \t" and right in " \t" and text.rfind("\n", 0, position) in equation_lines \
+                    and is_between_operands(position):
+                found.append((position, "-"))
+
+        return sorted(found)
+
+
+# ---------------------------------------------------------------------------
+# Ranges
+# ---------------------------------------------------------------------------
+
+RANGE_WORDS = {"en": "to", "fa": "تا"}
+
+RANGE_DASH = re.compile(r"[-–]")
+
+BARE_NUMBER_LEFT = re.compile(r"(?<![\w./,٫٬=:\-–$€£])(?P<number>" + AMOUNT + r")$")
+BARE_NUMBER_RIGHT = re.compile(r"(?P<number>" + AMOUNT + r")(?![\w/:\-–$€£%°]|[.,٫٬]\d)")
+CURRENCY_LEFT = re.compile(r"(?<![\w./,٫٬=:\-–])(?:[$€£]" + AMOUNT + r"|" + AMOUNT + r"[$€£])$")
+CURRENCY_RIGHT = re.compile(r"(?:[$€£]" + AMOUNT + r"|" + AMOUNT + r"[$€£])(?![\w/:\-–])")
+
+
+class RangeProcessor(BaseProcessor):
+    """Converts a dash between two amounts into `to` / `تا`.
+
+    English: `3 - 5`         -> `three to five`
+             `pages 3-5`     -> `pages three to five`
+             `3kg - 5 kg`    -> `three kilograms to five kilograms`
+             `10 - 20 km`    -> `ten to twenty kilometres`
+             `$3 - $5`       -> `three dollars to five dollars`
+             `1/2 - 3/4`     -> `one half to three quarters`
+    Persian: `۳ - ۵`         -> `سه تا پنج`
+             `۳kg - ۵ kg`    -> `سه کیلوگرم تا پنج کیلوگرم`
+
+    Both sides must be amounts: plain numbers (read here), or numbers with a
+    unit, fraction, percent or currency (read by their own processors, so this
+    must run before `UnitProcessor`). Hyphen and en dash (–) are accepted,
+    spaced on both sides (`3 - 5`) or on neither (`3-5`).
+
+    Left alone: dates and phone numbers (`2024-05-10`, `0912-123-4567`), long
+    numbers glued to a hyphen (`1990-2000`, `555-1234`; with spaces they are
+    read), leading zeros (`2024-05`), times (`10:30 - 11:30`), words (`well - maybe`),
+    URLs, and any line with an `=` (there `5 - 3` is a subtraction, which
+    `MathOperatorProcessor` reads).
+    """
+
+    def __init__(self, epsilon: float = 1.0):
+        super(RangeProcessor, self).__init__(epsilon=epsilon)
+
+    def process(self, text: str, language: str = None, *args, **kwargs) -> str:
+        if random.random() > self.epsilon:
+            return text
+        if RANGE_DASH.search(text) is None:
+            return text
+
+        ranges = self._find_ranges(text)
+        if not ranges:
+            return text
+
+        language = UnitProcessor._resolve_language(text, language)
+        if language not in ("fa", "en"):
+            return text
+
+        replacements = {}
+        for dash, attached, bare_spans in ranges:
+            for span in bare_spans:
+                replacements[span] = number_to_words(text[span[0]:span[1]], language)
+            padding = " " if attached else ""
+            replacements[(dash, dash + 1)] = f"{padding}{RANGE_WORDS[language]}{padding}"
+
+        pieces = []
+        last = 0
+        for (start, stop), replacement in sorted(replacements.items()):
+            pieces.append(text[last:start])
+            pieces.append(replacement)
+            last = stop
+        pieces.append(text[last:])
+        return "".join(pieces)
+
+    @staticmethod
+    def _find_ranges(text: str):
+        protected = [
+            match.span() for match in re.finditer(r"\S+", text) if "://" in match.group(0)
+        ]
+        found = []
+        for match in RANGE_DASH.finditer(text):
+            dash = match.start()
+            if dash == 0 or dash + 1 >= len(text):
+                continue
+            if any(start <= dash < end for start, end in protected):
+                continue
+
+            line_start = text.rfind("\n", 0, dash) + 1
+            line_end = text.find("\n", dash)
+            line_end = len(text) if line_end == -1 else line_end
+            if "=" in text[line_start:line_end]:
+                continue
+
+            left_char, right_char = text[dash - 1], text[dash + 1]
+            spaced = left_char in " \t" and right_char in " \t"
+            attached = not left_char.isspace() and not right_char.isspace()
+            if not (spaced or attached):
+                continue
+
+            # `(3) - (5)`: look past a closing wrapper on the left and an
+            # opening one on the right.
+            raw_left_end = len(text[:dash].rstrip(" \t"))
+            left_end = raw_left_end
+            while left_end and re.match(CLOSERS, text[left_end - 1]):
+                left_end = len(text[:left_end - 1].rstrip(" \t"))
+            raw_right_start = dash + 1
+            while raw_right_start < len(text) and text[raw_right_start] in " \t":
+                raw_right_start += 1
+            right_start = raw_right_start
+            while right_start < len(text) and re.match(rf"[ \t]|{OPENERS}", text[right_start]):
+                right_start += 1
+            # A wrapped unit (`3 (cm)`) ends with its closer, so try that first.
+            left = RangeProcessor._left_operand(text, raw_left_end) or RangeProcessor._left_operand(text, left_end)
+            right = RangeProcessor._right_operand(text, raw_right_start) or RangeProcessor._right_operand(text, right_start)
+            if left is None or right is None:
+                continue
+
+            if attached and match.group(0) == "-" and not RangeProcessor._is_plain_pair(text, left, right):
+                continue
+
+            found.append((dash, attached, [span for span in (left[0], right[0]) if span is not None]))
+        return found
+
+    @staticmethod
+    def _left_operand(text: str, end: int):
+        """(bare number span or None, digits) of the amount ending at `end`."""
+        for pattern in (UNIT_PATTERN, FRACTION_PATTERN):
+            for match in pattern.finditer(text, 0, end):
+                if match.end() == end:
+                    return None, match.group(0)
+        match = CURRENCY_LEFT.search(text, 0, end)
+        if match is not None:
+            return None, match.group(0)
+        match = BARE_NUMBER_LEFT.search(text, 0, end)
+        if match is not None:
+            return match.span("number"), match.group("number")
+        return None
+
+    @staticmethod
+    def _right_operand(text: str, start: int):
+        for pattern in (UNIT_PATTERN, FRACTION_PATTERN, CURRENCY_RIGHT):
+            match = pattern.match(text, start)
+            if match is not None:
+                return None, match.group(0)
+        match = BARE_NUMBER_RIGHT.match(text, start)
+        if match is not None:
+            return match.span("number"), match.group("number")
+        return None
+
+    @staticmethod
+    def _is_plain_pair(text: str, left, right) -> bool:
+        """Is `3-5` (no spaces) a range rather than a date, phone number or code?"""
+        for _, operand in (left, right):
+            integer_part = re.match(r"[\d,٬]*", digits.fa_to_en(operand)).group(0)
+            integer_part = integer_part.replace(",", "").replace("٬", "")
+            if not integer_part or len(integer_part) > 3:
+                return False
+            if len(integer_part) > 1 and integer_part.startswith("0"):
+                return False
+        return True

rules/times.py

@@ -0,0 +1,91 @@
+from rules.base import BaseProcessor
+from datetime import time
+from num2words import num2words
+from num2fawords import words
+import dateparser
+from utils import convert_fa_num_to_en
+import random
+import re
+
+
+class TimeProcessor(BaseProcessor):
+    def __init__(self, epsilon:float=1.0):
+        super(TimeProcessor, self).__init__(epsilon=epsilon)
+
+    def process(self, str_time:str, language:str='en') -> str:
+        if random.random() > self.epsilon:
+            return str_time
+
+        str_time_format = convert_fa_num_to_en(str_time)
+        if not self._is_time(str_time_format):
+            return str_time
+
+        try:
+            time_ = self._text_to_time(str_time_format)
+        except ValueError:
+            return str_time
+
+        if language == 'en':
+            return self._english_time_to_text(time_)
+        elif language == 'fa':
+            return self._farsi_time_to_text(time_)
+        return str_time
+
+    @staticmethod
+    def _text_to_time(text: str) -> time:
+        dt = dateparser.parse(
+            text,
+            languages=["en"],
+            settings={
+                "RETURN_AS_TIMEZONE_AWARE": False,
+                "PREFER_DATES_FROM": "current_period",
+            },
+        )
+
+        if dt is None:
+            raise ValueError(f"Could not parse time: {text!r}")
+
+        return dt.time()
+
+    @staticmethod
+    def _english_time_to_text(time_:time) -> str:
+        hour = time_.hour
+        minute = time_.minute
+        second = time_.second
+
+        ampm = ""
+        if hour > 12:
+            hour -= 12
+            ampm = " PM"
+
+        if second > 1:
+            sec_name = "seconds"
+        else:
+            sec_name = "second"
+
+        if minute == 0 and second == 0:
+            return f"{num2words(hour)}{ampm}"
+        elif minute != 0 and second == 0:
+            return f"{num2words(hour)} {num2words(minute)}{ampm}"
+        elif minute == 0 and second != 0:
+            return f"{num2words(hour)}{ampm} and {num2words(second)} {sec_name}"
+        else:
+            return f"{num2words(hour)} {num2words(minute)}{ampm} and {num2words(second)} {sec_name}"
+
+    @staticmethod
+    def _farsi_time_to_text(time_: time) -> str:
+        hour = time_.hour
+        minute = time_.minute
+        second = time_.second
+
+        text = f"{words(hour)}"
+        if minute != 0:
+            text += " و " + words(minute) + " دقیقه"
+        if second != 0:
+            text += " و " + words(second) + " ثانیه"
+        return text
+
+    @staticmethod
+    def _is_time(text: str) -> bool:
+        pattern = r"^(?:[0-9]|1[0-9]|2[0-3]):[0-5]\d(?::[0-5]\d)?$"
+        return bool(re.fullmatch(pattern, text))

rules/units.py

@@ -0,0 +1,324 @@
+import random
+import re
+from dataclasses import dataclass
+from decimal import Decimal
+
+from num2fawords import words
+from num2words import num2words
+from persiantools import digits
+
+from rules.base import BaseProcessor
+from rules.fraction import FRACTION_BODY, VULGAR_FRACTIONS, FractionProcessor
+from rules.wrappers import CLOSERS, OPENERS
+from utils import NO_LETTERS, detect_digit_language, detect_language
+
+
+@dataclass(frozen=True)
+class Unit:
+    fa: str
+    en: str
+    en_plural: str
+    # Next smaller unit and how many of it make one of this unit, used to read
+    # `1000.5 m` as "1000 metres and 50 centimetres".
+    minor: "Unit" = None
+    factor: int = 1
+
+
+MILLIMETRE = Unit("میلیمتر", "millimetre", "millimetres")
+CENTIMETRE = Unit("سانتیمتر", "centimetre", "centimetres", MILLIMETRE, 10)
+METRE = Unit("متر", "metre", "metres", CENTIMETRE, 100)
+KILOMETRE = Unit("کیلومتر", "kilometre", "kilometres", METRE, 1000)
+MILE = Unit("مایل", "mile", "miles")
+FOOT = Unit("فوت", "foot", "feet")
+
+MILLIGRAM = Unit("میلی‌گرم", "milligram", "milligrams")
+GRAM = Unit("گرم", "gram", "grams", MILLIGRAM, 1000)
+KILOGRAM = Unit("کیلوگرم", "kilogram", "kilograms", GRAM, 1000)
+POUND = Unit("پوند", "pound", "pounds")
+OUNCE = Unit("اونس", "ounce", "ounces")
+
+MILLILITRE = Unit("میلی‌لیتر", "millilitre", "millilitres")
+LITRE = Unit("لیتر", "litre", "litres", MILLILITRE, 1000)
+
+MILLISECOND = Unit("میلی‌ثانیه", "millisecond", "milliseconds")
+SECOND = Unit("ثانیه", "second", "seconds", MILLISECOND, 1000)
+MINUTE = Unit("دقیقه", "minute", "minutes", SECOND, 60)
+HOUR = Unit("ساعت", "hour", "hours", MINUTE, 60)
+DAY = Unit("روز", "day", "days", HOUR, 24)
+WEEK = Unit("هفته", "week", "weeks", DAY, 7)
+MONTH = Unit("ماه", "month", "months")
+YEAR = Unit("سال", "year", "years")
+
+PERCENT = Unit("درصد", "percent", "percent")
+
+KILOMETRE_PER_HOUR = Unit("کیلومتر بر ساعت", "kilometre per hour", "kilometres per hour")
+METRE_PER_SECOND = Unit("متر بر ثانیه", "metre per second", "metres per second")
+
+CELSIUS = Unit("درجه سانتیگراد", "degree Celsius", "degrees Celsius")
+FAHRENHEIT = Unit("درجه فارنهایت", "degree Fahrenheit", "degrees Fahrenheit")
+
+# Symbols are case-sensitive on purpose (e.g. `5G` is a network, not grams).
+# Single `s` and `d` are left out on purpose: `1990s` and `3d` are not units.
+UNITS = {
+    "km": KILOMETRE,
+    "m": METRE,
+    "cm": CENTIMETRE,
+    "mm": MILLIMETRE,
+    "mi": MILE,
+    "ft": FOOT,
+    "kg": KILOGRAM,
+    "KG": KILOGRAM,
+    "Kg": KILOGRAM,
+    "g": GRAM,
+    "mg": MILLIGRAM,
+    "lb": POUND,
+    "lbs": POUND,
+    "oz": OUNCE,
+    "l": LITRE,
+    "L": LITRE,
+    "ml": MILLILITRE,
+    "mL": MILLILITRE,
+    "ms": MILLISECOND,
+    "sec": SECOND,
+    "secs": SECOND,
+    "second": SECOND,
+    "seconds": SECOND,
+    "min": MINUTE,
+    "mins": MINUTE,
+    "minute": MINUTE,
+    "minutes": MINUTE,
+    "h": HOUR,
+    "hr": HOUR,
+    "hrs": HOUR,
+    "hour": HOUR,
+    "hours": HOUR,
+    "day": DAY,
+    "days": DAY,
+    "week": WEEK,
+    "weeks": WEEK,
+    "month": MONTH,
+    "months": MONTH,
+    "year": YEAR,
+    "years": YEAR,
+    "%": PERCENT,
+    "٪": PERCENT,
+    "km/h": KILOMETRE_PER_HOUR,
+    "m/s": METRE_PER_SECOND,
+    "°C": CELSIUS,
+    "°F": FAHRENHEIT,
+}
+
+AMOUNT = r"\d+(?:[.,٫٬]\d+)*"
+
+# Longest symbols first so `km/h` wins over `km`, and `mg` over `m`.
+SYMBOLS = "|".join(re.escape(symbol) for symbol in sorted(UNITS, key=len, reverse=True))
+
+PERCENT_SIGNS = ("%", "٪")
+OTHER_SYMBOLS = "|".join(
+    re.escape(symbol)
+    for symbol in sorted(UNITS, key=len, reverse=True)
+    if symbol not in PERCENT_SIGNS
+)
+
+# A number or fraction, optional spaces, then a unit symbol: `5km`, `۵ km`,
+# `2.5 kg`, `1/2 kg`, `2 1/2 kg`. The symbol must stand alone, so `5 km²`
+# or `5G` don't match unless they are in the table.
+# The amount and/or the symbol may be wrapped: `2 (kg)`, `3 {m}`, `(5) kg`.
+# After a percent sign a unit symbol may follow (`5% kg`): that is `per_unit`.
+UNIT_PATTERN = re.compile(
+    rf"""
+    (?<![\w./,٫٬])
+    (?:(?P<aopen>{OPENERS})[^\S\n]*)?
+    (?:{FRACTION_BODY}|(?P<amount>{AMOUNT}))
+    (?(aopen)[^\S\n]*(?P<aclose>{CLOSERS}))
+    [^\S\n]*
+    (?:(?P<sopen>{OPENERS})[^\S\n]*)?
+    (?P<symbol>{SYMBOLS})
+    (?(sopen)[^\S\n]*(?P<sclose>{CLOSERS}))
+    (?![\w/])
+    (?P<tail>[^\S\n]+(?P<per_unit>{OTHER_SYMBOLS})(?![\w/]))?
+    """,
+    re.VERBOSE,
+)
+
+
+# A percent sign typed before the number (`%20`, `% 20`, `٪ ۲۰`). It is moved
+# after the number so the normal pattern can read it. `%d`, `%s`, `% 5d` and
+# `a%20b` don't match.
+PREFIX_PERCENT_PATTERN = re.compile(
+    rf"(?<![\w%٪])(?P<sign>[%٪])[^\S\n]*(?P<amount>{AMOUNT})(?![\w%٪/])"
+)
+
+
+class UnitProcessor(BaseProcessor):
+    """Converts a number followed by a unit symbol into spoken text.
+
+    Persian: `5 km`  -> `پنج کیلو��تر`
+             `۷۵kg`  -> `هفتاد و پنج کیلوگرم`
+    English: `5 km`  -> `five kilometres`
+             `1 kg`  -> `one kilogram`
+             `25°C`  -> `twenty-five degrees Celsius`
+             `2 %`   -> `two percent` (Persian: `۲ ٪` -> `دو درصد`)
+             `10 sec` -> `ten seconds`
+
+    Decimals are split into smaller units when the unit has one:
+             `1000.5 m` -> `یک هزار متر و پنجاه سانتیمتر`
+             `2.5 kg`   -> `two kilograms and five hundred grams`
+             `1.5 h`    -> `one hour and thirty minutes`
+             `1/2 %`    -> `one half percent`
+    Other decimals are read as a plain number: `2.5 mi` -> `two point five miles`.
+
+    A percent sign may come before the number (`% 5`, `٪ ۵`), and a unit symbol
+    after a percent sign is read on its own: `5% kg` -> `five percent kilogram`.
+
+    With nothing but units in the text, Persian digits give Persian
+    (`۱۰۰۰m` -> `یک هزار متر`) and ASCII digits give English.
+
+    Works on the whole text (not word by word), so both `5km` and `5 km`
+    are handled. Add a unit by adding one line to `UNITS`.
+    """
+
+    def __init__(self, epsilon: float = 1.0):
+        super(UnitProcessor, self).__init__(epsilon=epsilon)
+
+    def process(self, text: str, language: str = None, *args, **kwargs) -> str:
+        if random.random() > self.epsilon:
+            return text
+
+        language = self._resolve_language(text, language)
+        if language is None:
+            return text
+
+        def convert(match: re.Match) -> str:
+            return self._convert(match, language)
+
+        text = PREFIX_PERCENT_PATTERN.sub(self._move_prefix_percent, text)
+        return UNIT_PATTERN.sub(convert, text)
+
+    @staticmethod
+    def _move_prefix_percent(match: re.Match) -> str:
+        """`% 20 cm` -> `20% cm`, so it reads as `twenty percent centimetre`."""
+        before = match.string[: match.start()].rstrip(" \t")
+        # In `5 % 2` the sign belongs to the 5 before it, not to the 2 after it.
+        if before and (before[-1].isdigit() or before[-1] in VULGAR_FRACTIONS):
+            return match.group(0)
+        return f"{match.group('amount')}{match.group('sign')}"
+
+    @staticmethod
+    def _resolve_language(text: str, language: str):
+        """Detects the language ignoring the unit symbols themselves.
+
+        Symbols like `km` are Latin letters and would otherwise tip a Persian
+        sentence towards English. If nothing but units is left (e.g. `۱۰۰۰m`),
+        Persian digits mean Persian; otherwise the given `language` is used,
+        and with no language given ASCII digits mean English.
+        """
+        detected = detect_language(UNIT_PATTERN.sub(" ", text))
+        if detected in ("fa", "en"):
+            return detected
+        if detected != NO_LETTERS:
+            return None
+
+        digit_language = detect_digit_language(text)
+        if digit_language == "fa":
+            return "fa"
+        if language in ("fa", "en"):
+            return language
+        if language is None or language == NO_LETTERS:
+            return digit_language
+        return None
+
+    @staticmethod
+    def _convert(match: re.Match, language: str) -> str:
+        unit = UNITS[match.group("symbol")]
+        pieces = UnitProcessor._amount_pieces(match, unit, language)
+        if pieces is None:
+            return match.group(0)
+
+        # Wrappers stay where they were: `2 (kg)` -> `two (kilograms)`.
+        rendered = []
+        for index, (amount_text, unit_text) in enumerate(pieces):
+            if index == 0 and match.group("aopen"):
+                amount_text = f"{match.group('aopen')}{amount_text}{match.group('aclose')}"
+            if match.group("sopen"):
+                unit_text = f"{match.group('sopen')}{unit_text}{match.group('sclose')}"
+            rendered.append(f"{amount_text} {unit_text}")
+
+        if language == "fa" or len(rendered) == 1:
+            text = " و ".join(rendered) if language == "fa" else rendered[0]
+        else:
+            text = f"{', '.join(rendered[:-1])} and {rendered[-1]}"
+
+        per_unit = match.group("per_unit")
+        if per_unit is None:
+            return text
+
+        if unit is PERCENT:
+            # `5% kg` -> `five percent kilogram`: the unit has no number of its own.
+            other = UNITS[per_unit]
+            return f"{text} {other.fa if language == 'fa' else other.en}"
+
+        # A symbol after any other unit is not ours to convert: put it back.
+        return text + match.group("tail")
+
+    @staticmethod
+    def _amount_pieces(match: re.Match, unit: Unit, language: str):
+        """Reads the amount as [(amount words, unit words), ...]; None if it
+        should stay unchanged. Several pieces mean a split decimal (`2.5 kg`)."""
+        if match.group("amount") is None:
+            return UnitProcessor._fraction_pieces(match, unit, language)
+
+        amount = digits.fa_to_en(match.group("amount"))
+        clean = amount.replace(",", "").replace("٬", "").replace("٫", ".")
+
+        parts = UnitProcessor._split_into_units(Decimal(clean), unit)
+
+        if language == "fa":
+            if parts is None:
+                return [(words(float(clean)), unit.fa)]
+            return [(words(number), part.fa) for number, part in parts]
+
+        if parts is None:
+            return [(num2words(Decimal(clean)), unit.en_plural)]
+        return [
+            (num2words(number), part.en if number == 1 else part.en_plural)
+            for number, part in parts
+        ]
+
+    @staticmethod
+    def _split_into_units(value: Decimal, unit: Unit):
+        """Splits a decimal amount into whole numbers of smaller units.
+
+        `1000.5 m` -> [(1000, metre), (50, centimetre)]. Returns None when the
+        fraction does not end exactly on the smallest known unit (or the unit
+        has no smaller unit), so the caller reads it as a plain decimal.
+        """
+        parts = []
+        while True:
+            whole = int(value)
+            fraction = value - whole
+            if whole:
+                parts.append((whole, unit))
+            if fraction == 0:
+                return parts or [(0, unit)]
+            if unit.minor is None:
+                return None
+            value = fraction * unit.factor
+            unit = unit.minor
+
+    @staticmethod
+    def _fraction_pieces(match: re.Match, unit: Unit, language: str):
+        """`1/2 kg` -> `one half` + `kilogram`, `1,000/2 kg` -> `one thousand halves` + `kilograms`."""
+        parsed = FractionProcessor.parse(match)
+        if parsed is None:
+            return None
+
+        whole, numerator, denominator = parsed
+        text = FractionProcessor.to_text(whole, numerator, denominator, language)
+
+        if language == "fa":
+            return [(text, unit.fa)]
+
+        # A part of one unit stays singular ("one half kilogram").
+        is_singular = whole is None and numerator <= denominator
+        return [(text, unit.en if is_singular else unit.en_plural)]

rules/wrappers.py

@@ -0,0 +1,9 @@
+"""Characters that can wrap a number, symbol or unit: `(kg)`, `[V]`, `{m}`, `"m"`, `«V»`."""
+import re
+
+OPEN_CHARS = "([{\"“«‘'"
+CLOSE_CHARS = ")]}\"”»’'"
+
+OPENERS = "[" + re.escape(OPEN_CHARS) + "]"
+CLOSERS = "[" + re.escape(CLOSE_CHARS) + "]"
+WRAPPERS = "[" + re.escape(OPEN_CHARS + CLOSE_CHARS) + "]"