Spaces:
Sleeping
Sleeping
| import random | |
| import re | |
| from dataclasses import dataclass | |
| from decimal import Decimal | |
| from num2fawords import words | |
| from num2words import num2words | |
| from persiantools import digits | |
| from rules.base import BaseProcessor | |
| from rules.fraction import FRACTION_BODY, VULGAR_FRACTIONS, FractionProcessor | |
| from rules.wrappers import CLOSERS, OPENERS | |
| from utils import NO_LETTERS, detect_digit_language, detect_language | |
| class Unit: | |
| fa: str | |
| en: str | |
| en_plural: str | |
| # Next smaller unit and how many of it make one of this unit, used to read | |
| # `1000.5 m` as "1000 metres and 50 centimetres". | |
| minor: "Unit" = None | |
| factor: int = 1 | |
| MILLIMETRE = Unit("میلیمتر", "millimetre", "millimetres") | |
| CENTIMETRE = Unit("سانتیمتر", "centimetre", "centimetres", MILLIMETRE, 10) | |
| METRE = Unit("متر", "metre", "metres", CENTIMETRE, 100) | |
| KILOMETRE = Unit("کیلومتر", "kilometre", "kilometres", METRE, 1000) | |
| MILE = Unit("مایل", "mile", "miles") | |
| FOOT = Unit("فوت", "foot", "feet") | |
| MILLIGRAM = Unit("میلی‌گرم", "milligram", "milligrams") | |
| GRAM = Unit("گرم", "gram", "grams", MILLIGRAM, 1000) | |
| KILOGRAM = Unit("کیلوگرم", "kilogram", "kilograms", GRAM, 1000) | |
| POUND = Unit("پوند", "pound", "pounds") | |
| OUNCE = Unit("اونس", "ounce", "ounces") | |
| MILLILITRE = Unit("میلی‌لیتر", "millilitre", "millilitres") | |
| LITRE = Unit("لیتر", "litre", "litres", MILLILITRE, 1000) | |
| MILLISECOND = Unit("میلی‌ثانیه", "millisecond", "milliseconds") | |
| SECOND = Unit("ثانیه", "second", "seconds", MILLISECOND, 1000) | |
| MINUTE = Unit("دقیقه", "minute", "minutes", SECOND, 60) | |
| HOUR = Unit("ساعت", "hour", "hours", MINUTE, 60) | |
| DAY = Unit("روز", "day", "days", HOUR, 24) | |
| WEEK = Unit("هفته", "week", "weeks", DAY, 7) | |
| MONTH = Unit("ماه", "month", "months") | |
| YEAR = Unit("سال", "year", "years") | |
| PERCENT = Unit("درصد", "percent", "percent") | |
| KILOMETRE_PER_HOUR = Unit("کیلومتر بر ساعت", "kilometre per hour", "kilometres per hour") | |
| METRE_PER_SECOND = Unit("متر بر ثانیه", "metre per second", "metres per second") | |
| CELSIUS = Unit("درجه سانتیگراد", "degree Celsius", "degrees Celsius") | |
| FAHRENHEIT = Unit("درجه فارنهایت", "degree Fahrenheit", "degrees Fahrenheit") | |
| # Symbols are case-sensitive on purpose (e.g. `5G` is a network, not grams). | |
| # Single `s` and `d` are left out on purpose: `1990s` and `3d` are not units. | |
| UNITS = { | |
| "km": KILOMETRE, | |
| "m": METRE, | |
| "cm": CENTIMETRE, | |
| "mm": MILLIMETRE, | |
| "mi": MILE, | |
| "ft": FOOT, | |
| "kg": KILOGRAM, | |
| "KG": KILOGRAM, | |
| "Kg": KILOGRAM, | |
| "g": GRAM, | |
| "mg": MILLIGRAM, | |
| "lb": POUND, | |
| "lbs": POUND, | |
| "oz": OUNCE, | |
| "l": LITRE, | |
| "L": LITRE, | |
| "ml": MILLILITRE, | |
| "mL": MILLILITRE, | |
| "ms": MILLISECOND, | |
| "sec": SECOND, | |
| "secs": SECOND, | |
| "second": SECOND, | |
| "seconds": SECOND, | |
| "min": MINUTE, | |
| "mins": MINUTE, | |
| "minute": MINUTE, | |
| "minutes": MINUTE, | |
| "h": HOUR, | |
| "hr": HOUR, | |
| "hrs": HOUR, | |
| "hour": HOUR, | |
| "hours": HOUR, | |
| "day": DAY, | |
| "days": DAY, | |
| "week": WEEK, | |
| "weeks": WEEK, | |
| "month": MONTH, | |
| "months": MONTH, | |
| "year": YEAR, | |
| "years": YEAR, | |
| "%": PERCENT, | |
| "٪": PERCENT, | |
| "km/h": KILOMETRE_PER_HOUR, | |
| "m/s": METRE_PER_SECOND, | |
| "°C": CELSIUS, | |
| "°F": FAHRENHEIT, | |
| } | |
| AMOUNT = r"\d+(?:[.,٫٬]\d+)*" | |
| # Longest symbols first so `km/h` wins over `km`, and `mg` over `m`. | |
| SYMBOLS = "|".join(re.escape(symbol) for symbol in sorted(UNITS, key=len, reverse=True)) | |
| PERCENT_SIGNS = ("%", "٪") | |
| OTHER_SYMBOLS = "|".join( | |
| re.escape(symbol) | |
| for symbol in sorted(UNITS, key=len, reverse=True) | |
| if symbol not in PERCENT_SIGNS | |
| ) | |
| # A number or fraction, optional spaces, then a unit symbol: `5km`, `۵ km`, | |
| # `2.5 kg`, `1/2 kg`, `2 1/2 kg`. The symbol must stand alone, so `5 km²` | |
| # or `5G` don't match unless they are in the table. | |
| # The amount and/or the symbol may be wrapped: `2 (kg)`, `3 {m}`, `(5) kg`. | |
| # After a percent sign a unit symbol may follow (`5% kg`): that is `per_unit`. | |
| UNIT_PATTERN = re.compile( | |
| rf""" | |
| (?<![\w./,٫٬]) | |
| (?:(?P<aopen>{OPENERS})[^\S\n]*)? | |
| (?:{FRACTION_BODY}|(?P<amount>{AMOUNT})) | |
| (?(aopen)[^\S\n]*(?P<aclose>{CLOSERS})) | |
| [^\S\n]* | |
| (?:(?P<sopen>{OPENERS})[^\S\n]*)? | |
| (?P<symbol>{SYMBOLS}) | |
| (?(sopen)[^\S\n]*(?P<sclose>{CLOSERS})) | |
| (?![\w/]) | |
| (?P<tail>[^\S\n]+(?P<per_unit>{OTHER_SYMBOLS})(?![\w/]))? | |
| """, | |
| re.VERBOSE, | |
| ) | |
| # A percent sign typed before the number (`%20`, `% 20`, `٪ ۲۰`). It is moved | |
| # after the number so the normal pattern can read it. `%d`, `%s`, `% 5d` and | |
| # `a%20b` don't match. | |
| PREFIX_PERCENT_PATTERN = re.compile( | |
| rf"(?<![\w%٪])(?P<sign>[%٪])[^\S\n]*(?P<amount>{AMOUNT})(?![\w%٪/])" | |
| ) | |
| class UnitProcessor(BaseProcessor): | |
| """Converts a number followed by a unit symbol into spoken text. | |
| Persian: `5 km` -> `پنج کیلومتر` | |
| `۷۵kg` -> `هفتاد و پنج کیلوگرم` | |
| English: `5 km` -> `five kilometres` | |
| `1 kg` -> `one kilogram` | |
| `25°C` -> `twenty-five degrees Celsius` | |
| `2 %` -> `two percent` (Persian: `۲ ٪` -> `دو درصد`) | |
| `10 sec` -> `ten seconds` | |
| Decimals are split into smaller units when the unit has one: | |
| `1000.5 m` -> `یک هزار متر و پنجاه سانتیمتر` | |
| `2.5 kg` -> `two kilograms and five hundred grams` | |
| `1.5 h` -> `one hour and thirty minutes` | |
| `1/2 %` -> `one half percent` | |
| Other decimals are read as a plain number: `2.5 mi` -> `two point five miles`. | |
| A percent sign may come before the number (`% 5`, `٪ ۵`), and a unit symbol | |
| after a percent sign is read on its own: `5% kg` -> `five percent kilogram`. | |
| With nothing but units in the text, Persian digits give Persian | |
| (`۱۰۰۰m` -> `یک هزار متر`) and ASCII digits give English. | |
| Works on the whole text (not word by word), so both `5km` and `5 km` | |
| are handled. Add a unit by adding one line to `UNITS`. | |
| """ | |
| def __init__(self, epsilon: float = 1.0): | |
| super(UnitProcessor, self).__init__(epsilon=epsilon) | |
| def process(self, text: str, language: str = None, *args, **kwargs) -> str: | |
| if random.random() > self.epsilon: | |
| return text | |
| language = self._resolve_language(text, language) | |
| if language is None: | |
| return text | |
| def convert(match: re.Match) -> str: | |
| return self._convert(match, language) | |
| text = PREFIX_PERCENT_PATTERN.sub(self._move_prefix_percent, text) | |
| return UNIT_PATTERN.sub(convert, text) | |
| def _move_prefix_percent(match: re.Match) -> str: | |
| """`% 20 cm` -> `20% cm`, so it reads as `twenty percent centimetre`.""" | |
| before = match.string[: match.start()].rstrip(" \t") | |
| # In `5 % 2` the sign belongs to the 5 before it, not to the 2 after it. | |
| if before and (before[-1].isdigit() or before[-1] in VULGAR_FRACTIONS): | |
| return match.group(0) | |
| return f"{match.group('amount')}{match.group('sign')}" | |
| def _resolve_language(text: str, language: str): | |
| """Detects the language ignoring the unit symbols themselves. | |
| Symbols like `km` are Latin letters and would otherwise tip a Persian | |
| sentence towards English. If nothing but units is left (e.g. `۱۰۰۰m`), | |
| Persian digits mean Persian; otherwise the given `language` is used, | |
| and with no language given ASCII digits mean English. | |
| """ | |
| detected = detect_language(UNIT_PATTERN.sub(" ", text)) | |
| if detected in ("fa", "en"): | |
| return detected | |
| if detected != NO_LETTERS: | |
| return None | |
| digit_language = detect_digit_language(text) | |
| if digit_language == "fa": | |
| return "fa" | |
| if language in ("fa", "en"): | |
| return language | |
| if language is None or language == NO_LETTERS: | |
| return digit_language | |
| return None | |
| def _convert(match: re.Match, language: str) -> str: | |
| unit = UNITS[match.group("symbol")] | |
| pieces = UnitProcessor._amount_pieces(match, unit, language) | |
| if pieces is None: | |
| return match.group(0) | |
| # Wrappers stay where they were: `2 (kg)` -> `two (kilograms)`. | |
| rendered = [] | |
| for index, (amount_text, unit_text) in enumerate(pieces): | |
| if index == 0 and match.group("aopen"): | |
| amount_text = f"{match.group('aopen')}{amount_text}{match.group('aclose')}" | |
| if match.group("sopen"): | |
| unit_text = f"{match.group('sopen')}{unit_text}{match.group('sclose')}" | |
| rendered.append(f"{amount_text} {unit_text}") | |
| if language == "fa" or len(rendered) == 1: | |
| text = " و ".join(rendered) if language == "fa" else rendered[0] | |
| else: | |
| text = f"{', '.join(rendered[:-1])} and {rendered[-1]}" | |
| per_unit = match.group("per_unit") | |
| if per_unit is None: | |
| return text | |
| if unit is PERCENT: | |
| # `5% kg` -> `five percent kilogram`: the unit has no number of its own. | |
| other = UNITS[per_unit] | |
| return f"{text} {other.fa if language == 'fa' else other.en}" | |
| # A symbol after any other unit is not ours to convert: put it back. | |
| return text + match.group("tail") | |
| def _amount_pieces(match: re.Match, unit: Unit, language: str): | |
| """Reads the amount as [(amount words, unit words), ...]; None if it | |
| should stay unchanged. Several pieces mean a split decimal (`2.5 kg`).""" | |
| if match.group("amount") is None: | |
| return UnitProcessor._fraction_pieces(match, unit, language) | |
| amount = digits.fa_to_en(match.group("amount")) | |
| clean = amount.replace(",", "").replace("٬", "").replace("٫", ".") | |
| parts = UnitProcessor._split_into_units(Decimal(clean), unit) | |
| if language == "fa": | |
| if parts is None: | |
| return [(words(float(clean)), unit.fa)] | |
| return [(words(number), part.fa) for number, part in parts] | |
| if parts is None: | |
| return [(num2words(Decimal(clean)), unit.en_plural)] | |
| return [ | |
| (num2words(number), part.en if number == 1 else part.en_plural) | |
| for number, part in parts | |
| ] | |
| def _split_into_units(value: Decimal, unit: Unit): | |
| """Splits a decimal amount into whole numbers of smaller units. | |
| `1000.5 m` -> [(1000, metre), (50, centimetre)]. Returns None when the | |
| fraction does not end exactly on the smallest known unit (or the unit | |
| has no smaller unit), so the caller reads it as a plain decimal. | |
| """ | |
| parts = [] | |
| while True: | |
| whole = int(value) | |
| fraction = value - whole | |
| if whole: | |
| parts.append((whole, unit)) | |
| if fraction == 0: | |
| return parts or [(0, unit)] | |
| if unit.minor is None: | |
| return None | |
| value = fraction * unit.factor | |
| unit = unit.minor | |
| def _fraction_pieces(match: re.Match, unit: Unit, language: str): | |
| """`1/2 kg` -> `one half` + `kilogram`, `1,000/2 kg` -> `one thousand halves` + `kilograms`.""" | |
| parsed = FractionProcessor.parse(match) | |
| if parsed is None: | |
| return None | |
| whole, numerator, denominator = parsed | |
| text = FractionProcessor.to_text(whole, numerator, denominator, language) | |
| if language == "fa": | |
| return [(text, unit.fa)] | |
| # A part of one unit stays singular ("one half kilogram"). | |
| is_singular = whole is None and numerator <= denominator | |
| return [(text, unit.en if is_singular else unit.en_plural)] |