diff --git "a/tests.py" "b/tests.py" new file mode 100644--- /dev/null +++ "b/tests.py" @@ -0,0 +1,2182 @@ +import unittest +from rules.dates import ( + DateProcessor +) +from rules.times import TimeProcessor +from rules.currency import CurrencyProcessor +from rules.spaces import CurrencySpaceProcessor +from rules.units import UnitProcessor +from rules.fraction import FractionProcessor +from rules.phone import PhoneNumberProcessor +from rules.cards import CardNumberProcessor +from rules.postal_national_id import PostalCodeAndNationalIdProcessor +from rules.symbols import ( + AmpersandProcessor, + MathOperatorProcessor, + NegativeSignProcessor, + RangeProcessor, + RomanNumeralProcessor, + roman_to_int, +) +from normalizer import Normalizer + + +class MyTestCase(unittest.TestCase): + def setUp(self): + self.date_processor = DateProcessor() + self.time_processor = TimeProcessor() + self.normalizer = Normalizer() + + def test_not_a_date(self): + inputs = ['1380', 'hello', '21 21 21', '12, 8, 20', "Sep 28 2020"] + for inp in inputs: + self.assertFalse(self.date_processor.is_date(inp), f'{inp} is not a date') + + def test_date_to_text_gregorian(self): + inputs = ["9-28-2020", "1/1/1998", "2002/07/18"] + corrects = ["twenty-eighth of September, twenty twenty".lower(), + "first of January, nineteen ninety-eight".lower(), + "eighteenth of July, two thousand and two".lower()] + normalized = [] + + for i, c in zip(inputs, corrects): + text = self.date_processor.process(i, language='en') + normalized.append(text.lower()) + + self.assertListEqual(normalized, corrects) + + + def test_date_to_text_jalali(self): + inputs = ['1381/4/31', '۱۳۸۱/۰۴/27', "29/12/1305"] + corrects = ["سی و یک چهار یک هزار و سیصد و هشتاد و یک", + "بیست و هفت چهار یک هزار و سیصد و هشتاد و یک", + "بیست و نه دوازده یک هزار و سیصد و پنج"] + normalized = [] + + for i, c in zip(inputs, corrects): + text = self.date_processor.process(i, language='fa') + normalized.append(text.strip()) + + self.assertListEqual(normalized, corrects) + + def test_englih_time(self): + inputs = ['1:35', '5:50', '15:20', '10:00:11', '3:12:01'] + corrects = [ + 'one thirty-five', + 'five fifty', + 'three twenty pm', + 'ten and eleven seconds', + 'three twelve and one second'] + + normalized = [] + for i, c in zip(inputs, corrects): + text = self.time_processor.process(i, language='en') + normalized.append(text.lower()) + + self.assertListEqual(normalized, corrects) + + def test_farsi_time(self): + inputs = ['1:35', '5:50', '15:20', '10:00:11', '3:12:01'] + corrects = [ + 'یک و سی و پنج دقیقه', + 'پنج و پنجاه دقیقه', + 'پانزده و بیست دقیقه', + 'ده و یازده ثانیه', + 'سه و دوازده دقیقه و یک ثانیه'] + + normalized = [] + for i, c in zip(inputs, corrects): + text = self.time_processor.process(i, language='fa') + normalized.append(text.lower()) + + self.assertListEqual(normalized, corrects) + + def test_overall_normalization(self): + inputs = [ + "I was born on 2002/07/18 when it was pouring.", + "من متولد سال ۱۳۸۱/۴/۲۷ هستم و الان بیست و چهار سال دارم.", + "I have a date at 3:30 on 9/10/2026 and I have to be ready.", + "در تاریخ ۱۲/۶/۱۳۸۸ و در ساعت ۲:۵۱:۰۳ اتفاق افتاد." + ] + correct = [ + "I was born on eighteenth of July, two thousand and two when it was pouring.".lower(), + "من متولد سال بیست و هفت چهار یک هزار و سیصد و هشتاد و یک هستم و الان بیست و چهار سال دارم.", + "I have a date at three thirty on tenth of September, twenty twenty-six and I have to be ready.".lower(), + "در تاریخ دوازده شش یک هزار و سیصد و هشتاد و هشت و در ساعت دو و پنجاه و یک دقیقه و سه ثانیه اتفاق افتاد." + ] + normalized = [] + + for i, c in zip(inputs, correct): + text = self.normalizer.process(i) + normalized.append(text.lower()) + self.assertListEqual(normalized, correct) + + def test_overall_normalization_currency(self): + inputs = [ + "قیمت امروز ارز، ۳$ بود", + "قیمت امروز ارز، 3$ بود", + "I have 3$", + "I have 1 $", + "قیمت امروز ارز، ۳ $ بود", + "I want $12.50 today.", + "من ۱۲.۵۰$ می‌خواهم", + "It costs €12.50 or 10 £.", + "قیمت ۵ € و £۲.۵ است", + "۱۰۰۰$", + "1000$", + "۱۰۰۰ $", + "1000 $", + ] + correct = [ + "قیمت امروز ارز، سه دلار بود", + "قیمت امروز ارز، سه دلار بود", + "I have three dollars", + "I have one dollar", + "قیمت امروز ارز، سه دلار بود", + "I want twelve dollars and fifty cents today.", + "من دوازده دلار و پنجاه سنت می‌خواهم", + "It costs twelve euros and fifty cents or ten pounds.", + "قیمت پنج یورو و دو پوند و پنجاه پنس است", + "یک هزار دلار", + "one thousand dollars", + "یک هزار دلار", + "one thousand dollars", + ] + normalized = [self.normalizer.process(i) for i in inputs] + self.assertListEqual(normalized, correct) + + + def test_overall_normalization_units(self): + inputs = [ + "I ran 5km today", + "I ran 5 km today", + "وزن او ۷۵ kg است", + "من ۱۰km دویدم", + "It is 25°C outside", + "دما ۲۵ °C است", + "The car goes 60 km/h and weighs 1 kg", + "از اینجا تا آنجا ۱۰۰۰.۵m فاصله است", + "The movie is 1.5 hours long", + "این کار ۱۰ min طول می‌کشد", + "۱۰۰۰m", + "1000m", + ] + correct = [ + "I ran five kilometres today", + "I ran five kilometres today", + "وزن او هفتاد و پنج کیلوگرم است", + "من ده کیلومتر دویدم", + "It is twenty-five degrees Celsius outside", + "دما بیست و پنج درجه سانتیگراد است", + "The car goes sixty kilometres per hour and weighs one kilogram", + "از اینجا تا آنجا یک هزار متر و پنجاه سانتیمتر فاصله است", + "The movie is one hour and thirty minutes long", + "این کار ده دقیقه طول می‌کشد", + "یک هزار متر", + "one thousand metres", + ] + normalized = [self.normalizer.process(i) for i in inputs] + self.assertListEqual(normalized, correct) + + def test_overall_normalization_units_and_currency_together(self): + inputs = [ + "It costs 3$ per 2 kg", + "قیمت هر ۲ kg برابر ۳$ است", + ] + correct = [ + "It costs three dollars per two kilograms", + "قیمت هر دو کیلوگرم برابر سه دلار است", + ] + normalized = [self.normalizer.process(i) for i in inputs] + self.assertListEqual(normalized, correct) + + + def test_overall_normalization_fractions(self): + inputs = [ + "I drank 1/3 of the bottle", + "من ۱/۳ بطری نوشیدم", + "Add 2 1/2 cups of flour", + "۱/۳", + "1/3", + "من ۵km و ۱/۳ بطری", + "۱/۲ kg", + "1,000/2 kg", + "Add 1/2 kg of rice", + "۲ ۱/۲ kg برنج", + "2 %", + "۲ ٪", + "Discount is 20% on 3$", + "تخفیف ۲۰٪ روی ۳$ است", + "از ۱/۲ kg برنج ۲.۵٪ باقی ماند", + "٪ ۵", + "% 5", + "5% kg", + "۵٪ kg", + "% 20 cm", + "٪ ۲۰ cm", + ] + correct = [ + "I drank one third of the bottle", + "من یک سوم بطری نوشیدم", + "Add two and one half cups of flour", + "یک سوم", + "one third", + "من پنج کیلومتر و یک سوم بطری", + "یک دوم کیلوگرم", + "one thousand halves kilograms", + "Add one half kilogram of rice", + "دو و یک دوم کیلوگرم برنج", + "two percent", + "دو درصد", + "Discount is twenty percent on three dollars", + "تخفیف بیست درصد روی سه دلار است", + "از یک دوم کیلوگرم برنج دو و پنج دهم درصد باقی ماند", + "پنج درصد", + "five percent", + "five percent kilogram", + "پنج درصد کیلوگرم", + "twenty percent centimetre", + "بیست درصد سانتیمتر", + ] + normalized = [self.normalizer.process(i) for i in inputs] + self.assertListEqual(normalized, correct) + + + def test_overall_normalization_roman_numerals_and_ampersand(self): + inputs = [ + "Henry VIII was king", + "Tom & Jerry read Chapter IV", + "قرن XX و R&D", + "تام & جری فصل IV را خواندند", + "جنگ جهانی II", + "Q&A about World War I", + "I think Washington DC is nice", + "3 & 4", + "۳ & ۴", + "1/2 & 1/3", + "۱/۲ & ۱/۳", + "5 km & 3 kg", + "۵ km & ۳ kg", + "5 & 3 kg", + "5% & 10%", + ] + correct = [ + "Henry eight was king", + "Tom and Jerry read Chapter four", + "قرن بیست و R و D", + "تام و جری فصل چهار را خواندند", + "جنگ جهانی دو", + "Q and A about World War one", + "I think Washington DC is nice", + "three and four", + "سه و چهار", + "one half and one third", + "یک دوم و یک سوم", + "five kilometres and three kilograms", + "پنج کیلومتر و سه کیلوگرم", + "five and three kilograms", + "five percent and ten percent", + ] + normalized = [self.normalizer.process(i) for i in inputs] + self.assertListEqual(normalized, correct) + + + def test_overall_normalization_negative_sign_and_operators(self): + inputs = [ + "-2 m", + "- 2 m", + "-۲ m", + "-2", + "-۲", + "The temp is -2 degrees", + "دما -۲ درجه است", + "-1/2 kg", + "-5%", + "-$5", + "x = -2", + "3 + 4", + "۳ + ۴", + "3+4", + "6 × 2", + "۶ × ۲", + "8 ÷ 2", + "۸ ÷ ۲", + "7 = 7", + "۷ = ۷", + "3 + 4 = 7", + "۳ + ۴ = ۷", + "5 - 3 = 2", + "۵ - ۳ = ۲", + "2 m + 3 m", + "1/2 + 1/3", + "$3 + $4", + "3 & 4 = 7", + "5 - 3", + "10 - 20 km", + "C++ is great", + ] + correct = [ + "negative two metres", + "negative two metres", + "منفی دو متر", + "negative two", + "منفی دو", + "The temp is negative two degrees", + "دما منفی دو درجه است", + "negative one half kilogram", + "negative five percent", + "negative five dollars", + "x equals negative two", + "three plus four", + "سه بعلاوه چهار", + "three plus four", + "six times two", + "شش ضربدر دو", + "eight divided by two", + "هشت تقسیم بر دو", + "seven equals seven", + "هفت مساوی است با هفت", + "three plus four equals seven", + "سه بعلاوه چهار مساوی است با هفت", + "five minus three equals two", + "پنج منهای سه مساوی است با دو", + "two metres plus three metres", + "one half plus one third", + "three dollars plus four dollars", + "three and four equals seven", + "five to three", + "ten to twenty kilometres", + "C++ is great", + ] + for inp, expected in zip(inputs, correct): + self.assertEqual(self.normalizer.process(inp), expected, inp) + + + def test_overall_normalization_ranges(self): + inputs = [ + "3 - 5", + "3-5", + "3 – 5", + "3–5", + "pages 3-5", + "2.5 - 3.5", + "3kg - 5 kg", + "3 kg-5 kg", + "3 - 5 kg", + "3 kg - 5", + "10 - 20 km", + "$3 - $5", + "3$ - 5$", + "1/2 - 3/4", + "5% - 10%", + "10 - 20%", + "1 - 2 - 3", + "Is it 3 - 5?", + "۳ - ۵", + "۳-۵", + "۳kg - ۵ kg", + "۱۰ - ۲۰ km", + "از ۳ - ۵ ساعت", + "۵٪ - ۱۰٪", + "5 - 3 = 2", + "2024-05", + "0912-123-4567", + "1990-2000", + "well - maybe", + ] + correct = [ + "three to five", + "three to five", + "three to five", + "three to five", + "pages three to five", + "two point five to three point five", + "three kilograms to five kilograms", + "three kilograms to five kilograms", + "three to five kilograms", + "three kilograms to five", + "ten to twenty kilometres", + "three dollars to five dollars", + "three dollars to five dollars", + "one half to three quarters", + "five percent to ten percent", + "ten to twenty percent", + "one to two to three", + "Is it three to five?", + "سه تا پنج", + "سه تا پنج", + "سه کیلوگرم تا پنج کیلوگرم", + "ده تا بیست کیلومتر", + "از سه تا پنج ساعت", + "پنج درصد تا ده درصد", + "five minus three equals two", + "2024-05", + "zero nine one two, one two three, four five, six seven", + "1990-2000", + "well - maybe", + ] + for inp, expected in zip(inputs, correct): + self.assertEqual(self.normalizer.process(inp), expected, inp) + + + def test_overall_normalization_wrapped_symbols(self): + cases = { + "2 (kg)": "two (kilograms)", + "2(kg)": "two (kilograms)", + "2 [kg]": "two [kilograms]", + "3 {m}": "three {metres}", + '3 "m"': 'three "metres"', + "3 “m”": "three “metres”", + "(2 kg)": "(two kilograms)", + "(5) kg": "(five) kilograms", + "(1/2) kg": "(one half) kilogram", + "50 (%)": "fifty (percent)", + "2 (km/h)": "two (kilometres per hour)", + "2 (kg) and 3 (m)": "two (kilograms) and three (metres)", + "-2 (m)": "negative two (metres)", + "chapter (V)": "chapter (five)", + "Chapter [V]": "Chapter [five]", + "Part {I}": "Part {one}", + 'Chapter "IV"': 'Chapter "four"', + "Henry (VIII)": "Henry (eight)", + "I think (I) am": "I think (I) am", + "Tom (&) Jerry": "Tom (and) Jerry", + "3 (&) 4": "three (and) four", + "(3) & (4)": "(three) and (four)", + "(3) + (4)": "(three) plus (four)", + "3 (+) 4": "three (plus) four", + "(3) - (5)": "(three) to (five)", + "3 (cm) - 5 (cm)": "three (centimetres) to five (centimetres)", + "(2) kg - (5) kg": "(two) kilograms to (five) kilograms", + "2 (kg) + 3 (kg)": "two (kilograms) plus three (kilograms)", + "3 (a)": "3 (a)", + "Section 2 (b)": "Section 2 (b)", + "۲ (kg)": "دو (کیلوگرم)", + "۳ {m}": "سه {متر}", + "۲ (kg) و ۳ (m)": "دو (کیلوگرم) و سه (متر)", + "فصل (V)": "فصل (پنج)", + "فصل «V»": "فصل «پنج»", + "(۳) + (۴)": "(سه) بعلاوه (چهار)", + "(۳) - (۵)": "(سه) تا (پنج)", + "۳ (+) ۴": "سه (بعلاوه) چهار", + "۳ (kg) - ۵ (kg)": "سه (کیلوگرم) تا پنج (کیلوگرم)", + } + for inp, expected in cases.items(): + self.assertEqual(self.normalizer.process(inp), expected, inp) + + + def test_overall_normalization_phone_numbers(self): + mobile_en = "zero nine one two, four three three, eight one, three five" + mobile_plus_en = "plus nine eight, nine one two, four three three, eight one, three five" + landline_en = "zero two one, eight eight, zero six, nine eight, nine five" + mobile_fa = "صفر نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج" + mobile_plus_fa = "مثبت نود و هشت، نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج" + landline_fa = "صفر بیست و یک، هشتاد و هشت، صفر شش، نود و هشت، نود و پنج" + cases = { + "09124338135": mobile_en, + "+989124338135": mobile_plus_en, + "+98-912-4338135": mobile_plus_en, + "+98 912 433 8135": mobile_plus_en, + "021-88069895": landline_en, + "(021) 88069895": "(zero two one), eight eight, zero six, nine eight, nine five", + "(0915)6486320": "(zero nine one five), six four eight, six three, two zero", + "Call me at 09124338135.": f"Call me at {mobile_en}.", + "Call 09124338135, then 021-88069895.": f"Call {mobile_en}, then {landline_en}.", + "tel: 88069895": "tel: eight eight, zero six, nine eight, nine five", + "۰۹۱۲۴۳۳۸۱۳۵": mobile_fa, + "+۹۸-۹۱۲-۴۳۳۸۱۳۵": mobile_plus_fa, + "۰۲۱-۸۸۰۶۹۸۹۵": landline_fa, + "شماره من ۰۹۱۲۴۳۳۸۱۳۵ است": f"شماره من {mobile_fa} است", + "۰۰۱-۸۸۵۲۳۰۰۶": "دو صفر یک، هشتاد و هشت، پنجاه و دو، سی، صفر شش", + "با 09124338135 تماس بگیرید": f"با {mobile_fa} تماس بگیرید", + "تلفن: ۸۸۰۶۹۸۹۵": "تلفن: هشتاد و هشت، صفر شش، نود و هشت، نود و پنج", + "شماره تماس ۰۲۱-۸۸۰۶۹۸۹۵ و ۰۹۱۲۴۳۳۸۱۳۵": f"شماره تماس {landline_fa} و {mobile_fa}", + "4338135": "4338135", + "call 5 times": "call 5 times", + } + for inp, expected in cases.items(): + self.assertEqual(self.normalizer.process(inp), expected, inp) + + +class CurrencyProcessorTestCase(unittest.TestCase): + def setUp(self): + self.processor = CurrencyProcessor() + + def test_is_currency(self): + valid = ['3$', '۳$', '3.5$', '1,000$', '(3$)', '3$.', '$3', '$12.50', '$12.50.', '€3', '3€', '£12.50', '۳£'] + invalid = ['3', '$', 'hello', '3$a', '3 $', '3$$', '$3$', '$$3', '€3£', '3€€'] + for inp in valid: + self.assertTrue(self.processor.is_currency(inp), f'{inp} is a currency') + for inp in invalid: + self.assertFalse(self.processor.is_currency(inp), f'{inp} is not a currency') + + def test_persian(self): + inputs = ['۳$', '3$', '$3', '۱۲۰$', '1,000$'] + corrects = ['سه دلار', 'سه دلار', 'سه دلار', 'یکصد و بیست دلار', 'یک هزار دلار'] + normalized = [self.processor.process(i, language='fa') for i in inputs] + self.assertListEqual(normalized, corrects) + + def test_english(self): + inputs = ['3$', '۳$', '$3', '1$', '0$', '25$', '1,000$'] + corrects = [ + 'three dollars', + 'three dollars', + 'three dollars', + 'one dollar', + 'zero dollars', + 'twenty-five dollars', + 'one thousand dollars', + ] + normalized = [self.processor.process(i, language='en') for i in inputs] + self.assertListEqual(normalized, corrects) + + def test_english_dollars_and_cents(self): + inputs = ['$12.50', '12.5$', '12.50$', '۱۲.۵۰$', '$1.01', '$12.05', '$0.50', '$0.01', '$12.00', '$1,000.75'] + corrects = [ + 'twelve dollars and fifty cents', + 'twelve dollars and fifty cents', + 'twelve dollars and fifty cents', + 'twelve dollars and fifty cents', + 'one dollar and one cent', + 'twelve dollars and five cents', + 'fifty cents', + 'one cent', + 'twelve dollars', + 'one thousand dollars and seventy-five cents', + ] + normalized = [self.processor.process(i, language='en') for i in inputs] + self.assertListEqual(normalized, corrects) + + def test_persian_dollars_and_cents(self): + inputs = ['$12.50', '12.5$', '۱۲٫۵۰$', '$12.05', '$0.50', '$12.00'] + corrects = [ + 'دوازده دلار و پنجاه سنت', + 'دوازده دلار و پنجاه سنت', + 'دوازده دلار و پنجاه سنت', + 'دوازده دلار و پنج سنت', + 'پنجاه سنت', + 'دوازده دلار', + ] + normalized = [self.processor.process(i, language='fa') for i in inputs] + self.assertListEqual(normalized, corrects) + + def test_more_than_two_decimals_is_read_as_decimal_number(self): + self.assertEqual( + self.processor.process('3.141$', language='en'), + 'three point one four one dollars', + ) + + def test_dollars_and_cents_keep_surrounding_punctuation(self): + self.assertEqual( + self.processor.process('($12.50).', language='en'), + '(twelve dollars and fifty cents).', + ) + + + def test_euro_english(self): + inputs = ['3€', '€3', '1€', '€12.50', '12.5€', '€1.01', '€0.50', '€12.00', '۳€'] + corrects = [ + 'three euros', + 'three euros', + 'one euro', + 'twelve euros and fifty cents', + 'twelve euros and fifty cents', + 'one euro and one cent', + 'fifty cents', + 'twelve euros', + 'three euros', + ] + normalized = [self.processor.process(i, language='en') for i in inputs] + self.assertListEqual(normalized, corrects) + + def test_pound_english(self): + inputs = ['3£', '£3', '1£', '£12.50', '12.5£', '£1.01', '£0.50', '£0.01', '£12.00'] + corrects = [ + 'three pounds', + 'three pounds', + 'one pound', + 'twelve pounds and fifty pence', + 'twelve pounds and fifty pence', + 'one pound and one penny', + 'fifty pence', + 'one penny', + 'twelve pounds', + ] + normalized = [self.processor.process(i, language='en') for i in inputs] + self.assertListEqual(normalized, corrects) + + def test_euro_persian(self): + inputs = ['3€', '€3', '۳€', '€12.50', '€0.50', '€12.00'] + corrects = [ + 'سه یورو', + 'سه یورو', + 'سه یورو', + 'دوازده یورو و پنجاه سنت', + 'پنجاه سنت', + 'دوازده یورو', + ] + normalized = [self.processor.process(i, language='fa') for i in inputs] + self.assertListEqual(normalized, corrects) + + def test_pound_persian(self): + inputs = ['3£', '£3', '۳£', '£12.50', '£12.05', '£0.50', '£12.00'] + corrects = [ + 'سه پوند', + 'سه پوند', + 'سه پوند', + 'دوازده پوند و پنجاه پنس', + 'دوازده پوند و پنج پنس', + 'پنجاه پنس', + 'دوازده پوند', + ] + normalized = [self.processor.process(i, language='fa') for i in inputs] + self.assertListEqual(normalized, corrects) + + def test_more_than_two_decimals_uses_currency_name(self): + self.assertEqual(self.processor.process('3.141€', language='en'), 'three point one four one euros') + self.assertEqual(self.processor.process('3.141£', language='en'), 'three point one four one pounds') + + def test_punctuation_is_preserved_for_all_symbols(self): + self.assertEqual(self.processor.process('(€12.50).', language='en'), '(twelve euros and fifty cents).') + self.assertEqual(self.processor.process('£3،', language='fa'), 'سه پوند،') + + + def test_surrounding_punctuation_is_preserved(self): + self.assertEqual(self.processor.process('3$.', language='en'), 'three dollars.') + self.assertEqual(self.processor.process('(3$)', language='en'), '(three dollars)') + self.assertEqual(self.processor.process('۳$،', language='fa'), 'سه دلار،') + + def test_non_currency_is_unchanged(self): + inputs = ['hello', '3', '$', 'سلام', '3$a', '$3a'] + for inp in inputs: + self.assertEqual(self.processor.process(inp, language='en'), inp) + self.assertEqual(self.processor.process(inp, language='fa'), inp) + + def test_unsupported_language_is_unchanged(self): + for language in ('mixed', 'de'): + self.assertEqual(self.processor.process('3$', language=language), '3$') + + def test_no_letters_language_is_decided_by_digits(self): + no_letters = 'Unknown / No letters found' + inputs = ['۱۰۰۰$', '1000$', '$۱۰۰۰', '$1000', '۱۲.۵۰€', '12.50€', '£۳', '£3'] + corrects = [ + 'یک هزار دلار', + 'one thousand dollars', + 'یک هزار دلار', + 'one thousand dollars', + 'دوازده یورو و پنجاه سنت', + 'twelve euros and fifty cents', + 'سه پوند', + 'three pounds', + ] + normalized = [self.processor.process(i, language=no_letters) for i in inputs] + self.assertListEqual(normalized, corrects) + + def test_epsilon_zero_disables_processor(self): + processor = CurrencyProcessor(epsilon=0.0) + outputs = {processor.process('3$', language='en') for _ in range(50)} + self.assertEqual(outputs, {'3$'}) + + +class CurrencySpaceProcessorTestCase(unittest.TestCase): + def setUp(self): + self.processor = CurrencySpaceProcessor() + + def test_removes_space_between_number_and_dollar(self): + inputs = ['I have 1 $', 'I have ۳ $', 'it is 2.5 $.', '1,000 $ total', '5 €', '۳ £', '2.5 €.'] + corrects = ['I have 1$', 'I have ۳$', 'it is 2.5$.', '1,000$ total', '5€', '۳£', '2.5€.'] + normalized = [self.processor.process(i) for i in inputs] + self.assertListEqual(normalized, corrects) + + def test_already_adjacent_is_unchanged(self): + self.assertEqual(self.processor.process('I have 1$'), 'I have 1$') + + def test_unrelated_dollar_signs_are_unchanged(self): + inputs = ['1 $5', 'price $ 5', 'hello $', 'a1 $', '1 $a', '1\n$', '1 €5', 'price £ 5', '1 £a'] + for inp in inputs: + self.assertEqual(self.processor.process(inp), inp) + + def test_multiple_occurrences(self): + self.assertEqual(self.processor.process('1 $ and 2 $'), '1$ and 2$') + + def test_epsilon_zero_disables_processor(self): + processor = CurrencySpaceProcessor(epsilon=0.0) + outputs = {processor.process('1 $') for _ in range(50)} + self.assertEqual(outputs, {'1 $'}) + + +class UnitProcessorTestCase(unittest.TestCase): + def setUp(self): + self.processor = UnitProcessor() + + def test_length_english(self): + inputs = ['5km', '5 km', '1 km', '2.5 km', '10 m', '3 cm', '7 mm', '1 mi', '6 ft', '1 ft'] + corrects = [ + 'five kilometres', + 'five kilometres', + 'one kilometre', + 'two kilometres and five hundred metres', + 'ten metres', + 'three centimetres', + 'seven millimetres', + 'one mile', + 'six feet', + 'one foot', + ] + normalized = [self.processor.process(i, language='en') for i in inputs] + self.assertListEqual(normalized, corrects) + + def test_length_persian(self): + inputs = ['5km', '۵ km', '1 km', '10 m', '۳ cm', '7 mm', '2 mi'] + corrects = [ + 'پنج کیلومتر', + 'پنج کیلومتر', + 'یک کیلومتر', + 'ده متر', + 'سه سانتیمتر', + 'هفت میلیمتر', + 'دو مایل', + ] + normalized = [self.processor.process(i, language='fa') for i in inputs] + self.assertListEqual(normalized, corrects) + + def test_mass_and_volume_english(self): + inputs = ['2 kg', '1 kg', '2 KG', '500 g', '250 mg', '3 lb', '3 lbs', '8 oz', '2 l', '2 L', '330 ml', '1 mL'] + corrects = [ + 'two kilograms', + 'one kilogram', + 'two kilograms', + 'five hundred grams', + 'two hundred and fifty milligrams', + 'three pounds', + 'three pounds', + 'eight ounces', + 'two litres', + 'two litres', + 'three hundred and thirty millilitres', + 'one millilitre', + ] + normalized = [self.processor.process(i, language='en') for i in inputs] + self.assertListEqual(normalized, corrects) + + def test_mass_and_volume_persian(self): + inputs = ['2 kg', '500 g', '250 mg', '3 lb', '8 oz', '2 l', '330 ml'] + corrects = [ + 'دو کیلوگرم', + 'پانصد گرم', + 'دویست و پنجاه میلی‌گرم', + 'سه پوند', + 'هشت اونس', + 'دو لیتر', + 'سیصد و سی میلی‌لیتر', + ] + normalized = [self.processor.process(i, language='fa') for i in inputs] + self.assertListEqual(normalized, corrects) + + def test_time_english(self): + inputs = ['10 sec', '1 sec', '5 secs', '5 min', '1 min', '3 mins', '2 h', '1 hr', '2 hrs', '2 hours', '1 hour', '250 ms', '3 days', '1 week', '2 years'] + corrects = [ + 'ten seconds', + 'one second', + 'five seconds', + 'five minutes', + 'one minute', + 'three minutes', + 'two hours', + 'one hour', + 'two hours', + 'two hours', + 'one hour', + 'two hundred and fifty milliseconds', + 'three days', + 'one week', + 'two years', + ] + normalized = [self.processor.process(i, language='en') for i in inputs] + self.assertListEqual(normalized, corrects) + + def test_time_persian(self): + inputs = ['10 sec', '۵ min', '2 h', '2 hours', '1 hr', '250 ms', '3 days', '1 week', '2 months', '2 years'] + corrects = [ + 'ده ثانیه', + 'پنج دقیقه', + 'دو ساعت', + 'دو ساعت', + 'یک ساعت', + 'دویست و پنجاه میلی‌ثانیه', + 'سه روز', + 'یک هفته', + 'دو ماه', + 'دو سال', + ] + normalized = [self.processor.process(i, language='fa') for i in inputs] + self.assertListEqual(normalized, corrects) + + def test_time_decimals_are_split_into_smaller_units(self): + cases_en = { + '1.5 h': 'one hour and thirty minutes', + '0.5 h': 'thirty minutes', + '2.5 min': 'two minutes and thirty seconds', + '1.5 sec': 'one second and five hundred milliseconds', + '1.5 days': 'one day and twelve hours', + '1.5 weeks': 'one week, three days and twelve hours', + '1.5 months': 'one point five months', + } + for inp, correct in cases_en.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + cases_fa = { + '1.5 h': 'یک ساعت و سی دقیقه', + '2.5 min': 'دو دقیقه و سی ثانیه', + '1.5 sec': 'یک ثانیه و پانصد میلی‌ثانیه', + } + for inp, correct in cases_fa.items(): + self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) + + def test_speed_and_temperature(self): + self.assertEqual(self.processor.process('60 km/h', language='en'), 'sixty kilometres per hour') + self.assertEqual(self.processor.process('1 m/s', language='en'), 'one metre per second') + self.assertEqual(self.processor.process('60 km/h', language='fa'), 'شصت کیلومتر بر ساعت') + self.assertEqual(self.processor.process('10 m/s', language='fa'), 'ده متر بر ثانیه') + self.assertEqual(self.processor.process('25°C', language='en'), 'twenty-five degrees Celsius') + self.assertEqual(self.processor.process('1 °F', language='en'), 'one degree Fahrenheit') + self.assertEqual(self.processor.process('25 °C', language='fa'), 'بیست و پنج درجه سانتیگراد') + self.assertEqual(self.processor.process('۷۷°F', language='fa'), 'هفتاد و هفت درجه فارنهایت') + + def test_thousand_separators(self): + self.assertEqual(self.processor.process('1,000 m', language='en'), 'one thousand metres') + self.assertEqual(self.processor.process('۱۰۰۰ m', language='fa'), 'یک هزار متر') + + def test_decimals_are_split_into_smaller_units(self): + cases_en = { + '1000.5 m': 'one thousand metres and fifty centimetres', + '2.5 km': 'two kilometres and five hundred metres', + '2.75 kg': 'two kilograms and seven hundred and fifty grams', + '1.5 L': 'one litre and five hundred millilitres', + '2.5 cm': 'two centimetres and five millimetres', + '1.01 m': 'one metre and one centimetre', + '0.5 kg': 'five hundred grams', + '1.234 m': 'one metre, twenty-three centimetres and four millimetres', + '2.0 kg': 'two kilograms', + } + for inp, correct in cases_en.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + cases_fa = { + '۱۰۰۰.۵m': 'یک هزار متر و پنجاه سانتیمتر', + '2.5 km': 'دو کیلومتر و پانصد متر', + '2.75 kg': 'دو کیلوگرم و هفتصد و پنجاه گرم', + '0.5 kg': 'پانصد گرم', + '1.234 m': 'یک متر و بیست و سه سانتیمتر و چهار میلیمتر', + } + for inp, correct in cases_fa.items(): + self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) + + def test_decimals_without_smaller_unit_are_read_as_plain_numbers(self): + # `mi` has no smaller unit, and 1.2345 m does not end on a whole millimetre. + self.assertEqual(self.processor.process('2.5 mi', language='en'), 'two point five miles') + self.assertEqual(self.processor.process('2.5 mi', language='fa'), 'دو و پنج دهم مایل') + self.assertEqual(self.processor.process('36.6 °C', language='en'), 'thirty-six point six degrees Celsius') + self.assertEqual( + self.processor.process('1.2345 m', language='en'), + 'one point two three four five metres', + ) + + def test_fraction_amounts_english(self): + cases = { + '1/2 kg': 'one half kilogram', + '1/2kg': 'one half kilogram', + '3/4 km': 'three quarters kilometre', + '1/3 L': 'one third litre', + '3/2 kg': 'three halves kilograms', + '1,000/2 kg': 'one thousand halves kilograms', + '2 1/2 kg': 'two and one half kilograms', + '1 1/2 hours': 'one and one half hours', + '½ kg': 'one half kilogram', + '1½ kg': 'one and one half kilograms', + '1/2 m/s': 'one half metre per second', + '3/4 km/h': 'three quarters kilometre per hour', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + def test_fraction_amounts_persian(self): + cases = { + '۱/۲ kg': 'یک دوم کیلوگرم', + '۱/۲kg': 'یک دوم کیلوگرم', + '۳/۴ km': 'سه چهارم کیلومتر', + '1/3 L': 'یک سوم لیتر', + '۱٬۰۰۰/۲ kg': 'یک هزار دوم کیلوگرم', + '۲ ۱/۲ kg': 'دو و یک دوم کیلوگرم', + '½ kg': 'یک دوم کیلوگرم', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) + + def test_fraction_amounts_language_from_digits(self): + self.assertEqual(self.processor.process('۱/۲ kg', language='en'), 'یک دوم کیلوگرم') + self.assertEqual(self.processor.process('1/2 kg', language=None), 'one half kilogram') + + def test_invalid_fraction_with_unit_is_unchanged(self): + for inp in ('1/0 kg', '1/1 kg', '3/1 km'): + self.assertEqual(self.processor.process(inp, language='en'), inp, inp) + + def test_percent_english(self): + cases = { + '2 %': 'two percent', + '2%': 'two percent', + '1%': 'one percent', + '100%': 'one hundred percent', + '2 ٪': 'two percent', + '2.5%': 'two point five percent', + '0.5 %': 'zero point five percent', + '1/2 %': 'one half percent', + '2 1/2%': 'two and one half percent', + '1,000%': 'one thousand percent', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + def test_percent_persian(self): + cases = { + '۲ ٪': 'دو درصد', + '۲٪': 'دو درصد', + '۲ %': 'دو درصد', + '2 %': 'دو درصد', + '۱۰۰٪': 'یکصد درصد', + '۲٫۵٪': 'دو و پنج دهم درصد', + '۱/۲ ٪': 'یک دوم درصد', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) + + def test_percent_sign_before_the_number(self): + self.assertEqual(self.processor.process('%20', language='en'), 'twenty percent') + self.assertEqual(self.processor.process('Save %20 today', language='en'), 'Save twenty percent today') + self.assertEqual(self.processor.process('٪۲۰', language='fa'), 'بیست درصد') + self.assertEqual(self.processor.process('تخفیف %۲۰ است', language='fa'), 'تخفیف بیست درصد است') + + def test_percent_sign_before_the_number_with_space(self): + self.assertEqual(self.processor.process('% 5', language='en'), 'five percent') + self.assertEqual(self.processor.process('% 5', language='en'), 'five percent') + self.assertEqual(self.processor.process('Save % 20 today', language='en'), 'Save twenty percent today') + self.assertEqual(self.processor.process('% 2.5', language='en'), 'two point five percent') + self.assertEqual(self.processor.process('٪ ۵', language='fa'), 'پنج درصد') + self.assertEqual(self.processor.process('تخفیف ٪ ۲۰ است', language='fa'), 'تخفیف بیست درصد است') + + def test_number_before_the_sign_wins_over_prefix_reading(self): + # In `5 % 2` the sign belongs to the 5. + self.assertEqual(self.processor.process('5 % 2', language='en'), 'five percent 2') + + def test_unit_after_percent_is_read_as_singular(self): + cases_en = { + '5% kg': 'five percent kilogram', + '5 % kg': 'five percent kilogram', + '2.5% m': 'two point five percent metre', + '5% km/h': 'five percent kilometre per hour', + '1/2 % kg': 'one half percent kilogram', + '% 20 cm': 'twenty percent centimetre', + '%20 cm': 'twenty percent centimetre', + '% 2.5 kg': 'two point five percent kilogram', + '٪ 5 kg': 'five percent kilogram', + } + for inp, correct in cases_en.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + self.assertEqual(self.processor.process('۵٪ kg', language='fa'), 'پنج درصد کیلوگرم') + self.assertEqual(self.processor.process('۵ ٪ km', language='fa'), 'پنج درصد کیلومتر') + self.assertEqual(self.processor.process('٪ ۲۰ cm', language='fa'), 'بیست درصد سانتیمتر') + + def test_symbol_after_another_unit_is_not_consumed(self): + self.assertEqual(self.processor.process('5 km kg', language='en'), 'five kilometres kg') + self.assertEqual(self.processor.process('5 km 3 kg', language='en'), 'five kilometres three kilograms') + self.assertEqual(self.processor.process('10% of 2 kg', language='en'), 'ten percent of two kilograms') + + def test_percent_language_from_digits(self): + self.assertEqual(self.processor.process('۲٪', language='en'), 'دو درصد') + self.assertEqual(self.processor.process('2%', language=None), 'two percent') + + def test_percent_mixed_with_other_units(self): + self.assertEqual( + self.processor.process('Sales rose 5% and 2 kg of rice', language='en'), + 'Sales rose five percent and two kilograms of rice', + ) + self.assertEqual( + self.processor.process('فروش ۵٪ بیشتر شد و ۲ kg برنج', language='fa'), + 'فروش پنج درصد بیشتر شد و دو کیلوگرم برنج', + ) + self.assertEqual( + self.processor.process('5 km, 2 kg and 10 %.', language='en'), + 'five kilometres, two kilograms and ten percent.', + ) + + def test_percent_sign_without_a_number_is_unchanged(self): + inputs = ['%', '٪', 'hello %', 'x % y', '% 1/2', 'printf %d and %s', '% d', '% 5d', 'a%20b', '%20b', '100%a', '1/0 % kg'] + for inp in inputs: + self.assertEqual(self.processor.process(inp, language='en'), inp, inp) + + def test_wrapped_units_and_amounts(self): + cases = { + '2 (kg)': 'two (kilograms)', + '2(kg)': 'two (kilograms)', + '2 [kg]': 'two [kilograms]', + '3 {m}': 'three {metres}', + '3 "m"': 'three "metres"', + '2 ( kg )': 'two (kilograms)', + '(5) kg': '(five) kilograms', + '(1/2) kg': '(one half) kilogram', + '50 (%)': 'fifty (percent)', + '2.5 (kg)': 'two (kilograms) and five hundred (grams)', + '2 (kg) and 3 (m)': 'two (kilograms) and three (metres)', + '(2 kg)': '(two kilograms)', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + self.assertEqual(self.processor.process('۲ (kg)', language='fa'), 'دو (کیلوگرم)') + self.assertEqual(self.processor.process('۳ {m}', language='fa'), 'سه {متر}') + + def test_wrapped_non_units_are_unchanged(self): + for inp in ('3 (a)', '2 (kgx)', '2 (kg', '(5)', '3 (hello)'): + self.assertEqual(self.processor.process(inp, language='en'), inp, inp) + + def test_multiple_units_and_punctuation(self): + self.assertEqual( + self.processor.process('I ran 5 km and lifted 3 kg.', language='en'), + 'I ran five kilometres and lifted three kilograms.', + ) + self.assertEqual(self.processor.process('(5km),', language='en'), '(five kilometres),') + + def test_not_a_unit_is_unchanged(self): + inputs = [ + 'hello', 'km', '5', '5 s', '1990s', '3d', '5 mn', '5G', 'A5 km', '5 km²', + '5 kmh', 'size is 5 in', '5 KM', '5 K', + ] + for inp in inputs: + self.assertEqual(self.processor.process(inp, language='en'), inp) + + def test_language_ignores_unit_symbols(self): + # `km` / `mm` are Latin letters, but the sentence itself is Persian. + self.assertEqual( + self.processor.process('من 10 km 20 mm', language='en'), + 'من ده کیلومتر بیست میلیمتر', + ) + + def test_mixed_language_is_unchanged(self): + self.assertEqual(self.processor.process('hi ما 5 km', language='en'), 'hi ما 5 km') + + def test_unsupported_fallback_language_is_unchanged(self): + for language in ('mixed', 'de'): + self.assertEqual(self.processor.process('5 km', language=language), '5 km') + + def test_only_units_in_text_is_decided_by_digits(self): + # Normalizer detects "en" here because `m` / `km` are Latin letters. + inputs = ['۱۰۰۰m', '۱۰۰۰ m', '1000m', '1000 m', '۵ km', '5 km', '۲.۵ kg', '2.5 kg'] + corrects = [ + 'یک هزار متر', + 'یک هزار متر', + 'one thousand metres', + 'one thousand metres', + 'پنج کیلومتر', + 'five kilometres', + 'دو کیلوگرم و پانصد گرم', + 'two kilograms and five hundred grams', + ] + for language in ('en', None, 'Unknown / No letters found'): + normalized = [self.processor.process(i, language=language) for i in inputs] + self.assertListEqual(normalized, corrects, language) + + def test_epsilon_zero_disables_processor(self): + processor = UnitProcessor(epsilon=0.0) + outputs = {processor.process('5 km', language='en') for _ in range(50)} + self.assertEqual(outputs, {'5 km'}) + + +class FractionProcessorTestCase(unittest.TestCase): + def setUp(self): + self.processor = FractionProcessor() + + def test_english(self): + cases = { + '1/3': 'one third', + '2/3': 'two thirds', + '1/2': 'one half', + '3/2': 'three halves', + '1/4': 'one quarter', + '3/4': 'three quarters', + '5/8': 'five eighths', + '7/10': 'seven tenths', + '1/21': 'one twenty-first', + '1/100': 'one hundredth', + '3/100': 'three hundredths', + '1/1000': 'one thousandth', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + def test_persian(self): + cases = { + '۱/۳': 'یک سوم', + '۲/۳': 'دو سوم', + '۱/۲': 'یک دوم', + '۳/۴': 'سه چهارم', + '۵/۸': 'پنج هشتم', + '۷/۱۰': 'هفت دهم', + '۱/۲۳': 'یک بیست و سوم', + '۱/۳۰': 'یک سی\u200cام', + '۱/۱۰۰': 'یک صدم', + '۳/۱۰۰': 'سه صدم', + '۱/۱۰۰۰': 'یک هزارم', + '1/3': 'یک سوم', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) + + def test_mixed_numbers(self): + cases_en = { + '2 1/2': 'two and one half', + '1 3/4': 'one and three quarters', + '10 2/3': 'ten and two thirds', + } + for inp, correct in cases_en.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + cases_fa = { + '۲ ۱/۲': 'دو و یک دوم', + '۱ ۳/۴': 'یک و سه چهارم', + } + for inp, correct in cases_fa.items(): + self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) + + def test_vulgar_fractions(self): + cases_en = { + '½': 'one half', + '¾': 'three quarters', + '⅔': 'two thirds', + '⅞': 'seven eighths', + '1½': 'one and one half', + '2 ¼': 'two and one quarter', + } + for inp, correct in cases_en.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + self.assertEqual(self.processor.process('½', language='fa'), 'یک دوم') + self.assertEqual(self.processor.process('۱½', language='fa'), 'یک و یک دوم') + + def test_thousands_separators(self): + cases_en = { + '1,000/2': 'one thousand halves', + '12,345/3': 'twelve thousand, three hundred and forty-five thirds', + '1,000 1/2': 'one thousand and one half', + } + for inp, correct in cases_en.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + self.assertEqual(self.processor.process('۱٬۰۰۰/۲', language='fa'), 'یک هزار دوم') + self.assertEqual(self.processor.process('۱,۰۰۰/۲', language='fa'), 'یک هزار دوم') + + def test_fractions_inside_a_sentence_and_punctuation(self): + self.assertEqual( + self.processor.process('I ate 1/3 and 2/3 more.', language='en'), + 'I ate one third and two thirds more.', + ) + self.assertEqual(self.processor.process('(1/3).', language='en'), '(one third).') + self.assertEqual(self.processor.process('1/2.', language='en'), 'one half.') + self.assertEqual( + self.processor.process('من ۱/۳ و ۲/۳ خوردم', language='fa'), + 'من یک سوم و دو سوم خوردم', + ) + + def test_not_a_fraction_is_unchanged(self): + inputs = [ + '1402/05/10', '2024/05/10', '10/05/2024', '3.1/2', '1/2.5', '1/0', '3/1', + 'and/or', '1/2$', '1/2%', 'a1/2', '1/2a', '60 km/h', '12', 'hello', '1,00/2', '1,2/3', + ] + for inp in inputs: + self.assertEqual(self.processor.process(inp, language='en'), inp, inp) + self.assertEqual(self.processor.process(inp, language='fa'), inp, inp) + + def test_no_letters_language_is_decided_by_digits(self): + no_letters = 'Unknown / No letters found' + for language in (None, no_letters): + self.assertEqual(self.processor.process('۱/۳', language=language), 'یک سوم') + self.assertEqual(self.processor.process('1/3', language=language), 'one third') + self.assertEqual(self.processor.process('2 1/2', language=language), 'two and one half') + + def test_unsupported_language_is_unchanged(self): + for language in ('mixed', 'de'): + self.assertEqual(self.processor.process('1/3', language=language), '1/3') + + def test_epsilon_zero_disables_processor(self): + processor = FractionProcessor(epsilon=0.0) + outputs = {processor.process('1/3', language='en') for _ in range(50)} + self.assertEqual(outputs, {'1/3'}) + + +class RomanNumeralProcessorTestCase(unittest.TestCase): + def setUp(self): + self.processor = RomanNumeralProcessor() + + def test_roman_to_int(self): + valid = {'I': 1, 'IV': 4, 'IX': 9, 'XIV': 14, 'XL': 40, 'XCIX': 99, 'MCMXC': 1990, 'MMXXIV': 2024, 'MMMCMXCIX': 3999} + for token, value in valid.items(): + self.assertEqual(roman_to_int(token), value, token) + for token in ('IIII', 'VX', 'IC', 'IIV', 'MMMM', 'ABC', 'iv', ''): + self.assertIsNone(roman_to_int(token), token) + + def test_english(self): + cases = { + 'Henry VIII was king': 'Henry eight was king', + 'Louis XIV and Louis XVI': 'Louis fourteen and Louis sixteen', + 'Chapter IV': 'Chapter four', + 'World War II': 'World War two', + 'World War I': 'World War one', + 'Part V and Section X': 'Part five and Section ten', + 'Napoleon III': 'Napoleon three', + 'MMXXIV': 'two thousand and twenty-four', + 'XIV': 'fourteen', + '(XIV).': '(fourteen).', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + def test_persian(self): + cases = { + 'فصل IV': 'فصل چهار', + 'قرن XX': 'قرن بیست', + 'قرن XVIII': 'قرن هجده', + 'جنگ جهانی II': 'جنگ جهانی دو', + 'جنگ جهانی I': 'جنگ جهانی یک', + 'بخش V را بخوان': 'بخش پنج را بخوان', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) + + def test_wrapped_numerals_after_a_trigger_word(self): + cases = { + 'chapter (V)': 'chapter (five)', + 'Chapter [V]': 'Chapter [five]', + 'Part {I}': 'Part {one}', + 'Chapter "IV"': 'Chapter "four"', + 'Chapter ( V )': 'Chapter ( five )', + 'Henry (VIII)': 'Henry (eight)', + '(XIV)': '(fourteen)', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + self.assertEqual(self.processor.process('فصل (V)', language='fa'), 'فصل (پنج)') + self.assertEqual(self.processor.process('فصل «V»', language='fa'), 'فصل «پنج»') + # a pronoun or a label in a wrapper is still left alone + for inp in ('(I) think so', 'Vitamin (C)', 'Henry (V)'): + self.assertEqual(self.processor.process(inp, language='en'), inp, inp) + + def test_language_ignores_the_numerals(self): + # `XVIII` has more letters than the Persian words, but the sentence is Persian. + self.assertEqual(self.processor.process('قرن XVIII', language='en'), 'قرن هجده') + self.assertEqual(self.processor.process('Chapter IV', language='fa'), 'Chapter four') + + def test_pronoun_and_labels_are_unchanged(self): + inputs = [ + 'I think so', 'After the war I went home', 'Vitamin C', 'Hepatitis C', + 'Appendix C', 'Part D', 'Section L', 'Henry V', 'Washington DC', + 'Dr. Smith, MD', 'CD player', 'XL shirt', 'MIX the flour', + ] + for inp in inputs: + self.assertEqual(self.processor.process(inp, language='en'), inp, inp) + + def test_invalid_and_lowercase_are_unchanged(self): + for inp in ('IIII', 'VX', 'IC', 'MMMM', 'iv', 'xiv', 'Xiv', 'XIVa', 'aXIV'): + self.assertEqual(self.processor.process(inp, language='en'), inp, inp) + + def test_unsupported_language_is_unchanged(self): + for language in ('mixed', 'de', None): + self.assertEqual(self.processor.process('XIV', language=language), 'XIV') + + def test_epsilon_zero_disables_processor(self): + processor = RomanNumeralProcessor(epsilon=0.0) + outputs = {processor.process('Chapter IV', language='en') for _ in range(50)} + self.assertEqual(outputs, {'Chapter IV'}) + + +class AmpersandProcessorTestCase(unittest.TestCase): + def setUp(self): + self.processor = AmpersandProcessor() + + def test_english(self): + cases = { + 'Tom & Jerry': 'Tom and Jerry', + 'R&D': 'R and D', + 'Q&A': 'Q and A', + 'AT&T': 'AT and T', + 'Tom &Jerry': 'Tom and Jerry', + '& Jerry': 'and Jerry', + 'Tom &': 'Tom and', + '(R&D)': '(R and D)', + 'salt & pepper & oil': 'salt and pepper and oil', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + def test_persian(self): + cases = { + 'من & تو': 'من و تو', + 'تام & جری': 'تام و جری', + 'بخش R&D فعال است': 'بخش R و D فعال است', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) + + def test_numbers_beside_ampersand_english(self): + cases = { + '3 & 4': 'three and four', + '3&4': 'three and four', + '1 & 2 & 3': 'one and two and three', + '2.5 & 3.5': 'two point five and three point five', + '1,000 & 2,000': 'one thousand and two thousand', + 'Windows 10 & 11': 'Windows ten and eleven', + 'Tom & 2 friends': 'Tom and two friends', + 'What is 3 & 4?': 'What is three and four?', + '1 & 2.': 'one and two.', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + def test_numbers_beside_ampersand_persian(self): + cases = { + '۳ & ۴': 'سه و چهار', + '3 & 4': 'سه و چهار', + '۱ & ۲ & ۳': 'یک و دو و سه', + '۲٫۵ & ۳٫۵': 'دو و پنج دهم و سه و پنج دهم', + 'سیب ۳ & ۴ پرتقال': 'سیب سه و چهار پرتقال', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) + + def test_numbers_that_are_part_of_something_else_are_not_read_as_plain_numbers(self): + # Only the `&` is read; the dates, times and currency are left for their own processors. + cases = { + '10:30 & 11:30': '10:30 and 11:30', + '2024-05-10 & 2024-05-11': '2024-05-10 and 2024-05-11', + '5-6 & 7': '5-6 and seven', + 'Q3 & Q4': 'Q3 and Q4', + '3rd & 4th': '3rd and 4th', + '$3 & $4': '$3 and $4', + '3$ & 4$': '3$ and 4$', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + def test_urls_and_non_ampersands_are_unchanged(self): + for inp in ('a=1&b=2', 'http://x.com/?a=1&2', '3 && 4', '& 3'): + self.assertEqual(self.processor.process(inp, language='en'), inp, inp) + + def test_wrapped_ampersand_and_numbers(self): + cases = { + 'Tom (&) Jerry': 'Tom (and) Jerry', + 'Tom [&] Jerry': 'Tom [and] Jerry', + '3 (&) 4': 'three (and) four', + '(3) & (4)': '(three) and (four)', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + self.assertEqual(self.processor.process('من (&) تو', language='fa'), 'من (و) تو') + self.assertEqual(self.processor.process('(۳) & (۴)', language='fa'), '(سه) و (چهار)') + + def test_question_mark_does_not_block_ampersand(self): + self.assertEqual(self.processor.process('Is it R&D?', language='en'), 'Is it R and D?') + + def test_spacing_of_the_rest_of_the_text_is_kept(self): + self.assertEqual(self.processor.process('a b & c\nd', language='en'), 'a b and c\nd') + + def test_not_an_ampersand_word_is_unchanged(self): + inputs = ['&', 'a & b', '&', '&', 'a && b', 'a=1&b=2', 'http://x.com?a=1&b=2', 'hello'] + for inp in inputs: + self.assertEqual(self.processor.process(inp, language='en'), inp, inp) + + def test_no_letters_language_is_decided_by_digits(self): + for language in (None, 'Unknown / No letters found'): + self.assertEqual(self.processor.process('1 & 2', language=language), 'one and two') + self.assertEqual(self.processor.process('۱ & ۲', language=language), 'یک و دو') + + def test_unsupported_language_is_unchanged(self): + for language in ('mixed', 'de'): + self.assertEqual(self.processor.process('Tom & Jerry', language=language), 'Tom & Jerry') + + def test_epsilon_zero_disables_processor(self): + processor = AmpersandProcessor(epsilon=0.0) + outputs = {processor.process('Tom & Jerry', language='en') for _ in range(50)} + self.assertEqual(outputs, {'Tom & Jerry'}) + + +class NegativeSignProcessorTestCase(unittest.TestCase): + def setUp(self): + self.processor = NegativeSignProcessor() + + def test_bare_numbers_are_read_as_words(self): + cases_en = { + '-2': 'negative two', + '−2': 'negative two', + '(-2)': '(negative two)', + '= -2': '= negative two', + 'The temp is -2 degrees': 'The temp is negative two degrees', + '-2.5': 'negative two point five', + '-1,000': 'negative one thousand', + '-2.': 'negative two.', + } + for inp, correct in cases_en.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + self.assertEqual(self.processor.process('-۲', language='fa'), 'منفی دو') + self.assertEqual(self.processor.process('دما -۲ درجه است', language='fa'), 'دما منفی دو درجه است') + + def test_number_with_unit_fraction_percent_or_currency_is_left_for_their_processors(self): + cases = { + '-2 m': 'negative 2 m', + '-2m': 'negative 2m', + '- 2 m': 'negative 2 m', + '-1/2 kg': 'negative 1/2 kg', + '-5%': 'negative 5%', + '-$5': 'negative $5', + '-5$': 'negative 5$', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + # The language comes from the Persian digits, not from the Latin `m`. + self.assertEqual(self.processor.process('-۲ m', language='en'), 'منفی ۲ m') + self.assertEqual(self.processor.process('- ۲ m', language='en'), 'منفی ۲ m') + + def test_space_after_minus_needs_a_unit_or_an_operator(self): + self.assertEqual(self.processor.process('3 + - 2', language='en'), '3 + negative two') + self.assertEqual(self.processor.process('x = - 2', language='en'), 'x = negative two') + for inp in ('- 2024 report', '- 2', '- item', '10 - 20 km', 'He said - 2 m'): + self.assertEqual(self.processor.process(inp, language='en'), inp, inp) + + def test_not_a_negative_number_is_unchanged(self): + inputs = [ + '5-3', 'a-2', 'COVID-19', '2024-05-10', '0912-123-4567', '-10:30', + '-2nd', '-2x', '--2', '5--2', 'hello', '-', '- ', + ] + for inp in inputs: + self.assertEqual(self.processor.process(inp, language='en'), inp, inp) + + def test_unsupported_language_is_unchanged(self): + for language in ('mixed', 'de'): + self.assertEqual(self.processor.process('-2', language=language), '-2') + + def test_epsilon_zero_disables_processor(self): + processor = NegativeSignProcessor(epsilon=0.0) + outputs = {processor.process('-2', language='en') for _ in range(50)} + self.assertEqual(outputs, {'-2'}) + + +class MathOperatorProcessorTestCase(unittest.TestCase): + def setUp(self): + self.processor = MathOperatorProcessor() + + def test_english(self): + cases = { + '3 + 4': 'three plus four', + '3+4': 'three plus four', + '6 × 2': 'six times two', + '6×2': 'six times two', + '3 * 4': 'three times four', + '8 ÷ 2': 'eight divided by two', + '7 = 7': 'seven equals seven', + '1+1=2': 'one plus one equals two', + '3 + 4 = 7': 'three plus four equals seven', + '1 + 2 + 3': 'one plus two plus three', + '2.5 + 1,000': 'two point five plus one thousand', + 'Tom + Jerry': 'Tom plus Jerry', + 'x + y = z': 'x plus y equals z', + 'Is 2 + 2 = 4?': 'Is two plus two equals four?', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + def test_persian(self): + cases = { + '۳ + ۴': 'سه بعلاوه چهار', + '۳+۴': 'سه بعلاوه چهار', + '۶ × ۲': 'شش ضربدر دو', + '۸ ÷ ۲': 'هشت تقسیم بر دو', + '۷ = ۷': 'هفت مساوی است با هفت', + '۳ + ۴ = ۷': 'سه بعلاوه چهار مساوی است با هفت', + 'حاصل ۲ + ۲ = ۴ است': 'حاصل دو بعلاوه دو مساوی است با چهار است', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) + + def test_wrapped_operands_and_symbols(self): + self.assertEqual(self.processor.process('(3) + (4)', language='en'), '(three) plus (four)') + self.assertEqual(self.processor.process('3 (+) 4', language='en'), 'three (plus) four') + self.assertEqual(self.processor.process('[6] × [2]', language='en'), '[six] times [two]') + self.assertEqual(self.processor.process('(3 + 4)', language='en'), '(three plus four)') + self.assertEqual(self.processor.process('(۳) + (۴)', language='fa'), '(سه) بعلاوه (چهار)') + self.assertEqual(self.processor.process('۳ (+) ۴', language='fa'), 'سه (بعلاوه) چهار') + + def test_minus(self): + self.assertEqual(self.processor.process('5 − 3', language='en'), 'five minus three') + self.assertEqual(self.processor.process('۵ − ۳', language='fa'), 'پنج منهای سه') + # an ASCII hyphen is a minus only in a spaced equation + self.assertEqual(self.processor.process('5 - 3 = 2', language='en'), 'five minus three equals two') + self.assertEqual(self.processor.process('۵ - ۳ = ۲', language='fa'), 'پنج منهای سه مساوی است با دو') + self.assertEqual(self.processor.process('x - y = z', language='en'), 'x minus y equals z') + for inp in ('5 - 3', '5-3', '10 - 20', 'well - maybe'): + self.assertEqual(self.processor.process(inp, language='en'), inp, inp) + + def test_symbols_that_are_not_between_operands_are_unchanged(self): + inputs = [ + '+98 912 123 4567', 'C++ is great', 'A+ grade', 'x=5', 'a=1&b=2', + 'http://x.com/?a=1&b=2', 'a == b', 'a <= b', '* item', '+', '=', '3 +', + '+ 4', 'hello', + ] + for inp in inputs: + self.assertEqual(self.processor.process(inp, language='en'), inp, inp) + + def test_units_and_currency_are_left_for_their_processors(self): + # Only the symbol is read here; `$3` and `3$` are read later by CurrencyProcessor. + self.assertEqual(self.processor.process('$3 + $4', language='en'), '$3 plus $4') + self.assertEqual(self.processor.process('3$ + 4$', language='en'), '3$ plus 4$') + self.assertEqual(self.processor.process('10:30 + 2', language='en'), '10:30 plus two') + self.assertEqual(self.processor.process('2024-05-10 + 3', language='en'), '2024-05-10 plus three') + + def test_no_letters_language_is_decided_by_digits(self): + for language in (None, 'Unknown / No letters found'): + self.assertEqual(self.processor.process('3 + 4', language=language), 'three plus four') + self.assertEqual(self.processor.process('۳ + ۴', language=language), 'سه بعلاوه چهار') + + def test_unsupported_language_is_unchanged(self): + for language in ('mixed', 'de'): + self.assertEqual(self.processor.process('3 + 4', language=language), '3 + 4') + + def test_epsilon_zero_disables_processor(self): + processor = MathOperatorProcessor(epsilon=0.0) + outputs = {processor.process('3 + 4', language='en') for _ in range(50)} + self.assertEqual(outputs, {'3 + 4'}) + + +class RangeProcessorTestCase(unittest.TestCase): + def setUp(self): + self.processor = RangeProcessor() + + def test_plain_numbers_english(self): + cases = { + '3 - 5': 'three to five', + '3-5': 'three to five', + '3 – 5': 'three to five', + '3–5': 'three to five', + '10 - 20': 'ten to twenty', + '2.5 - 3.5': 'two point five to three point five', + 'pages 3-5': 'pages three to five', + 'Is it 3 - 5?': 'Is it three to five?', + '1 - 2 - 3': 'one to two to three', + '1990 - 2000': 'one thousand, nine hundred and ninety to two thousand', + '1990–2000': 'one thousand, nine hundred and ninety to two thousand', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + def test_plain_numbers_persian(self): + cases = { + '۳ - ۵': 'سه تا پنج', + '۳-۵': 'سه تا پنج', + '۳ – ۵': 'سه تا پنج', + '3 - 5': 'سه تا پنج', + 'از ۳ - ۵ ساعت': 'از سه تا پنج ساعت', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) + + def test_amounts_with_units_fractions_percent_and_currency_are_left_for_their_processors(self): + # Only the dash (and a plain number beside it) is read here. + cases = { + '3kg - 5 kg': '3kg to 5 kg', + '3 kg-5 kg': '3 kg to 5 kg', + '3 - 5 kg': 'three to 5 kg', + '3 kg - 5': '3 kg to five', + '10 - 20 km': 'ten to 20 km', + '$3 - $5': '$3 to $5', + '3$ - 5$': '3$ to 5$', + '1/2 - 3/4': '1/2 to 3/4', + '5% - 10%': '5% to 10%', + '10 - 20%': 'ten to 20%', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + self.assertEqual(self.processor.process('۳kg - ۵ kg', language='fa'), '۳kg تا ۵ kg') + + def test_wrapped_amounts(self): + self.assertEqual(self.processor.process('(3) - (5)', language='en'), '(three) to (five)') + self.assertEqual(self.processor.process('[3]-[5]', language='en'), '[three] to [five]') + self.assertEqual(self.processor.process('(3 - 5)', language='en'), '(three to five)') + self.assertEqual(self.processor.process('(۳) - (۵)', language='fa'), '(سه) تا (پنج)') + # wrapped amounts with units are left for UnitProcessor + self.assertEqual(self.processor.process('3 (cm) - 5 (cm)', language='en'), '3 (cm) to 5 (cm)') + self.assertEqual(self.processor.process('(2) kg - (5) kg', language='en'), '(2) kg to (5) kg') + + def test_not_a_range_is_unchanged(self): + inputs = [ + '2024-05-10', '2024-05', '05-10', '0912-123-4567', '555-1234', '1990-2000', + 'COVID-19', 'A-5', 'a - b', 'well - maybe', '10:30 - 11:30', + '2024/05/10 - 2024/05/12', '3 -5', '3- 5', '- 5', '3 -', '3 –', + 'http://x.com/3-5', '5 - 3 = 2', 'x = 3 - 5', + ] + for inp in inputs: + self.assertEqual(self.processor.process(inp, language='en'), inp, inp) + + def test_no_letters_language_is_decided_by_digits(self): + for language in (None, 'Unknown / No letters found'): + self.assertEqual(self.processor.process('3 - 5', language=language), 'three to five') + self.assertEqual(self.processor.process('۳ - ۵', language=language), 'سه تا پنج') + + def test_unsupported_language_is_unchanged(self): + for language in ('mixed', 'de'): + self.assertEqual(self.processor.process('3 - 5', language=language), '3 - 5') + + def test_epsilon_zero_disables_processor(self): + processor = RangeProcessor(epsilon=0.0) + outputs = {processor.process('3 - 5', language='en') for _ in range(50)} + self.assertEqual(outputs, {'3 - 5'}) + + +class PhoneNumberProcessorTestCase(unittest.TestCase): + MOBILE_EN = 'zero nine one two, four three three, eight one, three five' + MOBILE_PLUS_EN = 'plus nine eight, nine one two, four three three, eight one, three five' + LANDLINE_EN = 'zero two one, eight eight, zero six, nine eight, nine five' + MOBILE_FA = 'صفر نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج' + MOBILE_PLUS_FA = 'مثبت نود و هشت، نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج' + LANDLINE_FA = 'صفر بیست و یک، هشتاد و هشت، صفر شش، نود و هشت، نود و پنج' + + def setUp(self): + self.processor = PhoneNumberProcessor() + + def test_iranian_mobile_english(self): + for inp in ('09124338135', '0912-433-8135', '0912 433 8135', '0912.433.8135', '0912-4338135'): + self.assertEqual(self.processor.process(inp, language='en'), self.MOBILE_EN, inp) + + def test_international_english(self): + for inp in ('+989124338135', '+98-912-4338135', '+98 912 433 8135', '+98.912.433.8135', '(+98) 912 4338135'): + expected = self.MOBILE_PLUS_EN + if inp.startswith('('): + expected = '(plus nine eight), nine one two, four three three, eight one, three five' + self.assertEqual(self.processor.process(inp, language='en'), expected, inp) + self.assertEqual( + self.processor.process('0098 912 4338135', language='en'), + 'zero zero nine eight, nine one two, four three three, eight one, three five', + ) + self.assertEqual( + self.processor.process('+44 20 7946 0958', language='en'), + 'plus four four, two zero, seven nine four six, zero nine five eight', + ) + self.assertEqual( + self.processor.process('+1 (555) 123-4567', language='en'), + 'plus one, (five five five), one two three, four five six seven', + ) + + def test_iranian_landline_english(self): + for inp in ('021-88069895', '021 88069895', '02188069895', '021.88069895'): + self.assertEqual(self.processor.process(inp, language='en'), self.LANDLINE_EN, inp) + self.assertEqual( + self.processor.process('(021) 88069895', language='en'), + '(zero two one), eight eight, zero six, nine eight, nine five', + ) + + def test_us_style_english(self): + cases = { + '555-123-4567': 'five five five, one two three, four five six seven', + '555.123.4567': 'five five five, one two three, four five six seven', + '(555) 123-4567': '(five five five), one two three, four five six seven', + '1-555-123-4567': 'one, five five five, one two three, four five six seven', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + # plain spaces are not enough: this could be a list of numbers + self.assertEqual(self.processor.process('555 123 4567', language='en'), '555 123 4567') + + def test_persian_digits_and_persian_output(self): + cases = { + '۰۹۱۲۴۳۳۸۱۳۵': self.MOBILE_FA, + '۰۹۱۲-۴۳۳-۸۱۳۵': self.MOBILE_FA, + '+۹۸۹۱۲۴۳۳۸۱۳۵': self.MOBILE_PLUS_FA, + '+۹۸-۹۱۲-۴۳۳۸۱۳۵': self.MOBILE_PLUS_FA, + '۰۲۱-۸۸۰۶۹۸۹۵': self.LANDLINE_FA, + '٠٩١٢٤٣٣٨١٣٥': self.MOBILE_FA, + '09124338135': self.MOBILE_FA, + '+989124338135': self.MOBILE_PLUS_FA, + '021-88069895': self.LANDLINE_FA, + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) + self.assertEqual( + self.processor.process('(۰۲۱) ۸۸۰۶۹۸۹۵', language='fa'), + '(صفر بیست و یک)، هشتاد و هشت، صفر شش، نود و هشت، نود و پنج', + ) + + def test_persian_reads_iranian_numbers_in_groups(self): + cases = { + '۰۹۱۲۴۳۳۸۱۳۵': 'صفر نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج', + '09124338135': 'صفر نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج', + '02188037395': 'صفر بیست و یک، هشتاد و هشت، صفر سه، هفتاد و سه، نود و پنج', + '۰۲۱-۸۸۰۳۷۳۹۵': 'صفر بیست و یک، هشتاد و هشت، صفر سه، هفتاد و سه، نود و پنج', + '۰۴۱۳۱۲۳۴۵۶۷': 'صفر چهل و یک، سی و یک، بیست و سه، چهل و پنج، شصت و هفت', + # a leading zero inside a group is said on its own + '۰۹۱۲۰۰۰۰۵۰۸': 'صفر نهصد و دوازده، سه صفر، صفر پنج، صفر هشت', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) + + def test_persian_counts_repeated_leading_zeros(self): + cases = { + '۰۲۱-۸۸۵۲۳۰۰۶': 'صفر بیست و یک، هشتاد و هشت، پنجاه و دو، سی، صفر شش', + '۰۰۱-۸۸۵۲۳۰۰۶': 'دو صفر یک، هشتاد و هشت، پنجاه و دو، سی، صفر شش', + '00188523006': 'دو صفر یک، هشتاد و هشت، پنجاه و دو، سی، صفر شش', + '001 88 52 30 06': 'دو صفر یک، هشتاد و هشت، پنجاه و دو، سی، صفر شش', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) + + def test_english_reads_the_same_groups_for_a_zero_area_code(self): + self.assertEqual( + self.processor.process('۰۰۱-۸۸۵۲۳۰۰۶', language='en'), + 'zero zero one, eight eight, five two, three zero, zero six', + ) + + def test_persian_international_iranian_numbers(self): + cases = { + '+989124338135': 'مثبت نود و هشت، نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج', + '+98-912-4338135': 'مثبت نود و هشت، نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج', + '+98 21 88037395': 'مثبت نود و هشت، بیست و یک، هشتاد و هشت، صفر سه، هفتاد و سه، نود و پنج', + '0098 912 4338135': 'دو صفر نود و هشت، نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج', + '(+98) 912 4338135': '(مثبت نود و هشت)، نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج', + '(021) 88037395': '(صفر بیست و ی��)، هشتاد و هشت، صفر سه، هفتاد و سه، نود و پنج', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) + + def test_persian_reads_other_numbers_digit_by_digit(self): + cases = { + '+44 20 7946 0958': 'مثبت چهار چهار دو صفر هفت نه چهار شش صفر نه پنج هشت', + '555-123-4567': 'پنج پنج پنج یک دو سه چهار پنج شش هفت', + '0012345678': 'صفر صفر یک دو سه چهار پنج شش هفت هشت', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) + + def test_english_reads_iranian_numbers_in_groups_with_pauses(self): + cases = { + '09124338135': 'zero nine one two, four three three, eight one, three five', + '02188037395': 'zero two one, eight eight, zero three, seven three, nine five', + '+989124338135': 'plus nine eight, nine one two, four three three, eight one, three five', + '+98 21 88037395': 'plus nine eight, two one, eight eight, zero three, seven three, nine five', + '0098 912 4338135': 'zero zero nine eight, nine one two, four three three, eight one, three five', + '(021) 88037395': '(zero two one), eight eight, zero three, seven three, nine five', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + def test_english_other_numbers_use_the_written_groups(self): + cases = { + '+44 20 7946 0958': 'plus four four, two zero, seven nine four six, zero nine five eight', + '+1 (555) 123-4567': 'plus one, (five five five), one two three, four five six seven', + # no written groups: nothing to pause on + '+441234567890': 'plus four four one two three four five six seven eight nine zero', + '0012345678': 'zero zero one two three four five six seven eight', + } + for inp, correct in cases.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + def test_no_space_needed_after_a_closing_parenthesis(self): + for inp in ('(0915) 6486320', '(0915)6486320'): + self.assertEqual( + self.processor.process(inp, language='en'), + '(zero nine one five), six four eight, six three, two zero', + inp, + ) + cases_en = { + '(021)88069895': '(zero two one), eight eight, zero six, nine eight, nine five', + '(+98)9124338135': '(plus nine eight), nine one two, four three three, eight one, three five', + '(555)123-4567': '(five five five), one two three, four five six seven', + 'Call (0915)6486320, ok': 'Call (zero nine one five), six four eight, six three, two zero, ok', + '(0915)6486320 and (021)88069895': ( + '(zero nine one five), six four eight, six three, two zero and ' + '(zero two one), eight eight, zero six, nine eight, nine five' + ), + } + for inp, correct in cases_en.items(): + self.assertEqual(self.processor.process(inp, language='en'), correct, inp) + + cases_fa = { + '(۰۹۱۵)۶۴۸۶۳۲۰': '(صفر نهصد و پانزده)، ششصد و چهل و هشت، شصت و سه، بیست', + '(۰۹۱۵) ۶۴۸۶۳۲۰': '(صفر نهصد و پانزده)، ششصد و چهل و هشت، شصت و سه، بیست', + '(021)۸۸۰۶۹۸۹۵': '(صفر بیست و یک)، هشتاد و هشت، صفر شش، نود و هشت، نود و پنج', + } + for inp, correct in cases_fa.items(): + self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) + + def test_parentheses_without_a_phone_number_are_unchanged(self): + for inp in ('(1)23456789', '(2)(3)', '5 (kg)12', '(12)34'): + self.assertEqual(self.processor.process(inp, language='en'), inp, inp) + + def test_numbers_inside_sentences(self): + self.assertEqual( + self.processor.process('Call me at 09124338135.', language='en'), + f'Call me at {self.MOBILE_EN}.', + ) + self.assertEqual( + self.processor.process('Call 09124338135, then 021-88069895.', language='en'), + f'Call {self.MOBILE_EN}, then {self.LANDLINE_EN}.', + ) + self.assertEqual( + self.processor.process('(+98 912 4338135)', language='en'), + '(plus nine eight, nine one two, four three three, eight one, three five)', + ) + self.assertEqual( + self.processor.process('شماره من ۰۹۱۲۴۳۳۸۱۳۵ است', language='fa'), + f'شماره من {self.MOBILE_FA} است', + ) + self.assertEqual( + self.processor.process('تماس: ۰۲۱-۸۸۰۶۹۸۹۵ و ۰۹۱۲۴۳۳۸۱۳۵', language='fa'), + f'تماس: {self.LANDLINE_FA} و {self.MOBILE_FA}', + ) + self.assertEqual( + self.processor.process('In 2024 0912 433 8135', language='en'), + f'In 2024 {self.MOBILE_EN}', + ) + + def test_number_after_a_phone_word_may_be_shorter(self): + self.assertEqual( + self.processor.process('tel: 88069895', language='en'), + 'tel: eight eight, zero six, nine eight, nine five', + ) + self.assertEqual( + self.processor.process('Phone number 4338135', language='en'), + 'Phone number four three three eight one three five', + ) + self.assertEqual( + self.processor.process('تلفن: ۸۸۰۶۹۸۹۵', language='fa'), + 'تلفن: هشتاد و هشت، صفر شش، نود و هشت، نود و پنج', + ) + self.assertEqual( + self.processor.process('Call 1-800-555-1234', language='en'), + 'Call one, eight zero zero, five five five, one two three four', + ) + # too short, or no phone word: unchanged + for inp in ('call 911', 'call 5 times', '4338135', 'Tel 123456'): + self.assertEqual(self.processor.process(inp, language='en'), inp, inp) + self.assertEqual(self.processor.process('شماره 5', language='fa'), 'شماره 5') + + def test_language_comes_from_digits_when_there_are_no_letters(self): + for language in (None, 'Unknown / No letters found'): + self.assertEqual(self.processor.process('09124338135', language=language), self.MOBILE_EN) + self.assertEqual(self.processor.process('۰۹۱۲۴۳۳۸۱۳۵', language=language), self.MOBILE_FA) + + def test_not_a_phone_number_is_unchanged(self): + inputs = [ + '4338135', '9124338135', '123456', '2024-05-10', '2024/05/10', '01-02-2024', + '01.02.2024', '0.123456789', '3.0912433813', '0912433813.5', '0912433813.55', + '09124338135abc', 'a09124338135', 'https://x.com/09124338135', + 'user09124338135@mail.com', '1234 5678 9012 3456', '0912 433', '0912', + '021-88069895-12', '5 - 3', '1990 - 2000', '10:30', '1,000,000', 'hello', + ] + for inp in inputs: + self.assertEqual(self.processor.process(inp, language='en'), inp, inp) + + def test_ten_digit_zero_prefixed_id_is_read_digit_by_digit(self): + self.assertEqual( + self.processor.process('0012345678', language='en'), + 'zero zero one two three four five six seven eight', + ) + + def test_unsupported_language_is_unchanged(self): + for language in ('mixed', 'de'): + self.assertEqual(self.processor.process('09124338135', language=language), '09124338135') + + def test_epsilon_zero_disables_processor(self): + processor = PhoneNumberProcessor(epsilon=0.0) + outputs = {processor.process('09124338135', language='en') for _ in range(50)} + self.assertEqual(outputs, {'09124338135'}) + + +class CardNumberProcessorTestCase(unittest.TestCase): + def setUp(self): + self.processor = CardNumberProcessor() + + def assertFa(self, text, expected): + self.assertEqual(self.processor.process(text, language='fa'), expected) + + def assertEn(self, text, expected): + self.assertEqual(self.processor.process(text, language='en'), expected) + + # Persian + def test_persian_card_with_hyphens(self): + self.assertFa( + '۶۲۱۹ - ۸۷۶۵ - ۲۳۴۵ - ۹۰۳۱', + 'شصت و دو، نوزده - هشتاد و هفت، شصت و پنج - بیست و سه، چهل و پنج - نود، سی و یک', + ) + + def test_persian_card_with_tight_hyphens_spaces_and_no_separator(self): + expected = 'شصت و دو، نوزده - هشتاد و هفت، شصت و پنج - بیست و سه، چهل و پنج - نود، سی و یک' + for text in ('۶۲۱۹-۸۷۶۵-۲۳۴۵-۹۰۳۱', '۶۲۱۹ ۸۷۶۵ ۲۳۴۵ ۹۰۳۱', '۶۲۱۹۸۷۶۵۲۳۴۵۹۰۳۱'): + self.assertFa(text, expected) + + def test_persian_card_with_ascii_digits(self): + self.assertFa( + '6219 8765 2345 9031', + 'شصت و دو، نوزده - هشتاد و هفت، شصت و پنج - بیست و سه، چهل و پنج - نود، سی و یک', + ) + + def test_persian_card_in_sentence(self): + self.assertFa( + 'شماره کارت ۶۲۱۹ ۸۷۶۵ ۲۳۴۵ ۹۰۳۱ است', + 'شماره کارت شصت و دو، نوزده - هشتاد و هفت، شصت و پنج - بیست و سه، چهل و پنج - نود، سی و یک است', + ) + + def test_persian_zero_pairs(self): + result = self.processor.process('۶۰۰۵ ۱۲۰۰ ۳۴۵۶ ۷۸۹۰', language='fa') + self.assertIn('صفر پنج', result) + self.assertIn('دو صفر', result) + + # English + def test_english_card_with_spaces(self): + self.assertEn( + '4111 1111 1111 1111', + 'forty-one, eleven - eleven, eleven - eleven, eleven - eleven, eleven', + ) + + def test_english_card_with_hyphens(self): + self.assertEn( + '5105-1051-0510-5100', + 'fifty-one, oh five - ten, fifty-one - oh five, ten - fifty-one, double zero', + ) + + def test_english_card_without_separators(self): + self.assertEn( + '6219876523459031', + 'sixty-two, nineteen - eighty-seven, sixty-five - twenty-three, forty-five - ninety, thirty-one', + ) + + def test_english_card_in_sentence(self): + self.assertEn( + 'My card is 4111 1111 1111 1111 ok', + 'My card is forty-one, eleven - eleven, eleven - eleven, eleven - eleven, eleven ok', + ) + + def test_english_card_with_persian_digits(self): + self.assertEn( + '۴۱۱۱ ۱۱۱۱ ۱۱۱۱ ۱۱۱۱', + 'forty-one, eleven - eleven, eleven - eleven, eleven - eleven, eleven', + ) + + # Language detection + def test_language_detected_from_digits(self): + self.assertEqual( + self.processor.process('۶۲۱۹ ۸۷۶۵ ۲۳۴۵ ۹۰۳۱'), + 'شصت و دو، نوزده - هشتاد و هفت، شصت و پنج - بیست و سه، چهل و پنج - نود، سی و یک', + ) + self.assertEqual( + self.processor.process('4111 1111 1111 1111'), + 'forty-one, eleven - eleven, eleven - eleven, eleven - eleven, eleven', + ) + + # Left alone + def test_non_cards_are_unchanged(self): + for text in ( + '6219-****-****-9031', + '6219 8765 2345 9031.5', + '6219 8765 2345', + '6219 8765 2345 9031 1234', + '3782 822463 10005', + '09124338135', + 'abc6219876523459031', + ): + self.assertEqual(self.processor.process(text, language='en'), text) + + def test_any_first_digit_is_accepted(self): + self.assertEn( + '1234 5678 9012 3456', + 'twelve, thirty-four - fifty-six, seventy-eight - ninety, twelve - thirty-four, fifty-six', + ) + self.assertEn( + '0912 4338 1350 0000', + 'oh nine, twelve - forty-three, thirty-eight - thirteen, fifty - double zero, double zero', + ) + self.assertEn( + '8123 4567 8901 2345', + 'eighty-one, twenty-three - forty-five, sixty-seven - eighty-nine, oh one - twenty-three, forty-five', + ) + + def test_unsupported_language_is_unchanged(self): + for language in ('mixed', 'de'): + self.assertEqual( + self.processor.process('4111 1111 1111 1111', language=language), '4111 1111 1111 1111') + + def test_epsilon_zero_disables_processor(self): + processor = CardNumberProcessor(epsilon=0.0) + outputs = {processor.process('4111 1111 1111 1111', language='en') for _ in range(50)} + self.assertEqual(outputs, {'4111 1111 1111 1111'}) + + # Through the Normalizer + def test_normalizer_reads_card_not_phone(self): + normalizer = Normalizer() + self.assertEqual( + normalizer.process('شماره کارت ۶۲۱۹ ۸۷۶۵ ۲۳۴۵ ۹۰۳۱ است'), + 'شماره کارت شصت و دو، نوزده - هشتاد و هفت، شصت و پنج - بیست و سه، چهل و پنج - نود، سی و یک است', + ) + self.assertEqual( + normalizer.process('My card is 4111 1111 1111 1111 ok'), + 'My card is forty-one, eleven - eleven, eleven - eleven, eleven - eleven, eleven ok', + ) + + +class PostalCodeAndNationalIdProcessorTestCase(unittest.TestCase): + def setUp(self): + self.processor = PostalCodeAndNationalIdProcessor() + + def assertFa(self, text, expected): + self.assertEqual(self.processor.process(text, language='fa'), expected) + + def assertEn(self, text, expected): + self.assertEqual(self.processor.process(text, language='en'), expected) + + # Persian + def test_persian_postal_code(self): + self.assertFa('۱۵۱۴۷۱۷۱۱۸', 'پانزده، چهارده، هفتاد و یک، هفتاد و یک، هجده') + + def test_persian_postal_code_with_ascii_digits(self): + self.assertFa('1514717118', 'پانزده، چهارده، هفتاد و یک، هفتاد و یک، هجده') + + def test_persian_postal_code_split_in_two_halves(self): + expected = 'پانزده، چهارده، هفتاد و یک، هفتاد و یک، هجده' + for text in ('۱۵۱۴۷-۱۷۱۱۸', '۱۵۱۴۷ ۱۷۱۱۸', '۱۵۱۴۷ - ۱۷۱۱۸'): + self.assertFa(text, expected) + + def test_persian_postal_code_in_sentence(self): + self.assertFa( + 'کد پستی ۱۵۱۴۷۱۷۱۱۸ است', + 'کد پستی پانزده، چهارده، هفتاد و یک، هفتاد و یک، هجده است', + ) + + def test_persian_zero_pairs(self): + result = self.processor.process('۱۵۱۴۷۰۰۰۰۵', language='fa') + self.assertIn('دو صفر', result) + self.assertIn('صفر پنج', result) + + # English + def test_english_postal_code(self): + self.assertEn('1514717118', 'fifteen, fourteen, seventy-one, seventy-one, eighteen') + + def test_english_postal_code_split_in_two_halves(self): + self.assertEn('15147-17118', 'fifteen, fourteen, seventy-one, seventy-one, eighteen') + + def test_english_postal_code_with_persian_digits(self): + self.assertEn('۱۵۱۴۷۱۷۱۱۸', 'fifteen, fourteen, seventy-one, seventy-one, eighteen') + + def test_english_postal_code_in_sentence(self): + self.assertEn( + 'Postal code 1514717118 ok', + 'Postal code fifteen, fourteen, seventy-one, seventy-one, eighteen ok', + ) + + def test_english_zero_pairs(self): + self.assertEn('1514700508', 'fifteen, fourteen, seventy, oh five, oh eight') + self.assertEn('1514700018', 'fifteen, fourteen, seventy, double zero, eighteen') + + # National ID numbers + def test_persian_national_id_with_leading_zeros(self): + self.assertFa('۰۰۱۲۳۴۵۶۷۹', 'دو صفر، دوازده، سی و چهار، پنجاه و شش، هفتاد و نه') + + def test_english_national_id_with_leading_zeros(self): + self.assertEn('0012345679', 'double zero, twelve, thirty-four, fifty-six, seventy-nine') + + def test_national_id_with_one_leading_zero(self): + self.assertEn('0499370899', 'oh four, ninety-nine, thirty-seven, oh eight, ninety-nine') + self.assertFa('۰۴۹۹۳۷۰۸۹۹', 'صفر چهار، نود و نه، سی و هفت، صفر هشت، نود و نه') + + def test_national_id_with_hyphens_3_6_1(self): + self.assertEn('001-234567-9', 'double zero, twelve, thirty-four, fifty-six, seventy-nine') + self.assertFa('۰۰۱-۲۳۴۵۶۷-۹', 'دو صفر، دوازده، سی و چهار، پنجاه و شش، هفتاد و نه') + + def test_national_id_in_sentence(self): + self.assertFa( + 'کد ملی ۰۰۱۲۳۴۵۶۷۹ را وارد کنید', + 'کد ملی دو صفر، دوازده، سی و چهار، پنجاه و شش، هفتاد و نه را وارد کنید', + ) + self.assertEn( + 'National ID 0012345679 is valid', + 'National ID double zero, twelve, thirty-four, fifty-six, seventy-nine is valid', + ) + + def test_invalid_national_id_is_unchanged(self): + for text in ( + '0012345678', # wrong check digit + '001-234567-8', # wrong check digit, hyphenated layout + '0000000000', # all the same digit (passes the checksum but is not a real ID) + ): + self.assertEn(text, text) + + def test_hyphenated_3_6_1_layout_needs_valid_check_digit(self): + # Passes the postal-code rule when written as 5 + 5, but 3 + 6 + 1 is a national ID layout only. + self.assertEn('151-471711-8', '151-471711-8') + + # Language detection + def test_language_detected_from_digits(self): + self.assertEqual( + self.processor.process('۱۵۱۴۷۱۷۱۱۸'), + 'پانزده، چهارده، هفتاد و یک، هفتاد و یک، هجده', + ) + self.assertEqual( + self.processor.process('1514717118'), + 'fifteen, fourteen, seventy-one, seventy-one, eighteen', + ) + + # Left alone + def test_non_identifiers_are_unchanged(self): + for text in ( + '9124338135', # phone number without the leading zero (a 2 in the first five) + '1234567890', # a 0 or 2 in the first five digits + '0514717118', # starts with 0 + '15147171189', # 11 digits + '151471711', # 9 digits + '1514717118.5', # decimal + '1514 7171 18', # wrong grouping + 'abc1514717118', # glued to a word + ): + self.assertEqual(self.processor.process(text, language='en'), text) + + def test_card_number_is_not_read_as_postal_code(self): + self.assertEqual(self.processor.process('6219876523459031', language='en'), '6219876523459031') + + def test_unsupported_language_is_unchanged(self): + for language in ('mixed', 'de'): + self.assertEqual(self.processor.process('1514717118', language=language), '1514717118') + + def test_epsilon_zero_disables_processor(self): + processor = PostalCodeAndNationalIdProcessor(epsilon=0.0) + outputs = {processor.process('1514717118', language='en') for _ in range(50)} + self.assertEqual(outputs, {'1514717118'}) + + # Through the Normalizer + def test_normalizer_reads_postal_code(self): + normalizer = Normalizer() + self.assertEqual( + normalizer.process('کد پستی ۱۵۱۴۷۱۷۱۱۸ است'), + 'کد پستی پانزده، چهارده، هفتاد و یک، هفتاد و یک، هجده است', + ) + self.assertEqual( + normalizer.process('Postal code 1514717118 ok'), + 'Postal code fifteen, fourteen, seventy-one, seventy-one, eighteen ok', + ) + + def test_normalizer_still_reads_phone_and_card(self): + normalizer = Normalizer() + self.assertEqual( + normalizer.process('call 09124338135'), + 'call zero nine one two, four three three, eight one, three five', + ) + self.assertIn('شصت و دو، نوزده', normalizer.process('کارت ۶۲۱۹ ۸۷۶۵ ۲۳۴۵ ۹۰۳۱')) + + +if __name__ == '__main__': + unittest.main(verbosity=2) \ No newline at end of file