import unittest from rules.dates import ( DateProcessor ) from rules.times import TimeProcessor from rules.currency import CurrencyProcessor from rules.spaces import CurrencySpaceProcessor from rules.units import UnitProcessor from rules.fraction import FractionProcessor from rules.phone import PhoneNumberProcessor from rules.cards import CardNumberProcessor from rules.postal_national_id import PostalCodeAndNationalIdProcessor from rules.symbols import ( AmpersandProcessor, MathOperatorProcessor, NegativeSignProcessor, RangeProcessor, RomanNumeralProcessor, roman_to_int, ) from normalizer import Normalizer class MyTestCase(unittest.TestCase): def setUp(self): self.date_processor = DateProcessor() self.time_processor = TimeProcessor() self.normalizer = Normalizer() def test_not_a_date(self): inputs = ['1380', 'hello', '21 21 21', '12, 8, 20', "Sep 28 2020"] for inp in inputs: self.assertFalse(self.date_processor.is_date(inp), f'{inp} is not a date') def test_date_to_text_gregorian(self): inputs = ["9-28-2020", "1/1/1998", "2002/07/18"] corrects = ["twenty-eighth of September, twenty twenty".lower(), "first of January, nineteen ninety-eight".lower(), "eighteenth of July, two thousand and two".lower()] normalized = [] for i, c in zip(inputs, corrects): text = self.date_processor.process(i, language='en') normalized.append(text.lower()) self.assertListEqual(normalized, corrects) def test_date_to_text_jalali(self): inputs = ['1381/4/31', '۱۳۸۱/۰۴/27', "29/12/1305"] corrects = ["سی و یک چهار یک هزار و سیصد و هشتاد و یک", "بیست و هفت چهار یک هزار و سیصد و هشتاد و یک", "بیست و نه دوازده یک هزار و سیصد و پنج"] normalized = [] for i, c in zip(inputs, corrects): text = self.date_processor.process(i, language='fa') normalized.append(text.strip()) self.assertListEqual(normalized, corrects) def test_englih_time(self): inputs = ['1:35', '5:50', '15:20', '10:00:11', '3:12:01'] corrects = [ 'one thirty-five', 'five fifty', 'three twenty pm', 'ten and eleven seconds', 'three twelve and one second'] normalized = [] for i, c in zip(inputs, corrects): text = self.time_processor.process(i, language='en') normalized.append(text.lower()) self.assertListEqual(normalized, corrects) def test_farsi_time(self): inputs = ['1:35', '5:50', '15:20', '10:00:11', '3:12:01'] corrects = [ 'یک و سی و پنج دقیقه', 'پنج و پنجاه دقیقه', 'پانزده و بیست دقیقه', 'ده و یازده ثانیه', 'سه و دوازده دقیقه و یک ثانیه'] normalized = [] for i, c in zip(inputs, corrects): text = self.time_processor.process(i, language='fa') normalized.append(text.lower()) self.assertListEqual(normalized, corrects) def test_overall_normalization(self): inputs = [ "I was born on 2002/07/18 when it was pouring.", "من متولد سال ۱۳۸۱/۴/۲۷ هستم و الان بیست و چهار سال دارم.", "I have a date at 3:30 on 9/10/2026 and I have to be ready.", "در تاریخ ۱۲/۶/۱۳۸۸ و در ساعت ۲:۵۱:۰۳ اتفاق افتاد." ] correct = [ "I was born on eighteenth of July, two thousand and two when it was pouring.".lower(), "من متولد سال بیست و هفت چهار یک هزار و سیصد و هشتاد و یک هستم و الان بیست و چهار سال دارم.", "I have a date at three thirty on tenth of September, twenty twenty-six and I have to be ready.".lower(), "در تاریخ دوازده شش یک هزار و سیصد و هشتاد و هشت و در ساعت دو و پنجاه و یک دقیقه و سه ثانیه اتفاق افتاد." ] normalized = [] for i, c in zip(inputs, correct): text = self.normalizer.process(i) normalized.append(text.lower()) self.assertListEqual(normalized, correct) def test_overall_normalization_currency(self): inputs = [ "قیمت امروز ارز، ۳$ بود", "قیمت امروز ارز، 3$ بود", "I have 3$", "I have 1 $", "قیمت امروز ارز، ۳ $ بود", "I want $12.50 today.", "من ۱۲.۵۰$ می‌خواهم", "It costs €12.50 or 10 £.", "قیمت ۵ € و £۲.۵ است", "۱۰۰۰$", "1000$", "۱۰۰۰ $", "1000 $", ] correct = [ "قیمت امروز ارز، سه دلار بود", "قیمت امروز ارز، سه دلار بود", "I have three dollars", "I have one dollar", "قیمت امروز ارز، سه دلار بود", "I want twelve dollars and fifty cents today.", "من دوازده دلار و پنجاه سنت می‌خواهم", "It costs twelve euros and fifty cents or ten pounds.", "قیمت پنج یورو و دو پوند و پنجاه پنس است", "یک هزار دلار", "one thousand dollars", "یک هزار دلار", "one thousand dollars", ] normalized = [self.normalizer.process(i) for i in inputs] self.assertListEqual(normalized, correct) def test_overall_normalization_units(self): inputs = [ "I ran 5km today", "I ran 5 km today", "وزن او ۷۵ kg است", "من ۱۰km دویدم", "It is 25°C outside", "دما ۲۵ °C است", "The car goes 60 km/h and weighs 1 kg", "از اینجا تا آنجا ۱۰۰۰.۵m فاصله است", "The movie is 1.5 hours long", "این کار ۱۰ min طول می‌کشد", "۱۰۰۰m", "1000m", ] correct = [ "I ran five kilometres today", "I ran five kilometres today", "وزن او هفتاد و پنج کیلوگرم است", "من ده کیلومتر دویدم", "It is twenty-five degrees Celsius outside", "دما بیست و پنج درجه سانتیگراد است", "The car goes sixty kilometres per hour and weighs one kilogram", "از اینجا تا آنجا یک هزار متر و پنجاه سانتیمتر فاصله است", "The movie is one hour and thirty minutes long", "این کار ده دقیقه طول می‌کشد", "یک هزار متر", "one thousand metres", ] normalized = [self.normalizer.process(i) for i in inputs] self.assertListEqual(normalized, correct) def test_overall_normalization_units_and_currency_together(self): inputs = [ "It costs 3$ per 2 kg", "قیمت هر ۲ kg برابر ۳$ است", ] correct = [ "It costs three dollars per two kilograms", "قیمت هر دو کیلوگرم برابر سه دلار است", ] normalized = [self.normalizer.process(i) for i in inputs] self.assertListEqual(normalized, correct) def test_overall_normalization_fractions(self): inputs = [ "I drank 1/3 of the bottle", "من ۱/۳ بطری نوشیدم", "Add 2 1/2 cups of flour", "۱/۳", "1/3", "من ۵km و ۱/۳ بطری", "۱/۲ kg", "1,000/2 kg", "Add 1/2 kg of rice", "۲ ۱/۲ kg برنج", "2 %", "۲ ٪", "Discount is 20% on 3$", "تخفیف ۲۰٪ روی ۳$ است", "از ۱/۲ kg برنج ۲.۵٪ باقی ماند", "٪ ۵", "% 5", "5% kg", "۵٪ kg", "% 20 cm", "٪ ۲۰ cm", ] correct = [ "I drank one third of the bottle", "من یک سوم بطری نوشیدم", "Add two and one half cups of flour", "یک سوم", "one third", "من پنج کیلومتر و یک سوم بطری", "یک دوم کیلوگرم", "one thousand halves kilograms", "Add one half kilogram of rice", "دو و یک دوم کیلوگرم برنج", "two percent", "دو درصد", "Discount is twenty percent on three dollars", "تخفیف بیست درصد روی سه دلار است", "از یک دوم کیلوگرم برنج دو و پنج دهم درصد باقی ماند", "پنج درصد", "five percent", "five percent kilogram", "پنج درصد کیلوگرم", "twenty percent centimetre", "بیست درصد سانتیمتر", ] normalized = [self.normalizer.process(i) for i in inputs] self.assertListEqual(normalized, correct) def test_overall_normalization_roman_numerals_and_ampersand(self): inputs = [ "Henry VIII was king", "Tom & Jerry read Chapter IV", "قرن XX و R&D", "تام & جری فصل IV را خواندند", "جنگ جهانی II", "Q&A about World War I", "I think Washington DC is nice", "3 & 4", "۳ & ۴", "1/2 & 1/3", "۱/۲ & ۱/۳", "5 km & 3 kg", "۵ km & ۳ kg", "5 & 3 kg", "5% & 10%", ] correct = [ "Henry eight was king", "Tom and Jerry read Chapter four", "قرن بیست و R و D", "تام و جری فصل چهار را خواندند", "جنگ جهانی دو", "Q and A about World War one", "I think Washington DC is nice", "three and four", "سه و چهار", "one half and one third", "یک دوم و یک سوم", "five kilometres and three kilograms", "پنج کیلومتر و سه کیلوگرم", "five and three kilograms", "five percent and ten percent", ] normalized = [self.normalizer.process(i) for i in inputs] self.assertListEqual(normalized, correct) def test_overall_normalization_negative_sign_and_operators(self): inputs = [ "-2 m", "- 2 m", "-۲ m", "-2", "-۲", "The temp is -2 degrees", "دما -۲ درجه است", "-1/2 kg", "-5%", "-$5", "x = -2", "3 + 4", "۳ + ۴", "3+4", "6 × 2", "۶ × ۲", "8 ÷ 2", "۸ ÷ ۲", "7 = 7", "۷ = ۷", "3 + 4 = 7", "۳ + ۴ = ۷", "5 - 3 = 2", "۵ - ۳ = ۲", "2 m + 3 m", "1/2 + 1/3", "$3 + $4", "3 & 4 = 7", "5 - 3", "10 - 20 km", "C++ is great", ] correct = [ "negative two metres", "negative two metres", "منفی دو متر", "negative two", "منفی دو", "The temp is negative two degrees", "دما منفی دو درجه است", "negative one half kilogram", "negative five percent", "negative five dollars", "x equals negative two", "three plus four", "سه بعلاوه چهار", "three plus four", "six times two", "شش ضربدر دو", "eight divided by two", "هشت تقسیم بر دو", "seven equals seven", "هفت مساوی است با هفت", "three plus four equals seven", "سه بعلاوه چهار مساوی است با هفت", "five minus three equals two", "پنج منهای سه مساوی است با دو", "two metres plus three metres", "one half plus one third", "three dollars plus four dollars", "three and four equals seven", "five to three", "ten to twenty kilometres", "C++ is great", ] for inp, expected in zip(inputs, correct): self.assertEqual(self.normalizer.process(inp), expected, inp) def test_overall_normalization_ranges(self): inputs = [ "3 - 5", "3-5", "3 – 5", "3–5", "pages 3-5", "2.5 - 3.5", "3kg - 5 kg", "3 kg-5 kg", "3 - 5 kg", "3 kg - 5", "10 - 20 km", "$3 - $5", "3$ - 5$", "1/2 - 3/4", "5% - 10%", "10 - 20%", "1 - 2 - 3", "Is it 3 - 5?", "۳ - ۵", "۳-۵", "۳kg - ۵ kg", "۱۰ - ۲۰ km", "از ۳ - ۵ ساعت", "۵٪ - ۱۰٪", "5 - 3 = 2", "2024-05", "0912-123-4567", "1990-2000", "well - maybe", ] correct = [ "three to five", "three to five", "three to five", "three to five", "pages three to five", "two point five to three point five", "three kilograms to five kilograms", "three kilograms to five kilograms", "three to five kilograms", "three kilograms to five", "ten to twenty kilometres", "three dollars to five dollars", "three dollars to five dollars", "one half to three quarters", "five percent to ten percent", "ten to twenty percent", "one to two to three", "Is it three to five?", "سه تا پنج", "سه تا پنج", "سه کیلوگرم تا پنج کیلوگرم", "ده تا بیست کیلومتر", "از سه تا پنج ساعت", "پنج درصد تا ده درصد", "five minus three equals two", "2024-05", "zero nine one two, one two three, four five, six seven", "1990-2000", "well - maybe", ] for inp, expected in zip(inputs, correct): self.assertEqual(self.normalizer.process(inp), expected, inp) def test_overall_normalization_wrapped_symbols(self): cases = { "2 (kg)": "two (kilograms)", "2(kg)": "two (kilograms)", "2 [kg]": "two [kilograms]", "3 {m}": "three {metres}", '3 "m"': 'three "metres"', "3 “m”": "three “metres”", "(2 kg)": "(two kilograms)", "(5) kg": "(five) kilograms", "(1/2) kg": "(one half) kilogram", "50 (%)": "fifty (percent)", "2 (km/h)": "two (kilometres per hour)", "2 (kg) and 3 (m)": "two (kilograms) and three (metres)", "-2 (m)": "negative two (metres)", "chapter (V)": "chapter (five)", "Chapter [V]": "Chapter [five]", "Part {I}": "Part {one}", 'Chapter "IV"': 'Chapter "four"', "Henry (VIII)": "Henry (eight)", "I think (I) am": "I think (I) am", "Tom (&) Jerry": "Tom (and) Jerry", "3 (&) 4": "three (and) four", "(3) & (4)": "(three) and (four)", "(3) + (4)": "(three) plus (four)", "3 (+) 4": "three (plus) four", "(3) - (5)": "(three) to (five)", "3 (cm) - 5 (cm)": "three (centimetres) to five (centimetres)", "(2) kg - (5) kg": "(two) kilograms to (five) kilograms", "2 (kg) + 3 (kg)": "two (kilograms) plus three (kilograms)", "3 (a)": "3 (a)", "Section 2 (b)": "Section 2 (b)", "۲ (kg)": "دو (کیلوگرم)", "۳ {m}": "سه {متر}", "۲ (kg) و ۳ (m)": "دو (کیلوگرم) و سه (متر)", "فصل (V)": "فصل (پنج)", "فصل «V»": "فصل «پنج»", "(۳) + (۴)": "(سه) بعلاوه (چهار)", "(۳) - (۵)": "(سه) تا (پنج)", "۳ (+) ۴": "سه (بعلاوه) چهار", "۳ (kg) - ۵ (kg)": "سه (کیلوگرم) تا پنج (کیلوگرم)", } for inp, expected in cases.items(): self.assertEqual(self.normalizer.process(inp), expected, inp) def test_overall_normalization_phone_numbers(self): mobile_en = "zero nine one two, four three three, eight one, three five" mobile_plus_en = "plus nine eight, nine one two, four three three, eight one, three five" landline_en = "zero two one, eight eight, zero six, nine eight, nine five" mobile_fa = "صفر نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج" mobile_plus_fa = "مثبت نود و هشت، نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج" landline_fa = "صفر بیست و یک، هشتاد و هشت، صفر شش، نود و هشت، نود و پنج" cases = { "09124338135": mobile_en, "+989124338135": mobile_plus_en, "+98-912-4338135": mobile_plus_en, "+98 912 433 8135": mobile_plus_en, "021-88069895": landline_en, "(021) 88069895": "(zero two one), eight eight, zero six, nine eight, nine five", "(0915)6486320": "(zero nine one five), six four eight, six three, two zero", "Call me at 09124338135.": f"Call me at {mobile_en}.", "Call 09124338135, then 021-88069895.": f"Call {mobile_en}, then {landline_en}.", "tel: 88069895": "tel: eight eight, zero six, nine eight, nine five", "۰۹۱۲۴۳۳۸۱۳۵": mobile_fa, "+۹۸-۹۱۲-۴۳۳۸۱۳۵": mobile_plus_fa, "۰۲۱-۸۸۰۶۹۸۹۵": landline_fa, "شماره من ۰۹۱۲۴۳۳۸۱۳۵ است": f"شماره من {mobile_fa} است", "۰۰۱-۸۸۵۲۳۰۰۶": "دو صفر یک، هشتاد و هشت، پنجاه و دو، سی، صفر شش", "با 09124338135 تماس بگیرید": f"با {mobile_fa} تماس بگیرید", "تلفن: ۸۸۰۶۹۸۹۵": "تلفن: هشتاد و هشت، صفر شش، نود و هشت، نود و پنج", "شماره تماس ۰۲۱-۸۸۰۶۹۸۹۵ و ۰۹۱۲۴۳۳۸۱۳۵": f"شماره تماس {landline_fa} و {mobile_fa}", "4338135": "4338135", "call 5 times": "call 5 times", } for inp, expected in cases.items(): self.assertEqual(self.normalizer.process(inp), expected, inp) class CurrencyProcessorTestCase(unittest.TestCase): def setUp(self): self.processor = CurrencyProcessor() def test_is_currency(self): valid = ['3$', '۳$', '3.5$', '1,000$', '(3$)', '3$.', '$3', '$12.50', '$12.50.', '€3', '3€', '£12.50', '۳£'] invalid = ['3', '$', 'hello', '3$a', '3 $', '3$$', '$3$', '$$3', '€3£', '3€€'] for inp in valid: self.assertTrue(self.processor.is_currency(inp), f'{inp} is a currency') for inp in invalid: self.assertFalse(self.processor.is_currency(inp), f'{inp} is not a currency') def test_persian(self): inputs = ['۳$', '3$', '$3', '۱۲۰$', '1,000$'] corrects = ['سه دلار', 'سه دلار', 'سه دلار', 'یکصد و بیست دلار', 'یک هزار دلار'] normalized = [self.processor.process(i, language='fa') for i in inputs] self.assertListEqual(normalized, corrects) def test_english(self): inputs = ['3$', '۳$', '$3', '1$', '0$', '25$', '1,000$'] corrects = [ 'three dollars', 'three dollars', 'three dollars', 'one dollar', 'zero dollars', 'twenty-five dollars', 'one thousand dollars', ] normalized = [self.processor.process(i, language='en') for i in inputs] self.assertListEqual(normalized, corrects) def test_english_dollars_and_cents(self): inputs = ['$12.50', '12.5$', '12.50$', '۱۲.۵۰$', '$1.01', '$12.05', '$0.50', '$0.01', '$12.00', '$1,000.75'] corrects = [ 'twelve dollars and fifty cents', 'twelve dollars and fifty cents', 'twelve dollars and fifty cents', 'twelve dollars and fifty cents', 'one dollar and one cent', 'twelve dollars and five cents', 'fifty cents', 'one cent', 'twelve dollars', 'one thousand dollars and seventy-five cents', ] normalized = [self.processor.process(i, language='en') for i in inputs] self.assertListEqual(normalized, corrects) def test_persian_dollars_and_cents(self): inputs = ['$12.50', '12.5$', '۱۲٫۵۰$', '$12.05', '$0.50', '$12.00'] corrects = [ 'دوازده دلار و پنجاه سنت', 'دوازده دلار و پنجاه سنت', 'دوازده دلار و پنجاه سنت', 'دوازده دلار و پنج سنت', 'پنجاه سنت', 'دوازده دلار', ] normalized = [self.processor.process(i, language='fa') for i in inputs] self.assertListEqual(normalized, corrects) def test_more_than_two_decimals_is_read_as_decimal_number(self): self.assertEqual( self.processor.process('3.141$', language='en'), 'three point one four one dollars', ) def test_dollars_and_cents_keep_surrounding_punctuation(self): self.assertEqual( self.processor.process('($12.50).', language='en'), '(twelve dollars and fifty cents).', ) def test_euro_english(self): inputs = ['3€', '€3', '1€', '€12.50', '12.5€', '€1.01', '€0.50', '€12.00', '۳€'] corrects = [ 'three euros', 'three euros', 'one euro', 'twelve euros and fifty cents', 'twelve euros and fifty cents', 'one euro and one cent', 'fifty cents', 'twelve euros', 'three euros', ] normalized = [self.processor.process(i, language='en') for i in inputs] self.assertListEqual(normalized, corrects) def test_pound_english(self): inputs = ['3£', '£3', '1£', '£12.50', '12.5£', '£1.01', '£0.50', '£0.01', '£12.00'] corrects = [ 'three pounds', 'three pounds', 'one pound', 'twelve pounds and fifty pence', 'twelve pounds and fifty pence', 'one pound and one penny', 'fifty pence', 'one penny', 'twelve pounds', ] normalized = [self.processor.process(i, language='en') for i in inputs] self.assertListEqual(normalized, corrects) def test_euro_persian(self): inputs = ['3€', '€3', '۳€', '€12.50', '€0.50', '€12.00'] corrects = [ 'سه یورو', 'سه یورو', 'سه یورو', 'دوازده یورو و پنجاه سنت', 'پنجاه سنت', 'دوازده یورو', ] normalized = [self.processor.process(i, language='fa') for i in inputs] self.assertListEqual(normalized, corrects) def test_pound_persian(self): inputs = ['3£', '£3', '۳£', '£12.50', '£12.05', '£0.50', '£12.00'] corrects = [ 'سه پوند', 'سه پوند', 'سه پوند', 'دوازده پوند و پنجاه پنس', 'دوازده پوند و پنج پنس', 'پنجاه پنس', 'دوازده پوند', ] normalized = [self.processor.process(i, language='fa') for i in inputs] self.assertListEqual(normalized, corrects) def test_more_than_two_decimals_uses_currency_name(self): self.assertEqual(self.processor.process('3.141€', language='en'), 'three point one four one euros') self.assertEqual(self.processor.process('3.141£', language='en'), 'three point one four one pounds') def test_punctuation_is_preserved_for_all_symbols(self): self.assertEqual(self.processor.process('(€12.50).', language='en'), '(twelve euros and fifty cents).') self.assertEqual(self.processor.process('£3،', language='fa'), 'سه پوند،') def test_surrounding_punctuation_is_preserved(self): self.assertEqual(self.processor.process('3$.', language='en'), 'three dollars.') self.assertEqual(self.processor.process('(3$)', language='en'), '(three dollars)') self.assertEqual(self.processor.process('۳$،', language='fa'), 'سه دلار،') def test_non_currency_is_unchanged(self): inputs = ['hello', '3', '$', 'سلام', '3$a', '$3a'] for inp in inputs: self.assertEqual(self.processor.process(inp, language='en'), inp) self.assertEqual(self.processor.process(inp, language='fa'), inp) def test_unsupported_language_is_unchanged(self): for language in ('mixed', 'de'): self.assertEqual(self.processor.process('3$', language=language), '3$') def test_no_letters_language_is_decided_by_digits(self): no_letters = 'Unknown / No letters found' inputs = ['۱۰۰۰$', '1000$', '$۱۰۰۰', '$1000', '۱۲.۵۰€', '12.50€', '£۳', '£3'] corrects = [ 'یک هزار دلار', 'one thousand dollars', 'یک هزار دلار', 'one thousand dollars', 'دوازده یورو و پنجاه سنت', 'twelve euros and fifty cents', 'سه پوند', 'three pounds', ] normalized = [self.processor.process(i, language=no_letters) for i in inputs] self.assertListEqual(normalized, corrects) def test_epsilon_zero_disables_processor(self): processor = CurrencyProcessor(epsilon=0.0) outputs = {processor.process('3$', language='en') for _ in range(50)} self.assertEqual(outputs, {'3$'}) class CurrencySpaceProcessorTestCase(unittest.TestCase): def setUp(self): self.processor = CurrencySpaceProcessor() def test_removes_space_between_number_and_dollar(self): inputs = ['I have 1 $', 'I have ۳ $', 'it is 2.5 $.', '1,000 $ total', '5 €', '۳ £', '2.5 €.'] corrects = ['I have 1$', 'I have ۳$', 'it is 2.5$.', '1,000$ total', '5€', '۳£', '2.5€.'] normalized = [self.processor.process(i) for i in inputs] self.assertListEqual(normalized, corrects) def test_already_adjacent_is_unchanged(self): self.assertEqual(self.processor.process('I have 1$'), 'I have 1$') def test_unrelated_dollar_signs_are_unchanged(self): inputs = ['1 $5', 'price $ 5', 'hello $', 'a1 $', '1 $a', '1\n$', '1 €5', 'price £ 5', '1 £a'] for inp in inputs: self.assertEqual(self.processor.process(inp), inp) def test_multiple_occurrences(self): self.assertEqual(self.processor.process('1 $ and 2 $'), '1$ and 2$') def test_epsilon_zero_disables_processor(self): processor = CurrencySpaceProcessor(epsilon=0.0) outputs = {processor.process('1 $') for _ in range(50)} self.assertEqual(outputs, {'1 $'}) class UnitProcessorTestCase(unittest.TestCase): def setUp(self): self.processor = UnitProcessor() def test_length_english(self): inputs = ['5km', '5 km', '1 km', '2.5 km', '10 m', '3 cm', '7 mm', '1 mi', '6 ft', '1 ft'] corrects = [ 'five kilometres', 'five kilometres', 'one kilometre', 'two kilometres and five hundred metres', 'ten metres', 'three centimetres', 'seven millimetres', 'one mile', 'six feet', 'one foot', ] normalized = [self.processor.process(i, language='en') for i in inputs] self.assertListEqual(normalized, corrects) def test_length_persian(self): inputs = ['5km', '۵ km', '1 km', '10 m', '۳ cm', '7 mm', '2 mi'] corrects = [ 'پنج کیلومتر', 'پنج کیلومتر', 'یک کیلومتر', 'ده متر', 'سه سانتیمتر', 'هفت میلیمتر', 'دو مایل', ] normalized = [self.processor.process(i, language='fa') for i in inputs] self.assertListEqual(normalized, corrects) def test_mass_and_volume_english(self): inputs = ['2 kg', '1 kg', '2 KG', '500 g', '250 mg', '3 lb', '3 lbs', '8 oz', '2 l', '2 L', '330 ml', '1 mL'] corrects = [ 'two kilograms', 'one kilogram', 'two kilograms', 'five hundred grams', 'two hundred and fifty milligrams', 'three pounds', 'three pounds', 'eight ounces', 'two litres', 'two litres', 'three hundred and thirty millilitres', 'one millilitre', ] normalized = [self.processor.process(i, language='en') for i in inputs] self.assertListEqual(normalized, corrects) def test_mass_and_volume_persian(self): inputs = ['2 kg', '500 g', '250 mg', '3 lb', '8 oz', '2 l', '330 ml'] corrects = [ 'دو کیلوگرم', 'پانصد گرم', 'دویست و پنجاه میلی‌گرم', 'سه پوند', 'هشت اونس', 'دو لیتر', 'سیصد و سی میلی‌لیتر', ] normalized = [self.processor.process(i, language='fa') for i in inputs] self.assertListEqual(normalized, corrects) def test_time_english(self): inputs = ['10 sec', '1 sec', '5 secs', '5 min', '1 min', '3 mins', '2 h', '1 hr', '2 hrs', '2 hours', '1 hour', '250 ms', '3 days', '1 week', '2 years'] corrects = [ 'ten seconds', 'one second', 'five seconds', 'five minutes', 'one minute', 'three minutes', 'two hours', 'one hour', 'two hours', 'two hours', 'one hour', 'two hundred and fifty milliseconds', 'three days', 'one week', 'two years', ] normalized = [self.processor.process(i, language='en') for i in inputs] self.assertListEqual(normalized, corrects) def test_time_persian(self): inputs = ['10 sec', '۵ min', '2 h', '2 hours', '1 hr', '250 ms', '3 days', '1 week', '2 months', '2 years'] corrects = [ 'ده ثانیه', 'پنج دقیقه', 'دو ساعت', 'دو ساعت', 'یک ساعت', 'دویست و پنجاه میلی‌ثانیه', 'سه روز', 'یک هفته', 'دو ماه', 'دو سال', ] normalized = [self.processor.process(i, language='fa') for i in inputs] self.assertListEqual(normalized, corrects) def test_time_decimals_are_split_into_smaller_units(self): cases_en = { '1.5 h': 'one hour and thirty minutes', '0.5 h': 'thirty minutes', '2.5 min': 'two minutes and thirty seconds', '1.5 sec': 'one second and five hundred milliseconds', '1.5 days': 'one day and twelve hours', '1.5 weeks': 'one week, three days and twelve hours', '1.5 months': 'one point five months', } for inp, correct in cases_en.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) cases_fa = { '1.5 h': 'یک ساعت و سی دقیقه', '2.5 min': 'دو دقیقه و سی ثانیه', '1.5 sec': 'یک ثانیه و پانصد میلی‌ثانیه', } for inp, correct in cases_fa.items(): self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) def test_speed_and_temperature(self): self.assertEqual(self.processor.process('60 km/h', language='en'), 'sixty kilometres per hour') self.assertEqual(self.processor.process('1 m/s', language='en'), 'one metre per second') self.assertEqual(self.processor.process('60 km/h', language='fa'), 'شصت کیلومتر بر ساعت') self.assertEqual(self.processor.process('10 m/s', language='fa'), 'ده متر بر ثانیه') self.assertEqual(self.processor.process('25°C', language='en'), 'twenty-five degrees Celsius') self.assertEqual(self.processor.process('1 °F', language='en'), 'one degree Fahrenheit') self.assertEqual(self.processor.process('25 °C', language='fa'), 'بیست و پنج درجه سانتیگراد') self.assertEqual(self.processor.process('۷۷°F', language='fa'), 'هفتاد و هفت درجه فارنهایت') def test_thousand_separators(self): self.assertEqual(self.processor.process('1,000 m', language='en'), 'one thousand metres') self.assertEqual(self.processor.process('۱۰۰۰ m', language='fa'), 'یک هزار متر') def test_decimals_are_split_into_smaller_units(self): cases_en = { '1000.5 m': 'one thousand metres and fifty centimetres', '2.5 km': 'two kilometres and five hundred metres', '2.75 kg': 'two kilograms and seven hundred and fifty grams', '1.5 L': 'one litre and five hundred millilitres', '2.5 cm': 'two centimetres and five millimetres', '1.01 m': 'one metre and one centimetre', '0.5 kg': 'five hundred grams', '1.234 m': 'one metre, twenty-three centimetres and four millimetres', '2.0 kg': 'two kilograms', } for inp, correct in cases_en.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) cases_fa = { '۱۰۰۰.۵m': 'یک هزار متر و پنجاه سانتیمتر', '2.5 km': 'دو کیلومتر و پانصد متر', '2.75 kg': 'دو کیلوگرم و هفتصد و پنجاه گرم', '0.5 kg': 'پانصد گرم', '1.234 m': 'یک متر و بیست و سه سانتیمتر و چهار میلیمتر', } for inp, correct in cases_fa.items(): self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) def test_decimals_without_smaller_unit_are_read_as_plain_numbers(self): # `mi` has no smaller unit, and 1.2345 m does not end on a whole millimetre. self.assertEqual(self.processor.process('2.5 mi', language='en'), 'two point five miles') self.assertEqual(self.processor.process('2.5 mi', language='fa'), 'دو و پنج دهم مایل') self.assertEqual(self.processor.process('36.6 °C', language='en'), 'thirty-six point six degrees Celsius') self.assertEqual( self.processor.process('1.2345 m', language='en'), 'one point two three four five metres', ) def test_fraction_amounts_english(self): cases = { '1/2 kg': 'one half kilogram', '1/2kg': 'one half kilogram', '3/4 km': 'three quarters kilometre', '1/3 L': 'one third litre', '3/2 kg': 'three halves kilograms', '1,000/2 kg': 'one thousand halves kilograms', '2 1/2 kg': 'two and one half kilograms', '1 1/2 hours': 'one and one half hours', '½ kg': 'one half kilogram', '1½ kg': 'one and one half kilograms', '1/2 m/s': 'one half metre per second', '3/4 km/h': 'three quarters kilometre per hour', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) def test_fraction_amounts_persian(self): cases = { '۱/۲ kg': 'یک دوم کیلوگرم', '۱/۲kg': 'یک دوم کیلوگرم', '۳/۴ km': 'سه چهارم کیلومتر', '1/3 L': 'یک سوم لیتر', '۱٬۰۰۰/۲ kg': 'یک هزار دوم کیلوگرم', '۲ ۱/۲ kg': 'دو و یک دوم کیلوگرم', '½ kg': 'یک دوم کیلوگرم', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) def test_fraction_amounts_language_from_digits(self): self.assertEqual(self.processor.process('۱/۲ kg', language='en'), 'یک دوم کیلوگرم') self.assertEqual(self.processor.process('1/2 kg', language=None), 'one half kilogram') def test_invalid_fraction_with_unit_is_unchanged(self): for inp in ('1/0 kg', '1/1 kg', '3/1 km'): self.assertEqual(self.processor.process(inp, language='en'), inp, inp) def test_percent_english(self): cases = { '2 %': 'two percent', '2%': 'two percent', '1%': 'one percent', '100%': 'one hundred percent', '2 ٪': 'two percent', '2.5%': 'two point five percent', '0.5 %': 'zero point five percent', '1/2 %': 'one half percent', '2 1/2%': 'two and one half percent', '1,000%': 'one thousand percent', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) def test_percent_persian(self): cases = { '۲ ٪': 'دو درصد', '۲٪': 'دو درصد', '۲ %': 'دو درصد', '2 %': 'دو درصد', '۱۰۰٪': 'یکصد درصد', '۲٫۵٪': 'دو و پنج دهم درصد', '۱/۲ ٪': 'یک دوم درصد', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) def test_percent_sign_before_the_number(self): self.assertEqual(self.processor.process('%20', language='en'), 'twenty percent') self.assertEqual(self.processor.process('Save %20 today', language='en'), 'Save twenty percent today') self.assertEqual(self.processor.process('٪۲۰', language='fa'), 'بیست درصد') self.assertEqual(self.processor.process('تخفیف %۲۰ است', language='fa'), 'تخفیف بیست درصد است') def test_percent_sign_before_the_number_with_space(self): self.assertEqual(self.processor.process('% 5', language='en'), 'five percent') self.assertEqual(self.processor.process('% 5', language='en'), 'five percent') self.assertEqual(self.processor.process('Save % 20 today', language='en'), 'Save twenty percent today') self.assertEqual(self.processor.process('% 2.5', language='en'), 'two point five percent') self.assertEqual(self.processor.process('٪ ۵', language='fa'), 'پنج درصد') self.assertEqual(self.processor.process('تخفیف ٪ ۲۰ است', language='fa'), 'تخفیف بیست درصد است') def test_number_before_the_sign_wins_over_prefix_reading(self): # In `5 % 2` the sign belongs to the 5. self.assertEqual(self.processor.process('5 % 2', language='en'), 'five percent 2') def test_unit_after_percent_is_read_as_singular(self): cases_en = { '5% kg': 'five percent kilogram', '5 % kg': 'five percent kilogram', '2.5% m': 'two point five percent metre', '5% km/h': 'five percent kilometre per hour', '1/2 % kg': 'one half percent kilogram', '% 20 cm': 'twenty percent centimetre', '%20 cm': 'twenty percent centimetre', '% 2.5 kg': 'two point five percent kilogram', '٪ 5 kg': 'five percent kilogram', } for inp, correct in cases_en.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) self.assertEqual(self.processor.process('۵٪ kg', language='fa'), 'پنج درصد کیلوگرم') self.assertEqual(self.processor.process('۵ ٪ km', language='fa'), 'پنج درصد کیلومتر') self.assertEqual(self.processor.process('٪ ۲۰ cm', language='fa'), 'بیست درصد سانتیمتر') def test_symbol_after_another_unit_is_not_consumed(self): self.assertEqual(self.processor.process('5 km kg', language='en'), 'five kilometres kg') self.assertEqual(self.processor.process('5 km 3 kg', language='en'), 'five kilometres three kilograms') self.assertEqual(self.processor.process('10% of 2 kg', language='en'), 'ten percent of two kilograms') def test_percent_language_from_digits(self): self.assertEqual(self.processor.process('۲٪', language='en'), 'دو درصد') self.assertEqual(self.processor.process('2%', language=None), 'two percent') def test_percent_mixed_with_other_units(self): self.assertEqual( self.processor.process('Sales rose 5% and 2 kg of rice', language='en'), 'Sales rose five percent and two kilograms of rice', ) self.assertEqual( self.processor.process('فروش ۵٪ بیشتر شد و ۲ kg برنج', language='fa'), 'فروش پنج درصد بیشتر شد و دو کیلوگرم برنج', ) self.assertEqual( self.processor.process('5 km, 2 kg and 10 %.', language='en'), 'five kilometres, two kilograms and ten percent.', ) def test_percent_sign_without_a_number_is_unchanged(self): inputs = ['%', '٪', 'hello %', 'x % y', '% 1/2', 'printf %d and %s', '% d', '% 5d', 'a%20b', '%20b', '100%a', '1/0 % kg'] for inp in inputs: self.assertEqual(self.processor.process(inp, language='en'), inp, inp) def test_wrapped_units_and_amounts(self): cases = { '2 (kg)': 'two (kilograms)', '2(kg)': 'two (kilograms)', '2 [kg]': 'two [kilograms]', '3 {m}': 'three {metres}', '3 "m"': 'three "metres"', '2 ( kg )': 'two (kilograms)', '(5) kg': '(five) kilograms', '(1/2) kg': '(one half) kilogram', '50 (%)': 'fifty (percent)', '2.5 (kg)': 'two (kilograms) and five hundred (grams)', '2 (kg) and 3 (m)': 'two (kilograms) and three (metres)', '(2 kg)': '(two kilograms)', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) self.assertEqual(self.processor.process('۲ (kg)', language='fa'), 'دو (کیلوگرم)') self.assertEqual(self.processor.process('۳ {m}', language='fa'), 'سه {متر}') def test_wrapped_non_units_are_unchanged(self): for inp in ('3 (a)', '2 (kgx)', '2 (kg', '(5)', '3 (hello)'): self.assertEqual(self.processor.process(inp, language='en'), inp, inp) def test_multiple_units_and_punctuation(self): self.assertEqual( self.processor.process('I ran 5 km and lifted 3 kg.', language='en'), 'I ran five kilometres and lifted three kilograms.', ) self.assertEqual(self.processor.process('(5km),', language='en'), '(five kilometres),') def test_not_a_unit_is_unchanged(self): inputs = [ 'hello', 'km', '5', '5 s', '1990s', '3d', '5 mn', '5G', 'A5 km', '5 km²', '5 kmh', 'size is 5 in', '5 KM', '5 K', ] for inp in inputs: self.assertEqual(self.processor.process(inp, language='en'), inp) def test_language_ignores_unit_symbols(self): # `km` / `mm` are Latin letters, but the sentence itself is Persian. self.assertEqual( self.processor.process('من 10 km 20 mm', language='en'), 'من ده کیلومتر بیست میلیمتر', ) def test_mixed_language_is_unchanged(self): self.assertEqual(self.processor.process('hi ما 5 km', language='en'), 'hi ما 5 km') def test_unsupported_fallback_language_is_unchanged(self): for language in ('mixed', 'de'): self.assertEqual(self.processor.process('5 km', language=language), '5 km') def test_only_units_in_text_is_decided_by_digits(self): # Normalizer detects "en" here because `m` / `km` are Latin letters. inputs = ['۱۰۰۰m', '۱۰۰۰ m', '1000m', '1000 m', '۵ km', '5 km', '۲.۵ kg', '2.5 kg'] corrects = [ 'یک هزار متر', 'یک هزار متر', 'one thousand metres', 'one thousand metres', 'پنج کیلومتر', 'five kilometres', 'دو کیلوگرم و پانصد گرم', 'two kilograms and five hundred grams', ] for language in ('en', None, 'Unknown / No letters found'): normalized = [self.processor.process(i, language=language) for i in inputs] self.assertListEqual(normalized, corrects, language) def test_epsilon_zero_disables_processor(self): processor = UnitProcessor(epsilon=0.0) outputs = {processor.process('5 km', language='en') for _ in range(50)} self.assertEqual(outputs, {'5 km'}) class FractionProcessorTestCase(unittest.TestCase): def setUp(self): self.processor = FractionProcessor() def test_english(self): cases = { '1/3': 'one third', '2/3': 'two thirds', '1/2': 'one half', '3/2': 'three halves', '1/4': 'one quarter', '3/4': 'three quarters', '5/8': 'five eighths', '7/10': 'seven tenths', '1/21': 'one twenty-first', '1/100': 'one hundredth', '3/100': 'three hundredths', '1/1000': 'one thousandth', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) def test_persian(self): cases = { '۱/۳': 'یک سوم', '۲/۳': 'دو سوم', '۱/۲': 'یک دوم', '۳/۴': 'سه چهارم', '۵/۸': 'پنج هشتم', '۷/۱۰': 'هفت دهم', '۱/۲۳': 'یک بیست و سوم', '۱/۳۰': 'یک سی\u200cام', '۱/۱۰۰': 'یک صدم', '۳/۱۰۰': 'سه صدم', '۱/۱۰۰۰': 'یک هزارم', '1/3': 'یک سوم', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) def test_mixed_numbers(self): cases_en = { '2 1/2': 'two and one half', '1 3/4': 'one and three quarters', '10 2/3': 'ten and two thirds', } for inp, correct in cases_en.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) cases_fa = { '۲ ۱/۲': 'دو و یک دوم', '۱ ۳/۴': 'یک و سه چهارم', } for inp, correct in cases_fa.items(): self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) def test_vulgar_fractions(self): cases_en = { '½': 'one half', '¾': 'three quarters', '⅔': 'two thirds', '⅞': 'seven eighths', '1½': 'one and one half', '2 ¼': 'two and one quarter', } for inp, correct in cases_en.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) self.assertEqual(self.processor.process('½', language='fa'), 'یک دوم') self.assertEqual(self.processor.process('۱½', language='fa'), 'یک و یک دوم') def test_thousands_separators(self): cases_en = { '1,000/2': 'one thousand halves', '12,345/3': 'twelve thousand, three hundred and forty-five thirds', '1,000 1/2': 'one thousand and one half', } for inp, correct in cases_en.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) self.assertEqual(self.processor.process('۱٬۰۰۰/۲', language='fa'), 'یک هزار دوم') self.assertEqual(self.processor.process('۱,۰۰۰/۲', language='fa'), 'یک هزار دوم') def test_fractions_inside_a_sentence_and_punctuation(self): self.assertEqual( self.processor.process('I ate 1/3 and 2/3 more.', language='en'), 'I ate one third and two thirds more.', ) self.assertEqual(self.processor.process('(1/3).', language='en'), '(one third).') self.assertEqual(self.processor.process('1/2.', language='en'), 'one half.') self.assertEqual( self.processor.process('من ۱/۳ و ۲/۳ خوردم', language='fa'), 'من یک سوم و دو سوم خوردم', ) def test_not_a_fraction_is_unchanged(self): inputs = [ '1402/05/10', '2024/05/10', '10/05/2024', '3.1/2', '1/2.5', '1/0', '3/1', 'and/or', '1/2$', '1/2%', 'a1/2', '1/2a', '60 km/h', '12', 'hello', '1,00/2', '1,2/3', ] for inp in inputs: self.assertEqual(self.processor.process(inp, language='en'), inp, inp) self.assertEqual(self.processor.process(inp, language='fa'), inp, inp) def test_no_letters_language_is_decided_by_digits(self): no_letters = 'Unknown / No letters found' for language in (None, no_letters): self.assertEqual(self.processor.process('۱/۳', language=language), 'یک سوم') self.assertEqual(self.processor.process('1/3', language=language), 'one third') self.assertEqual(self.processor.process('2 1/2', language=language), 'two and one half') def test_unsupported_language_is_unchanged(self): for language in ('mixed', 'de'): self.assertEqual(self.processor.process('1/3', language=language), '1/3') def test_epsilon_zero_disables_processor(self): processor = FractionProcessor(epsilon=0.0) outputs = {processor.process('1/3', language='en') for _ in range(50)} self.assertEqual(outputs, {'1/3'}) class RomanNumeralProcessorTestCase(unittest.TestCase): def setUp(self): self.processor = RomanNumeralProcessor() def test_roman_to_int(self): valid = {'I': 1, 'IV': 4, 'IX': 9, 'XIV': 14, 'XL': 40, 'XCIX': 99, 'MCMXC': 1990, 'MMXXIV': 2024, 'MMMCMXCIX': 3999} for token, value in valid.items(): self.assertEqual(roman_to_int(token), value, token) for token in ('IIII', 'VX', 'IC', 'IIV', 'MMMM', 'ABC', 'iv', ''): self.assertIsNone(roman_to_int(token), token) def test_english(self): cases = { 'Henry VIII was king': 'Henry eight was king', 'Louis XIV and Louis XVI': 'Louis fourteen and Louis sixteen', 'Chapter IV': 'Chapter four', 'World War II': 'World War two', 'World War I': 'World War one', 'Part V and Section X': 'Part five and Section ten', 'Napoleon III': 'Napoleon three', 'MMXXIV': 'two thousand and twenty-four', 'XIV': 'fourteen', '(XIV).': '(fourteen).', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) def test_persian(self): cases = { 'فصل IV': 'فصل چهار', 'قرن XX': 'قرن بیست', 'قرن XVIII': 'قرن هجده', 'جنگ جهانی II': 'جنگ جهانی دو', 'جنگ جهانی I': 'جنگ جهانی یک', 'بخش V را بخوان': 'بخش پنج را بخوان', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) def test_wrapped_numerals_after_a_trigger_word(self): cases = { 'chapter (V)': 'chapter (five)', 'Chapter [V]': 'Chapter [five]', 'Part {I}': 'Part {one}', 'Chapter "IV"': 'Chapter "four"', 'Chapter ( V )': 'Chapter ( five )', 'Henry (VIII)': 'Henry (eight)', '(XIV)': '(fourteen)', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) self.assertEqual(self.processor.process('فصل (V)', language='fa'), 'فصل (پنج)') self.assertEqual(self.processor.process('فصل «V»', language='fa'), 'فصل «پنج»') # a pronoun or a label in a wrapper is still left alone for inp in ('(I) think so', 'Vitamin (C)', 'Henry (V)'): self.assertEqual(self.processor.process(inp, language='en'), inp, inp) def test_language_ignores_the_numerals(self): # `XVIII` has more letters than the Persian words, but the sentence is Persian. self.assertEqual(self.processor.process('قرن XVIII', language='en'), 'قرن هجده') self.assertEqual(self.processor.process('Chapter IV', language='fa'), 'Chapter four') def test_pronoun_and_labels_are_unchanged(self): inputs = [ 'I think so', 'After the war I went home', 'Vitamin C', 'Hepatitis C', 'Appendix C', 'Part D', 'Section L', 'Henry V', 'Washington DC', 'Dr. Smith, MD', 'CD player', 'XL shirt', 'MIX the flour', ] for inp in inputs: self.assertEqual(self.processor.process(inp, language='en'), inp, inp) def test_invalid_and_lowercase_are_unchanged(self): for inp in ('IIII', 'VX', 'IC', 'MMMM', 'iv', 'xiv', 'Xiv', 'XIVa', 'aXIV'): self.assertEqual(self.processor.process(inp, language='en'), inp, inp) def test_unsupported_language_is_unchanged(self): for language in ('mixed', 'de', None): self.assertEqual(self.processor.process('XIV', language=language), 'XIV') def test_epsilon_zero_disables_processor(self): processor = RomanNumeralProcessor(epsilon=0.0) outputs = {processor.process('Chapter IV', language='en') for _ in range(50)} self.assertEqual(outputs, {'Chapter IV'}) class AmpersandProcessorTestCase(unittest.TestCase): def setUp(self): self.processor = AmpersandProcessor() def test_english(self): cases = { 'Tom & Jerry': 'Tom and Jerry', 'R&D': 'R and D', 'Q&A': 'Q and A', 'AT&T': 'AT and T', 'Tom &Jerry': 'Tom and Jerry', '& Jerry': 'and Jerry', 'Tom &': 'Tom and', '(R&D)': '(R and D)', 'salt & pepper & oil': 'salt and pepper and oil', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) def test_persian(self): cases = { 'من & تو': 'من و تو', 'تام & جری': 'تام و جری', 'بخش R&D فعال است': 'بخش R و D فعال است', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) def test_numbers_beside_ampersand_english(self): cases = { '3 & 4': 'three and four', '3&4': 'three and four', '1 & 2 & 3': 'one and two and three', '2.5 & 3.5': 'two point five and three point five', '1,000 & 2,000': 'one thousand and two thousand', 'Windows 10 & 11': 'Windows ten and eleven', 'Tom & 2 friends': 'Tom and two friends', 'What is 3 & 4?': 'What is three and four?', '1 & 2.': 'one and two.', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) def test_numbers_beside_ampersand_persian(self): cases = { '۳ & ۴': 'سه و چهار', '3 & 4': 'سه و چهار', '۱ & ۲ & ۳': 'یک و دو و سه', '۲٫۵ & ۳٫۵': 'دو و پنج دهم و سه و پنج دهم', 'سیب ۳ & ۴ پرتقال': 'سیب سه و چهار پرتقال', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) def test_numbers_that_are_part_of_something_else_are_not_read_as_plain_numbers(self): # Only the `&` is read; the dates, times and currency are left for their own processors. cases = { '10:30 & 11:30': '10:30 and 11:30', '2024-05-10 & 2024-05-11': '2024-05-10 and 2024-05-11', '5-6 & 7': '5-6 and seven', 'Q3 & Q4': 'Q3 and Q4', '3rd & 4th': '3rd and 4th', '$3 & $4': '$3 and $4', '3$ & 4$': '3$ and 4$', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) def test_urls_and_non_ampersands_are_unchanged(self): for inp in ('a=1&b=2', 'http://x.com/?a=1&2', '3 && 4', '& 3'): self.assertEqual(self.processor.process(inp, language='en'), inp, inp) def test_wrapped_ampersand_and_numbers(self): cases = { 'Tom (&) Jerry': 'Tom (and) Jerry', 'Tom [&] Jerry': 'Tom [and] Jerry', '3 (&) 4': 'three (and) four', '(3) & (4)': '(three) and (four)', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) self.assertEqual(self.processor.process('من (&) تو', language='fa'), 'من (و) تو') self.assertEqual(self.processor.process('(۳) & (۴)', language='fa'), '(سه) و (چهار)') def test_question_mark_does_not_block_ampersand(self): self.assertEqual(self.processor.process('Is it R&D?', language='en'), 'Is it R and D?') def test_spacing_of_the_rest_of_the_text_is_kept(self): self.assertEqual(self.processor.process('a b & c\nd', language='en'), 'a b and c\nd') def test_not_an_ampersand_word_is_unchanged(self): inputs = ['&', 'a & b', '&', '&', 'a && b', 'a=1&b=2', 'http://x.com?a=1&b=2', 'hello'] for inp in inputs: self.assertEqual(self.processor.process(inp, language='en'), inp, inp) def test_no_letters_language_is_decided_by_digits(self): for language in (None, 'Unknown / No letters found'): self.assertEqual(self.processor.process('1 & 2', language=language), 'one and two') self.assertEqual(self.processor.process('۱ & ۲', language=language), 'یک و دو') def test_unsupported_language_is_unchanged(self): for language in ('mixed', 'de'): self.assertEqual(self.processor.process('Tom & Jerry', language=language), 'Tom & Jerry') def test_epsilon_zero_disables_processor(self): processor = AmpersandProcessor(epsilon=0.0) outputs = {processor.process('Tom & Jerry', language='en') for _ in range(50)} self.assertEqual(outputs, {'Tom & Jerry'}) class NegativeSignProcessorTestCase(unittest.TestCase): def setUp(self): self.processor = NegativeSignProcessor() def test_bare_numbers_are_read_as_words(self): cases_en = { '-2': 'negative two', '−2': 'negative two', '(-2)': '(negative two)', '= -2': '= negative two', 'The temp is -2 degrees': 'The temp is negative two degrees', '-2.5': 'negative two point five', '-1,000': 'negative one thousand', '-2.': 'negative two.', } for inp, correct in cases_en.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) self.assertEqual(self.processor.process('-۲', language='fa'), 'منفی دو') self.assertEqual(self.processor.process('دما -۲ درجه است', language='fa'), 'دما منفی دو درجه است') def test_number_with_unit_fraction_percent_or_currency_is_left_for_their_processors(self): cases = { '-2 m': 'negative 2 m', '-2m': 'negative 2m', '- 2 m': 'negative 2 m', '-1/2 kg': 'negative 1/2 kg', '-5%': 'negative 5%', '-$5': 'negative $5', '-5$': 'negative 5$', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) # The language comes from the Persian digits, not from the Latin `m`. self.assertEqual(self.processor.process('-۲ m', language='en'), 'منفی ۲ m') self.assertEqual(self.processor.process('- ۲ m', language='en'), 'منفی ۲ m') def test_space_after_minus_needs_a_unit_or_an_operator(self): self.assertEqual(self.processor.process('3 + - 2', language='en'), '3 + negative two') self.assertEqual(self.processor.process('x = - 2', language='en'), 'x = negative two') for inp in ('- 2024 report', '- 2', '- item', '10 - 20 km', 'He said - 2 m'): self.assertEqual(self.processor.process(inp, language='en'), inp, inp) def test_not_a_negative_number_is_unchanged(self): inputs = [ '5-3', 'a-2', 'COVID-19', '2024-05-10', '0912-123-4567', '-10:30', '-2nd', '-2x', '--2', '5--2', 'hello', '-', '- ', ] for inp in inputs: self.assertEqual(self.processor.process(inp, language='en'), inp, inp) def test_unsupported_language_is_unchanged(self): for language in ('mixed', 'de'): self.assertEqual(self.processor.process('-2', language=language), '-2') def test_epsilon_zero_disables_processor(self): processor = NegativeSignProcessor(epsilon=0.0) outputs = {processor.process('-2', language='en') for _ in range(50)} self.assertEqual(outputs, {'-2'}) class MathOperatorProcessorTestCase(unittest.TestCase): def setUp(self): self.processor = MathOperatorProcessor() def test_english(self): cases = { '3 + 4': 'three plus four', '3+4': 'three plus four', '6 × 2': 'six times two', '6×2': 'six times two', '3 * 4': 'three times four', '8 ÷ 2': 'eight divided by two', '7 = 7': 'seven equals seven', '1+1=2': 'one plus one equals two', '3 + 4 = 7': 'three plus four equals seven', '1 + 2 + 3': 'one plus two plus three', '2.5 + 1,000': 'two point five plus one thousand', 'Tom + Jerry': 'Tom plus Jerry', 'x + y = z': 'x plus y equals z', 'Is 2 + 2 = 4?': 'Is two plus two equals four?', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) def test_persian(self): cases = { '۳ + ۴': 'سه بعلاوه چهار', '۳+۴': 'سه بعلاوه چهار', '۶ × ۲': 'شش ضربدر دو', '۸ ÷ ۲': 'هشت تقسیم بر دو', '۷ = ۷': 'هفت مساوی است با هفت', '۳ + ۴ = ۷': 'سه بعلاوه چهار مساوی است با هفت', 'حاصل ۲ + ۲ = ۴ است': 'حاصل دو بعلاوه دو مساوی است با چهار است', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) def test_wrapped_operands_and_symbols(self): self.assertEqual(self.processor.process('(3) + (4)', language='en'), '(three) plus (four)') self.assertEqual(self.processor.process('3 (+) 4', language='en'), 'three (plus) four') self.assertEqual(self.processor.process('[6] × [2]', language='en'), '[six] times [two]') self.assertEqual(self.processor.process('(3 + 4)', language='en'), '(three plus four)') self.assertEqual(self.processor.process('(۳) + (۴)', language='fa'), '(سه) بعلاوه (چهار)') self.assertEqual(self.processor.process('۳ (+) ۴', language='fa'), 'سه (بعلاوه) چهار') def test_minus(self): self.assertEqual(self.processor.process('5 − 3', language='en'), 'five minus three') self.assertEqual(self.processor.process('۵ − ۳', language='fa'), 'پنج منهای سه') # an ASCII hyphen is a minus only in a spaced equation self.assertEqual(self.processor.process('5 - 3 = 2', language='en'), 'five minus three equals two') self.assertEqual(self.processor.process('۵ - ۳ = ۲', language='fa'), 'پنج منهای سه مساوی است با دو') self.assertEqual(self.processor.process('x - y = z', language='en'), 'x minus y equals z') for inp in ('5 - 3', '5-3', '10 - 20', 'well - maybe'): self.assertEqual(self.processor.process(inp, language='en'), inp, inp) def test_symbols_that_are_not_between_operands_are_unchanged(self): inputs = [ '+98 912 123 4567', 'C++ is great', 'A+ grade', 'x=5', 'a=1&b=2', 'http://x.com/?a=1&b=2', 'a == b', 'a <= b', '* item', '+', '=', '3 +', '+ 4', 'hello', ] for inp in inputs: self.assertEqual(self.processor.process(inp, language='en'), inp, inp) def test_units_and_currency_are_left_for_their_processors(self): # Only the symbol is read here; `$3` and `3$` are read later by CurrencyProcessor. self.assertEqual(self.processor.process('$3 + $4', language='en'), '$3 plus $4') self.assertEqual(self.processor.process('3$ + 4$', language='en'), '3$ plus 4$') self.assertEqual(self.processor.process('10:30 + 2', language='en'), '10:30 plus two') self.assertEqual(self.processor.process('2024-05-10 + 3', language='en'), '2024-05-10 plus three') def test_no_letters_language_is_decided_by_digits(self): for language in (None, 'Unknown / No letters found'): self.assertEqual(self.processor.process('3 + 4', language=language), 'three plus four') self.assertEqual(self.processor.process('۳ + ۴', language=language), 'سه بعلاوه چهار') def test_unsupported_language_is_unchanged(self): for language in ('mixed', 'de'): self.assertEqual(self.processor.process('3 + 4', language=language), '3 + 4') def test_epsilon_zero_disables_processor(self): processor = MathOperatorProcessor(epsilon=0.0) outputs = {processor.process('3 + 4', language='en') for _ in range(50)} self.assertEqual(outputs, {'3 + 4'}) class RangeProcessorTestCase(unittest.TestCase): def setUp(self): self.processor = RangeProcessor() def test_plain_numbers_english(self): cases = { '3 - 5': 'three to five', '3-5': 'three to five', '3 – 5': 'three to five', '3–5': 'three to five', '10 - 20': 'ten to twenty', '2.5 - 3.5': 'two point five to three point five', 'pages 3-5': 'pages three to five', 'Is it 3 - 5?': 'Is it three to five?', '1 - 2 - 3': 'one to two to three', '1990 - 2000': 'one thousand, nine hundred and ninety to two thousand', '1990–2000': 'one thousand, nine hundred and ninety to two thousand', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) def test_plain_numbers_persian(self): cases = { '۳ - ۵': 'سه تا پنج', '۳-۵': 'سه تا پنج', '۳ – ۵': 'سه تا پنج', '3 - 5': 'سه تا پنج', 'از ۳ - ۵ ساعت': 'از سه تا پنج ساعت', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) def test_amounts_with_units_fractions_percent_and_currency_are_left_for_their_processors(self): # Only the dash (and a plain number beside it) is read here. cases = { '3kg - 5 kg': '3kg to 5 kg', '3 kg-5 kg': '3 kg to 5 kg', '3 - 5 kg': 'three to 5 kg', '3 kg - 5': '3 kg to five', '10 - 20 km': 'ten to 20 km', '$3 - $5': '$3 to $5', '3$ - 5$': '3$ to 5$', '1/2 - 3/4': '1/2 to 3/4', '5% - 10%': '5% to 10%', '10 - 20%': 'ten to 20%', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) self.assertEqual(self.processor.process('۳kg - ۵ kg', language='fa'), '۳kg تا ۵ kg') def test_wrapped_amounts(self): self.assertEqual(self.processor.process('(3) - (5)', language='en'), '(three) to (five)') self.assertEqual(self.processor.process('[3]-[5]', language='en'), '[three] to [five]') self.assertEqual(self.processor.process('(3 - 5)', language='en'), '(three to five)') self.assertEqual(self.processor.process('(۳) - (۵)', language='fa'), '(سه) تا (پنج)') # wrapped amounts with units are left for UnitProcessor self.assertEqual(self.processor.process('3 (cm) - 5 (cm)', language='en'), '3 (cm) to 5 (cm)') self.assertEqual(self.processor.process('(2) kg - (5) kg', language='en'), '(2) kg to (5) kg') def test_not_a_range_is_unchanged(self): inputs = [ '2024-05-10', '2024-05', '05-10', '0912-123-4567', '555-1234', '1990-2000', 'COVID-19', 'A-5', 'a - b', 'well - maybe', '10:30 - 11:30', '2024/05/10 - 2024/05/12', '3 -5', '3- 5', '- 5', '3 -', '3 –', 'http://x.com/3-5', '5 - 3 = 2', 'x = 3 - 5', ] for inp in inputs: self.assertEqual(self.processor.process(inp, language='en'), inp, inp) def test_no_letters_language_is_decided_by_digits(self): for language in (None, 'Unknown / No letters found'): self.assertEqual(self.processor.process('3 - 5', language=language), 'three to five') self.assertEqual(self.processor.process('۳ - ۵', language=language), 'سه تا پنج') def test_unsupported_language_is_unchanged(self): for language in ('mixed', 'de'): self.assertEqual(self.processor.process('3 - 5', language=language), '3 - 5') def test_epsilon_zero_disables_processor(self): processor = RangeProcessor(epsilon=0.0) outputs = {processor.process('3 - 5', language='en') for _ in range(50)} self.assertEqual(outputs, {'3 - 5'}) class PhoneNumberProcessorTestCase(unittest.TestCase): MOBILE_EN = 'zero nine one two, four three three, eight one, three five' MOBILE_PLUS_EN = 'plus nine eight, nine one two, four three three, eight one, three five' LANDLINE_EN = 'zero two one, eight eight, zero six, nine eight, nine five' MOBILE_FA = 'صفر نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج' MOBILE_PLUS_FA = 'مثبت نود و هشت، نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج' LANDLINE_FA = 'صفر بیست و یک، هشتاد و هشت، صفر شش، نود و هشت، نود و پنج' def setUp(self): self.processor = PhoneNumberProcessor() def test_iranian_mobile_english(self): for inp in ('09124338135', '0912-433-8135', '0912 433 8135', '0912.433.8135', '0912-4338135'): self.assertEqual(self.processor.process(inp, language='en'), self.MOBILE_EN, inp) def test_international_english(self): for inp in ('+989124338135', '+98-912-4338135', '+98 912 433 8135', '+98.912.433.8135', '(+98) 912 4338135'): expected = self.MOBILE_PLUS_EN if inp.startswith('('): expected = '(plus nine eight), nine one two, four three three, eight one, three five' self.assertEqual(self.processor.process(inp, language='en'), expected, inp) self.assertEqual( self.processor.process('0098 912 4338135', language='en'), 'zero zero nine eight, nine one two, four three three, eight one, three five', ) self.assertEqual( self.processor.process('+44 20 7946 0958', language='en'), 'plus four four, two zero, seven nine four six, zero nine five eight', ) self.assertEqual( self.processor.process('+1 (555) 123-4567', language='en'), 'plus one, (five five five), one two three, four five six seven', ) def test_iranian_landline_english(self): for inp in ('021-88069895', '021 88069895', '02188069895', '021.88069895'): self.assertEqual(self.processor.process(inp, language='en'), self.LANDLINE_EN, inp) self.assertEqual( self.processor.process('(021) 88069895', language='en'), '(zero two one), eight eight, zero six, nine eight, nine five', ) def test_us_style_english(self): cases = { '555-123-4567': 'five five five, one two three, four five six seven', '555.123.4567': 'five five five, one two three, four five six seven', '(555) 123-4567': '(five five five), one two three, four five six seven', '1-555-123-4567': 'one, five five five, one two three, four five six seven', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) # plain spaces are not enough: this could be a list of numbers self.assertEqual(self.processor.process('555 123 4567', language='en'), '555 123 4567') def test_persian_digits_and_persian_output(self): cases = { '۰۹۱۲۴۳۳۸۱۳۵': self.MOBILE_FA, '۰۹۱۲-۴۳۳-۸۱۳۵': self.MOBILE_FA, '+۹۸۹۱۲۴۳۳۸۱۳۵': self.MOBILE_PLUS_FA, '+۹۸-۹۱۲-۴۳۳۸۱۳۵': self.MOBILE_PLUS_FA, '۰۲۱-۸۸۰۶۹۸۹۵': self.LANDLINE_FA, '٠٩١٢٤٣٣٨١٣٥': self.MOBILE_FA, '09124338135': self.MOBILE_FA, '+989124338135': self.MOBILE_PLUS_FA, '021-88069895': self.LANDLINE_FA, } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) self.assertEqual( self.processor.process('(۰۲۱) ۸۸۰۶۹۸۹۵', language='fa'), '(صفر بیست و یک)، هشتاد و هشت، صفر شش، نود و هشت، نود و پنج', ) def test_persian_reads_iranian_numbers_in_groups(self): cases = { '۰۹۱۲۴۳۳۸۱۳۵': 'صفر نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج', '09124338135': 'صفر نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج', '02188037395': 'صفر بیست و یک، هشتاد و هشت، صفر سه، هفتاد و سه، نود و پنج', '۰۲۱-۸۸۰۳۷۳۹۵': 'صفر بیست و یک، هشتاد و هشت، صفر سه، هفتاد و سه، نود و پنج', '۰۴۱۳۱۲۳۴۵۶۷': 'صفر چهل و یک، سی و یک، بیست و سه، چهل و پنج، شصت و هفت', # a leading zero inside a group is said on its own '۰۹۱۲۰۰۰۰۵۰۸': 'صفر نهصد و دوازده، سه صفر، صفر پنج، صفر هشت', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) def test_persian_counts_repeated_leading_zeros(self): cases = { '۰۲۱-۸۸۵۲۳۰۰۶': 'صفر بیست و یک، هشتاد و هشت، پنجاه و دو، سی، صفر شش', '۰۰۱-۸۸۵۲۳۰۰۶': 'دو صفر یک، هشتاد و هشت، پنجاه و دو، سی، صفر شش', '00188523006': 'دو صفر یک، هشتاد و هشت، پنجاه و دو، سی، صفر شش', '001 88 52 30 06': 'دو صفر یک، هشتاد و هشت، پنجاه و دو، سی، صفر شش', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) def test_english_reads_the_same_groups_for_a_zero_area_code(self): self.assertEqual( self.processor.process('۰۰۱-۸۸۵۲۳۰۰۶', language='en'), 'zero zero one, eight eight, five two, three zero, zero six', ) def test_persian_international_iranian_numbers(self): cases = { '+989124338135': 'مثبت نود و هشت، نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج', '+98-912-4338135': 'مثبت نود و هشت، نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج', '+98 21 88037395': 'مثبت نود و هشت، بیست و یک، هشتاد و هشت، صفر سه، هفتاد و سه، نود و پنج', '0098 912 4338135': 'دو صفر نود و هشت، نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج', '(+98) 912 4338135': '(مثبت نود و هشت)، نهصد و دوازده، چهارصد و سی و سه، هشتاد و یک، سی و پنج', '(021) 88037395': '(صفر بیست و یک)، هشتاد و هشت، صفر سه، هفتاد و سه، نود و پنج', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) def test_persian_reads_other_numbers_digit_by_digit(self): cases = { '+44 20 7946 0958': 'مثبت چهار چهار دو صفر هفت نه چهار شش صفر نه پنج هشت', '555-123-4567': 'پنج پنج پنج یک دو سه چهار پنج شش هفت', '0012345678': 'صفر صفر یک دو سه چهار پنج شش هفت هشت', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) def test_english_reads_iranian_numbers_in_groups_with_pauses(self): cases = { '09124338135': 'zero nine one two, four three three, eight one, three five', '02188037395': 'zero two one, eight eight, zero three, seven three, nine five', '+989124338135': 'plus nine eight, nine one two, four three three, eight one, three five', '+98 21 88037395': 'plus nine eight, two one, eight eight, zero three, seven three, nine five', '0098 912 4338135': 'zero zero nine eight, nine one two, four three three, eight one, three five', '(021) 88037395': '(zero two one), eight eight, zero three, seven three, nine five', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) def test_english_other_numbers_use_the_written_groups(self): cases = { '+44 20 7946 0958': 'plus four four, two zero, seven nine four six, zero nine five eight', '+1 (555) 123-4567': 'plus one, (five five five), one two three, four five six seven', # no written groups: nothing to pause on '+441234567890': 'plus four four one two three four five six seven eight nine zero', '0012345678': 'zero zero one two three four five six seven eight', } for inp, correct in cases.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) def test_no_space_needed_after_a_closing_parenthesis(self): for inp in ('(0915) 6486320', '(0915)6486320'): self.assertEqual( self.processor.process(inp, language='en'), '(zero nine one five), six four eight, six three, two zero', inp, ) cases_en = { '(021)88069895': '(zero two one), eight eight, zero six, nine eight, nine five', '(+98)9124338135': '(plus nine eight), nine one two, four three three, eight one, three five', '(555)123-4567': '(five five five), one two three, four five six seven', 'Call (0915)6486320, ok': 'Call (zero nine one five), six four eight, six three, two zero, ok', '(0915)6486320 and (021)88069895': ( '(zero nine one five), six four eight, six three, two zero and ' '(zero two one), eight eight, zero six, nine eight, nine five' ), } for inp, correct in cases_en.items(): self.assertEqual(self.processor.process(inp, language='en'), correct, inp) cases_fa = { '(۰۹۱۵)۶۴۸۶۳۲۰': '(صفر نهصد و پانزده)، ششصد و چهل و هشت، شصت و سه، بیست', '(۰۹۱۵) ۶۴۸۶۳۲۰': '(صفر نهصد و پانزده)، ششصد و چهل و هشت، شصت و سه، بیست', '(021)۸۸۰۶۹۸۹۵': '(صفر بیست و یک)، هشتاد و هشت، صفر شش، نود و هشت، نود و پنج', } for inp, correct in cases_fa.items(): self.assertEqual(self.processor.process(inp, language='fa'), correct, inp) def test_parentheses_without_a_phone_number_are_unchanged(self): for inp in ('(1)23456789', '(2)(3)', '5 (kg)12', '(12)34'): self.assertEqual(self.processor.process(inp, language='en'), inp, inp) def test_numbers_inside_sentences(self): self.assertEqual( self.processor.process('Call me at 09124338135.', language='en'), f'Call me at {self.MOBILE_EN}.', ) self.assertEqual( self.processor.process('Call 09124338135, then 021-88069895.', language='en'), f'Call {self.MOBILE_EN}, then {self.LANDLINE_EN}.', ) self.assertEqual( self.processor.process('(+98 912 4338135)', language='en'), '(plus nine eight, nine one two, four three three, eight one, three five)', ) self.assertEqual( self.processor.process('شماره من ۰۹۱۲۴۳۳۸۱۳۵ است', language='fa'), f'شماره من {self.MOBILE_FA} است', ) self.assertEqual( self.processor.process('تماس: ۰۲۱-۸۸۰۶۹۸۹۵ و ۰۹۱۲۴۳۳۸۱۳۵', language='fa'), f'تماس: {self.LANDLINE_FA} و {self.MOBILE_FA}', ) self.assertEqual( self.processor.process('In 2024 0912 433 8135', language='en'), f'In 2024 {self.MOBILE_EN}', ) def test_number_after_a_phone_word_may_be_shorter(self): self.assertEqual( self.processor.process('tel: 88069895', language='en'), 'tel: eight eight, zero six, nine eight, nine five', ) self.assertEqual( self.processor.process('Phone number 4338135', language='en'), 'Phone number four three three eight one three five', ) self.assertEqual( self.processor.process('تلفن: ۸۸۰۶۹۸۹۵', language='fa'), 'تلفن: هشتاد و هشت، صفر شش، نود و هشت، نود و پنج', ) self.assertEqual( self.processor.process('Call 1-800-555-1234', language='en'), 'Call one, eight zero zero, five five five, one two three four', ) # too short, or no phone word: unchanged for inp in ('call 911', 'call 5 times', '4338135', 'Tel 123456'): self.assertEqual(self.processor.process(inp, language='en'), inp, inp) self.assertEqual(self.processor.process('شماره 5', language='fa'), 'شماره 5') def test_language_comes_from_digits_when_there_are_no_letters(self): for language in (None, 'Unknown / No letters found'): self.assertEqual(self.processor.process('09124338135', language=language), self.MOBILE_EN) self.assertEqual(self.processor.process('۰۹۱۲۴۳۳۸۱۳۵', language=language), self.MOBILE_FA) def test_not_a_phone_number_is_unchanged(self): inputs = [ '4338135', '9124338135', '123456', '2024-05-10', '2024/05/10', '01-02-2024', '01.02.2024', '0.123456789', '3.0912433813', '0912433813.5', '0912433813.55', '09124338135abc', 'a09124338135', 'https://x.com/09124338135', 'user09124338135@mail.com', '1234 5678 9012 3456', '0912 433', '0912', '021-88069895-12', '5 - 3', '1990 - 2000', '10:30', '1,000,000', 'hello', ] for inp in inputs: self.assertEqual(self.processor.process(inp, language='en'), inp, inp) def test_ten_digit_zero_prefixed_id_is_read_digit_by_digit(self): self.assertEqual( self.processor.process('0012345678', language='en'), 'zero zero one two three four five six seven eight', ) def test_unsupported_language_is_unchanged(self): for language in ('mixed', 'de'): self.assertEqual(self.processor.process('09124338135', language=language), '09124338135') def test_epsilon_zero_disables_processor(self): processor = PhoneNumberProcessor(epsilon=0.0) outputs = {processor.process('09124338135', language='en') for _ in range(50)} self.assertEqual(outputs, {'09124338135'}) class CardNumberProcessorTestCase(unittest.TestCase): def setUp(self): self.processor = CardNumberProcessor() def assertFa(self, text, expected): self.assertEqual(self.processor.process(text, language='fa'), expected) def assertEn(self, text, expected): self.assertEqual(self.processor.process(text, language='en'), expected) # Persian def test_persian_card_with_hyphens(self): self.assertFa( '۶۲۱۹ - ۸۷۶۵ - ۲۳۴۵ - ۹۰۳۱', 'شصت و دو، نوزده - هشتاد و هفت، شصت و پنج - بیست و سه، چهل و پنج - نود، سی و یک', ) def test_persian_card_with_tight_hyphens_spaces_and_no_separator(self): expected = 'شصت و دو، نوزده - هشتاد و هفت، شصت و پنج - بیست و سه، چهل و پنج - نود، سی و یک' for text in ('۶۲۱۹-۸۷۶۵-۲۳۴۵-۹۰۳۱', '۶۲۱۹ ۸۷۶۵ ۲۳۴۵ ۹۰۳۱', '۶۲۱۹۸۷۶۵۲۳۴۵۹۰۳۱'): self.assertFa(text, expected) def test_persian_card_with_ascii_digits(self): self.assertFa( '6219 8765 2345 9031', 'شصت و دو، نوزده - هشتاد و هفت، شصت و پنج - بیست و سه، چهل و پنج - نود، سی و یک', ) def test_persian_card_in_sentence(self): self.assertFa( 'شماره کارت ۶۲۱۹ ۸۷۶۵ ۲۳۴۵ ۹۰۳۱ است', 'شماره کارت شصت و دو، نوزده - هشتاد و هفت، شصت و پنج - بیست و سه، چهل و پنج - نود، سی و یک است', ) def test_persian_zero_pairs(self): result = self.processor.process('۶۰۰۵ ۱۲۰۰ ۳۴۵۶ ۷۸۹۰', language='fa') self.assertIn('صفر پنج', result) self.assertIn('دو صفر', result) # English def test_english_card_with_spaces(self): self.assertEn( '4111 1111 1111 1111', 'forty-one, eleven - eleven, eleven - eleven, eleven - eleven, eleven', ) def test_english_card_with_hyphens(self): self.assertEn( '5105-1051-0510-5100', 'fifty-one, oh five - ten, fifty-one - oh five, ten - fifty-one, double zero', ) def test_english_card_without_separators(self): self.assertEn( '6219876523459031', 'sixty-two, nineteen - eighty-seven, sixty-five - twenty-three, forty-five - ninety, thirty-one', ) def test_english_card_in_sentence(self): self.assertEn( 'My card is 4111 1111 1111 1111 ok', 'My card is forty-one, eleven - eleven, eleven - eleven, eleven - eleven, eleven ok', ) def test_english_card_with_persian_digits(self): self.assertEn( '۴۱۱۱ ۱۱۱۱ ۱۱۱۱ ۱۱۱۱', 'forty-one, eleven - eleven, eleven - eleven, eleven - eleven, eleven', ) # Language detection def test_language_detected_from_digits(self): self.assertEqual( self.processor.process('۶۲۱۹ ۸۷۶۵ ۲۳۴۵ ۹۰۳۱'), 'شصت و دو، نوزده - هشتاد و هفت، شصت و پنج - بیست و سه، چهل و پنج - نود، سی و یک', ) self.assertEqual( self.processor.process('4111 1111 1111 1111'), 'forty-one, eleven - eleven, eleven - eleven, eleven - eleven, eleven', ) # Left alone def test_non_cards_are_unchanged(self): for text in ( '6219-****-****-9031', '6219 8765 2345 9031.5', '6219 8765 2345', '6219 8765 2345 9031 1234', '3782 822463 10005', '09124338135', 'abc6219876523459031', ): self.assertEqual(self.processor.process(text, language='en'), text) def test_any_first_digit_is_accepted(self): self.assertEn( '1234 5678 9012 3456', 'twelve, thirty-four - fifty-six, seventy-eight - ninety, twelve - thirty-four, fifty-six', ) self.assertEn( '0912 4338 1350 0000', 'oh nine, twelve - forty-three, thirty-eight - thirteen, fifty - double zero, double zero', ) self.assertEn( '8123 4567 8901 2345', 'eighty-one, twenty-three - forty-five, sixty-seven - eighty-nine, oh one - twenty-three, forty-five', ) def test_unsupported_language_is_unchanged(self): for language in ('mixed', 'de'): self.assertEqual( self.processor.process('4111 1111 1111 1111', language=language), '4111 1111 1111 1111') def test_epsilon_zero_disables_processor(self): processor = CardNumberProcessor(epsilon=0.0) outputs = {processor.process('4111 1111 1111 1111', language='en') for _ in range(50)} self.assertEqual(outputs, {'4111 1111 1111 1111'}) # Through the Normalizer def test_normalizer_reads_card_not_phone(self): normalizer = Normalizer() self.assertEqual( normalizer.process('شماره کارت ۶۲۱۹ ۸۷۶۵ ۲۳۴۵ ۹۰۳۱ است'), 'شماره کارت شصت و دو، نوزده - هشتاد و هفت، شصت و پنج - بیست و سه، چهل و پنج - نود، سی و یک است', ) self.assertEqual( normalizer.process('My card is 4111 1111 1111 1111 ok'), 'My card is forty-one, eleven - eleven, eleven - eleven, eleven - eleven, eleven ok', ) class PostalCodeAndNationalIdProcessorTestCase(unittest.TestCase): def setUp(self): self.processor = PostalCodeAndNationalIdProcessor() def assertFa(self, text, expected): self.assertEqual(self.processor.process(text, language='fa'), expected) def assertEn(self, text, expected): self.assertEqual(self.processor.process(text, language='en'), expected) # Persian def test_persian_postal_code(self): self.assertFa('۱۵۱۴۷۱۷۱۱۸', 'پانزده، چهارده، هفتاد و یک، هفتاد و یک، هجده') def test_persian_postal_code_with_ascii_digits(self): self.assertFa('1514717118', 'پانزده، چهارده، هفتاد و یک، هفتاد و یک، هجده') def test_persian_postal_code_split_in_two_halves(self): expected = 'پانزده، چهارده، هفتاد و یک، هفتاد و یک، هجده' for text in ('۱۵۱۴۷-۱۷۱۱۸', '۱۵۱۴۷ ۱۷۱۱۸', '۱۵۱۴۷ - ۱۷۱۱۸'): self.assertFa(text, expected) def test_persian_postal_code_in_sentence(self): self.assertFa( 'کد پستی ۱۵۱۴۷۱۷۱۱۸ است', 'کد پستی پانزده، چهارده، هفتاد و یک، هفتاد و یک، هجده است', ) def test_persian_zero_pairs(self): result = self.processor.process('۱۵۱۴۷۰۰۰۰۵', language='fa') self.assertIn('دو صفر', result) self.assertIn('صفر پنج', result) # English def test_english_postal_code(self): self.assertEn('1514717118', 'fifteen, fourteen, seventy-one, seventy-one, eighteen') def test_english_postal_code_split_in_two_halves(self): self.assertEn('15147-17118', 'fifteen, fourteen, seventy-one, seventy-one, eighteen') def test_english_postal_code_with_persian_digits(self): self.assertEn('۱۵۱۴۷۱۷۱۱۸', 'fifteen, fourteen, seventy-one, seventy-one, eighteen') def test_english_postal_code_in_sentence(self): self.assertEn( 'Postal code 1514717118 ok', 'Postal code fifteen, fourteen, seventy-one, seventy-one, eighteen ok', ) def test_english_zero_pairs(self): self.assertEn('1514700508', 'fifteen, fourteen, seventy, oh five, oh eight') self.assertEn('1514700018', 'fifteen, fourteen, seventy, double zero, eighteen') # National ID numbers def test_persian_national_id_with_leading_zeros(self): self.assertFa('۰۰۱۲۳۴۵۶۷۹', 'دو صفر، دوازده، سی و چهار، پنجاه و شش، هفتاد و نه') def test_english_national_id_with_leading_zeros(self): self.assertEn('0012345679', 'double zero, twelve, thirty-four, fifty-six, seventy-nine') def test_national_id_with_one_leading_zero(self): self.assertEn('0499370899', 'oh four, ninety-nine, thirty-seven, oh eight, ninety-nine') self.assertFa('۰۴۹۹۳۷۰۸۹۹', 'صفر چهار، نود و نه، سی و هفت، صفر هشت، نود و نه') def test_national_id_with_hyphens_3_6_1(self): self.assertEn('001-234567-9', 'double zero, twelve, thirty-four, fifty-six, seventy-nine') self.assertFa('۰۰۱-۲۳۴۵۶۷-۹', 'دو صفر، دوازده، سی و چهار، پنجاه و شش، هفتاد و نه') def test_national_id_in_sentence(self): self.assertFa( 'کد ملی ۰۰۱۲۳۴۵۶۷۹ را وارد کنید', 'کد ملی دو صفر، دوازده، سی و چهار، پنجاه و شش، هفتاد و نه را وارد کنید', ) self.assertEn( 'National ID 0012345679 is valid', 'National ID double zero, twelve, thirty-four, fifty-six, seventy-nine is valid', ) def test_invalid_national_id_is_unchanged(self): for text in ( '0012345678', # wrong check digit '001-234567-8', # wrong check digit, hyphenated layout '0000000000', # all the same digit (passes the checksum but is not a real ID) ): self.assertEn(text, text) def test_hyphenated_3_6_1_layout_needs_valid_check_digit(self): # Passes the postal-code rule when written as 5 + 5, but 3 + 6 + 1 is a national ID layout only. self.assertEn('151-471711-8', '151-471711-8') # Language detection def test_language_detected_from_digits(self): self.assertEqual( self.processor.process('۱۵۱۴۷۱۷۱۱۸'), 'پانزده، چهارده، هفتاد و یک، هفتاد و یک، هجده', ) self.assertEqual( self.processor.process('1514717118'), 'fifteen, fourteen, seventy-one, seventy-one, eighteen', ) # Left alone def test_non_identifiers_are_unchanged(self): for text in ( '9124338135', # phone number without the leading zero (a 2 in the first five) '1234567890', # a 0 or 2 in the first five digits '0514717118', # starts with 0 '15147171189', # 11 digits '151471711', # 9 digits '1514717118.5', # decimal '1514 7171 18', # wrong grouping 'abc1514717118', # glued to a word ): self.assertEqual(self.processor.process(text, language='en'), text) def test_card_number_is_not_read_as_postal_code(self): self.assertEqual(self.processor.process('6219876523459031', language='en'), '6219876523459031') def test_unsupported_language_is_unchanged(self): for language in ('mixed', 'de'): self.assertEqual(self.processor.process('1514717118', language=language), '1514717118') def test_epsilon_zero_disables_processor(self): processor = PostalCodeAndNationalIdProcessor(epsilon=0.0) outputs = {processor.process('1514717118', language='en') for _ in range(50)} self.assertEqual(outputs, {'1514717118'}) # Through the Normalizer def test_normalizer_reads_postal_code(self): normalizer = Normalizer() self.assertEqual( normalizer.process('کد پستی ۱۵۱۴۷۱۷۱۱۸ است'), 'کد پستی پانزده، چهارده، هفتاد و یک، هفتاد و یک، هجده است', ) self.assertEqual( normalizer.process('Postal code 1514717118 ok'), 'Postal code fifteen, fourteen, seventy-one, seventy-one, eighteen ok', ) def test_normalizer_still_reads_phone_and_card(self): normalizer = Normalizer() self.assertEqual( normalizer.process('call 09124338135'), 'call zero nine one two, four three three, eight one, three five', ) self.assertIn('شصت و دو، نوزده', normalizer.process('کارت ۶۲۱۹ ۸۷۶۵ ۲۳۴۵ ۹۰۳۱')) if __name__ == '__main__': unittest.main(verbosity=2)