from rules.base import BaseProcessor from rules.currency import CurrencyProcessor from rules.dates import DateProcessor from rules.fraction import FractionProcessor from rules.cards import CardNumberProcessor from rules.postal_national_id import PostalCodeAndNationalIdProcessor from rules.numbers import NumberProcessor from rules.phone import PhoneNumberProcessor from rules.spaces import CurrencySpaceProcessor from rules.symbols import ( AmpersandProcessor, MathOperatorProcessor, NegativeSignProcessor, RangeProcessor, RomanNumeralProcessor, ) from rules.times import TimeProcessor from rules.units import UnitProcessor from utils import detect_language class Normalizer(BaseProcessor): def __init__(self): super(Normalizer, self).__init__() self.text_processors = [ CurrencySpaceProcessor(), CardNumberProcessor(), PostalCodeAndNationalIdProcessor(), PhoneNumberProcessor(), NegativeSignProcessor(), RangeProcessor(), UnitProcessor(), FractionProcessor(), RomanNumeralProcessor(), AmpersandProcessor(), MathOperatorProcessor() ] self.processors = [ DateProcessor(), TimeProcessor(), CurrencyProcessor() ] # Runs last, on whatever number is still left as digits. self.number_processor = NumberProcessor() def process(self, text:str) -> str: language = detect_language(text) for text_processor in self.text_processors: text = text_processor.process(text, language=language) # A processor may have rewritten symbols into words (`5 km`). language = detect_language(text) for processor in self.processors: text = self._apply_a_processor(text=text, processor=processor, language=language) return self.number_processor.process(text, language=detect_language(text)) @staticmethod def _apply_a_processor(text:str, processor:BaseProcessor, language:str) -> str: words = text.split() processed = [] for word in words: word = word.strip() processed.append(processor.process(word, language=language)) return ' '.join(processed)