Spaces:
Sleeping
Sleeping
| from rules.base import BaseProcessor | |
| from rules.currency import CurrencyProcessor | |
| from rules.dates import DateProcessor | |
| from rules.fraction import FractionProcessor | |
| from rules.cards import CardNumberProcessor | |
| from rules.postal_national_id import PostalCodeAndNationalIdProcessor | |
| from rules.numbers import NumberProcessor | |
| from rules.phone import PhoneNumberProcessor | |
| from rules.spaces import CurrencySpaceProcessor | |
| from rules.symbols import ( | |
| AmpersandProcessor, | |
| MathOperatorProcessor, | |
| NegativeSignProcessor, | |
| RangeProcessor, | |
| RomanNumeralProcessor, | |
| ) | |
| from rules.times import TimeProcessor | |
| from rules.units import UnitProcessor | |
| from utils import detect_language | |
| class Normalizer(BaseProcessor): | |
| def __init__(self): | |
| super(Normalizer, self).__init__() | |
| self.text_processors = [ | |
| CurrencySpaceProcessor(), | |
| CardNumberProcessor(), | |
| PostalCodeAndNationalIdProcessor(), | |
| PhoneNumberProcessor(), | |
| NegativeSignProcessor(), | |
| RangeProcessor(), | |
| UnitProcessor(), | |
| FractionProcessor(), | |
| RomanNumeralProcessor(), | |
| AmpersandProcessor(), | |
| MathOperatorProcessor() | |
| ] | |
| self.processors = [ | |
| DateProcessor(), | |
| TimeProcessor(), | |
| CurrencyProcessor() | |
| ] | |
| # Runs last, on whatever number is still left as digits. | |
| self.number_processor = NumberProcessor() | |
| def process(self, text:str) -> str: | |
| language = detect_language(text) | |
| for text_processor in self.text_processors: | |
| text = text_processor.process(text, language=language) | |
| # A processor may have rewritten symbols into words (`5 km`). | |
| language = detect_language(text) | |
| for processor in self.processors: | |
| text = self._apply_a_processor(text=text, processor=processor, language=language) | |
| return self.number_processor.process(text, language=detect_language(text)) | |
| def _apply_a_processor(text:str, processor:BaseProcessor, language:str) -> str: | |
| words = text.split() | |
| processed = [] | |
| for word in words: | |
| word = word.strip() | |
| processed.append(processor.process(word, language=language)) | |
| return ' '.join(processed) |