Spaces:
Sleeping
Sleeping
File size: 2,322 Bytes
0a4e172 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | from rules.base import BaseProcessor
from rules.currency import CurrencyProcessor
from rules.dates import DateProcessor
from rules.fraction import FractionProcessor
from rules.cards import CardNumberProcessor
from rules.postal_national_id import PostalCodeAndNationalIdProcessor
from rules.numbers import NumberProcessor
from rules.phone import PhoneNumberProcessor
from rules.spaces import CurrencySpaceProcessor
from rules.symbols import (
AmpersandProcessor,
MathOperatorProcessor,
NegativeSignProcessor,
RangeProcessor,
RomanNumeralProcessor,
)
from rules.times import TimeProcessor
from rules.units import UnitProcessor
from utils import detect_language
class Normalizer(BaseProcessor):
def __init__(self):
super(Normalizer, self).__init__()
self.text_processors = [
CurrencySpaceProcessor(),
CardNumberProcessor(),
PostalCodeAndNationalIdProcessor(),
PhoneNumberProcessor(),
NegativeSignProcessor(),
RangeProcessor(),
UnitProcessor(),
FractionProcessor(),
RomanNumeralProcessor(),
AmpersandProcessor(),
MathOperatorProcessor()
]
self.processors = [
DateProcessor(),
TimeProcessor(),
CurrencyProcessor()
]
# Runs last, on whatever number is still left as digits.
self.number_processor = NumberProcessor()
def process(self, text:str) -> str:
language = detect_language(text)
for text_processor in self.text_processors:
text = text_processor.process(text, language=language)
# A processor may have rewritten symbols into words (`5 km`).
language = detect_language(text)
for processor in self.processors:
text = self._apply_a_processor(text=text, processor=processor, language=language)
return self.number_processor.process(text, language=detect_language(text))
@staticmethod
def _apply_a_processor(text:str, processor:BaseProcessor, language:str) -> str:
words = text.split()
processed = []
for word in words:
word = word.strip()
processed.append(processor.process(word, language=language))
return ' '.join(processed) |