File size: 1,422 Bytes
ba7290b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | import unicodedata
import re
import typing as tp
from functools import lru_cache
from typing import List
from sacremoses import MosesPunctNormalizer
from preprocessors.ssplit import sent_splitter
from preprocessors.tstrip import get_ascii_hashtag_replacer, get_non_printing_char_replacer, get_url_replacer
@lru_cache
def get_cleaned_splitter(lang_code: str) -> tp.Callable:
"""
Return sentence processor
"""
# Compile regexes at once
mpn = MosesPunctNormalizer(lang="en")
mpn.substitutions = [(re.compile(pat), sub) for pat, sub in mpn.substitutions]
# Strip functions
replace_hashtag = get_ascii_hashtag_replacer(" ")
replace_nonprint = get_non_printing_char_replacer(" ")
replace_url = get_url_replacer(" ")
def process(text: str) -> List:
"""
Normalize, split and clean sentences
"""
sentence_splits = sent_splitter(text, lang_code)
cleaned_sents = []
for sentence in sentence_splits:
clean = mpn.normalize(sentence)
clean = replace_nonprint(replace_hashtag(replace_url(clean)))
clean = unicodedata.normalize("NFC", clean)
cleaned_sents.append(clean)
return cleaned_sents
return process
def splitAndClean(text, lang_code: str) -> List:
"""Cleans input, splits into sentences"""
splitter = get_cleaned_splitter(lang_code)
return splitter(text) |