saskarkhadka's picture
add application src files
19cc583
Raw History Blame
2.43 kB
import string
import re
from nltk.corpus import stopwords
def nep_news_classification_preprocessing(text):
nep_stopwrods = stopwords.words("nepali")
text = (
text.replace("\n", " ")
.replace("\r", " ")
.replace("\t", " ")
.replace("–", " ")
.replace("-", " ")
.replace("।", " ")
# .replace("(", " ")
# .replace(")", " ")
.replace("—", " ")
.replace("ॽ", " ")
.replace("?", " ")
.replace("ः", "")
.replace(":", " ")
.replace("“", ' ')
.replace("”", ' ')
.replace("‘", " ")
.replace("’", " ")
.replace('"', " ")
.replace("'", " ")
.replace("0", "०")
.replace("1", "१")
.replace("2", "२")
.replace("3", "३")
.replace("4", "४")
.replace("5", "५")
.replace("6", "६")
.replace("7", "७")
.replace("8", "८")
.replace("9", "९")
)
# Remove emojis
regrex_pattern = re.compile(
pattern="["
"\U0001f600-\U0001f64f" # emoticons
"\U0001f300-\U0001f5ff" # symbols & pictographs
"\U0001f680-\U0001f6ff" # transport & map symbols
"\U0001f1e0-\U0001f1ff" # flags (iOS)
"\U00002500-\U00002bef" # chinese char
"\U00002702-\U000027b0"
"\U000024c2-\U0001f251"
"\U0001f926-\U0001f937"
"\U00010000-\U0010ffff"
"\u2640-\u2642"
"\u2600-\u2b55"
"\u200d"
"\u23cf"
"\u23e9"
"\u231a"
"\ufe0f" # dingbats
"\u3030"
"]+",
flags=re.UNICODE,
)
text = regrex_pattern.sub("", text)
pattern = re.compile(r'<.*?>') # HTML
text = pattern.sub('', text)
text = re.sub(r'([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,})', '', text) # Email
# Remove multiple white spaces
text = re.sub(r"\s+", " ", text)
text = re.sub('[a-zA-Z]', '', text)
# Text inside paranthesis
text = re.sub(r'\([^)]*\)', '', text)
text = " ".join(text.split())
punctuations = list(string.punctuation)
text = "".join([char for char in text if char not in punctuations and char not in "०१२३४५६७८९"])
text = [word.strip() for word in text.split() if word.strip() not in nep_stopwrods]
text = " ".join(text)
text = " ".join(text.split())
return text