File size: 2,430 Bytes
19cc583
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
import string
import re
from nltk.corpus import stopwords

def nep_news_classification_preprocessing(text):
    nep_stopwrods = stopwords.words("nepali")

    text = (
        text.replace("\n", " ")
        .replace("\r", " ")
        .replace("\t", " ")
        .replace("–", " ")
        .replace("-", " ")
        .replace("।", " ")
        # .replace("(", " ")
        # .replace(")", " ")
        .replace("—", " ")
        .replace("ॽ", " ")
        .replace("?", " ")
        .replace("ः", "")
        .replace(":", " ")
        .replace("“", ' ')
        .replace("”", ' ')
        .replace("‘", " ")
        .replace("’", " ")
        .replace('"', " ")
        .replace("'", " ")
        .replace("0", "०")
        .replace("1", "१")
        .replace("2", "२")
        .replace("3", "३")
        .replace("4", "४")
        .replace("5", "५")
        .replace("6", "६")
        .replace("7", "७")
        .replace("8", "८")
        .replace("9", "९")
    )

    # Remove emojis
    regrex_pattern = re.compile(
        pattern="["
        "\U0001f600-\U0001f64f"  # emoticons
        "\U0001f300-\U0001f5ff"  # symbols & pictographs
        "\U0001f680-\U0001f6ff"  # transport & map symbols
        "\U0001f1e0-\U0001f1ff"  # flags (iOS)
        "\U00002500-\U00002bef"  # chinese char
        "\U00002702-\U000027b0"
        "\U000024c2-\U0001f251"
        "\U0001f926-\U0001f937"
        "\U00010000-\U0010ffff"
        "\u2640-\u2642"
        "\u2600-\u2b55"
        "\u200d"
        "\u23cf"
        "\u23e9"
        "\u231a"
        "\ufe0f"  # dingbats
        "\u3030"
        "]+",
        flags=re.UNICODE,
    )
    text = regrex_pattern.sub("", text)

    pattern = re.compile(r'<.*?>') # HTML
    text = pattern.sub('', text)

    text = re.sub(r'([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,})', '', text) # Email

    # Remove multiple white spaces
    text = re.sub(r"\s+", " ", text)

    text = re.sub('[a-zA-Z]', '', text)

    # Text inside paranthesis
    text = re.sub(r'\([^)]*\)', '', text)

    text = " ".join(text.split())

    punctuations = list(string.punctuation)

    text = "".join([char for char in text if char not in punctuations and char not in "०१२३४५६७८९"])
    text = [word.strip() for word in text.split() if word.strip() not in nep_stopwrods]
    text = " ".join(text)
    text = " ".join(text.split())

    return text