Download app/preprocess/preprocess.py from saskarkhadka/classification-backend: direct link, hf CLI and curl.
- Browser
- Download file 2.43 kB
-
https://huggingface.co/spaces/saskarkhadka/classification-backend/resolve/82474ae2d9aa14227e2aeabdba76749230b7e838/app/preprocess/preprocess.py
- Command line
-
hf download hf://spaces/saskarkhadka/classification-backend@82474ae2d9aa14227e2aeabdba76749230b7e838/app/preprocess/preprocess.py
-
curl -L -o preprocess.py https://huggingface.co/spaces/saskarkhadka/classification-backend/resolve/82474ae2d9aa14227e2aeabdba76749230b7e838/app/preprocess/preprocess.py
2.43 kB
| import string | |
| import re | |
| from nltk.corpus import stopwords | |
| def nep_news_classification_preprocessing(text): | |
| nep_stopwrods = stopwords.words("nepali") | |
| text = ( | |
| text.replace("\n", " ") | |
| .replace("\r", " ") | |
| .replace("\t", " ") | |
| .replace("–", " ") | |
| .replace("-", " ") | |
| .replace("।", " ") | |
| # .replace("(", " ") | |
| # .replace(")", " ") | |
| .replace("—", " ") | |
| .replace("ॽ", " ") | |
| .replace("?", " ") | |
| .replace("ः", "") | |
| .replace(":", " ") | |
| .replace("“", ' ') | |
| .replace("”", ' ') | |
| .replace("‘", " ") | |
| .replace("’", " ") | |
| .replace('"', " ") | |
| .replace("'", " ") | |
| .replace("0", "०") | |
| .replace("1", "१") | |
| .replace("2", "२") | |
| .replace("3", "३") | |
| .replace("4", "४") | |
| .replace("5", "५") | |
| .replace("6", "६") | |
| .replace("7", "७") | |
| .replace("8", "८") | |
| .replace("9", "९") | |
| ) | |
| # Remove emojis | |
| regrex_pattern = re.compile( | |
| pattern="[" | |
| "\U0001f600-\U0001f64f" # emoticons | |
| "\U0001f300-\U0001f5ff" # symbols & pictographs | |
| "\U0001f680-\U0001f6ff" # transport & map symbols | |
| "\U0001f1e0-\U0001f1ff" # flags (iOS) | |
| "\U00002500-\U00002bef" # chinese char | |
| "\U00002702-\U000027b0" | |
| "\U000024c2-\U0001f251" | |
| "\U0001f926-\U0001f937" | |
| "\U00010000-\U0010ffff" | |
| "\u2640-\u2642" | |
| "\u2600-\u2b55" | |
| "\u200d" | |
| "\u23cf" | |
| "\u23e9" | |
| "\u231a" | |
| "\ufe0f" # dingbats | |
| "\u3030" | |
| "]+", | |
| flags=re.UNICODE, | |
| ) | |
| text = regrex_pattern.sub("", text) | |
| pattern = re.compile(r'<.*?>') # HTML | |
| text = pattern.sub('', text) | |
| text = re.sub(r'([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,})', '', text) # Email | |
| # Remove multiple white spaces | |
| text = re.sub(r"\s+", " ", text) | |
| text = re.sub('[a-zA-Z]', '', text) | |
| # Text inside paranthesis | |
| text = re.sub(r'\([^)]*\)', '', text) | |
| text = " ".join(text.split()) | |
| punctuations = list(string.punctuation) | |
| text = "".join([char for char in text if char not in punctuations and char not in "०१२३४५६७८९"]) | |
| text = [word.strip() for word in text.split() if word.strip() not in nep_stopwrods] | |
| text = " ".join(text) | |
| text = " ".join(text.split()) | |
| return text |