Download dataset.py from Shymaa98/cChat: direct link, hf CLI and curl.
- Browser
- Download file 2.22 kB
-
https://huggingface.co/spaces/Shymaa98/cChat/resolve/main/dataset.py
- Command line
-
hf download hf://spaces/Shymaa98/cChat/dataset.py
-
curl -L -o dataset.py https://huggingface.co/spaces/Shymaa98/cChat/resolve/main/dataset.py
2.22 kB
| import re | |
| from autocorrect import Speller | |
| import nltk | |
| from nltk.corpus import stopwords | |
| from nltk import word_tokenize | |
| from nltk.stem import WordNetLemmatizer | |
| from nltk import word_tokenize | |
| import string | |
| import nltk | |
| import os | |
| import nltk | |
| nltk_data_dir = os.path.expanduser("~/app/nltk_data") | |
| os.makedirs(nltk_data_dir, exist_ok=True) | |
| nltk.data.path.append(nltk_data_dir) | |
| nltk.download("stopwords", download_dir=nltk_data_dir) | |
| nltk.download("punkt_tab", download_dir=nltk_data_dir) | |
| nltk.download("punkt", download_dir=nltk_data_dir) | |
| nltk.download("wordnet", download_dir=nltk_data_dir) | |
| def remove_html_tags(text): | |
| html_pattern = r'<.*?>' | |
| without_html = re.sub(pattern=html_pattern, repl=' ', string=text) | |
| return without_html | |
| def convert_to_lower(text): | |
| return text.lower() | |
| def remove_urls(text): | |
| url_pattern = r'https?://\S+|www\.\S+' | |
| without_urls = re.sub(pattern=url_pattern, repl=' ', string=text) | |
| return without_urls | |
| def spell_checker(text): | |
| spellChecker = Speller(lang="en") | |
| correct_words = [] | |
| for word in nltk.word_tokenize(text): | |
| correct_word = spellChecker(word) | |
| correct_words.append(correct_word) | |
| correct_spell_text = " ".join(correct_words) | |
| return correct_spell_text | |
| def remove_punctuation(text): | |
| return text.translate(str.maketrans('', '', string.punctuation)) | |
| def remove_stopwords(text): | |
| removed = [] | |
| stop_words = list(stopwords.words("english")) | |
| tokens = word_tokenize(text) | |
| for i in range(len(tokens)): | |
| if tokens[i] not in stop_words: | |
| removed.append(tokens[i]) | |
| return " ".join(removed) | |
| def lemmatizing(text): | |
| lemmatizer = WordNetLemmatizer() | |
| tokens = word_tokenize(text) | |
| for i in range(len(tokens)): | |
| lemma_word = lemmatizer.lemmatize(tokens[i]) | |
| tokens[i] = lemma_word | |
| return " ".join(tokens) | |
| def clean(text): | |
| cleaned_text=convert_to_lower(text) | |
| cleaned_text=remove_html_tags(cleaned_text) | |
| cleaned_text=remove_urls(cleaned_text) | |
| cleaned_text=remove_punctuation(cleaned_text) | |
| cleaned_text=remove_stopwords(cleaned_text) | |
| cleaned_text=lemmatizing(cleaned_text) | |
| cleaned_text= spell_checker(cleaned_text) | |
| return cleaned_text | |