Spaces:
Runtime error
Runtime error
Download src/preprocessing.py from bhushan4829/novel_based_chatbot: direct link, hf CLI and curl.
- Browser
- Download file 1.17 kB
-
https://huggingface.co/spaces/bhushan4829/novel_based_chatbot/resolve/main/src/preprocessing.py
- Command line
-
hf download hf://spaces/bhushan4829/novel_based_chatbot/src/preprocessing.py
-
curl -L -o preprocessing.py https://huggingface.co/spaces/bhushan4829/novel_based_chatbot/resolve/main/src/preprocessing.py
1.17 kB
| import download_nltk | |
| import re | |
| import nltk | |
| from nltk.corpus import stopwords | |
| from nltk.tokenize import word_tokenize | |
| from nltk.stem import WordNetLemmatizer | |
| from nltk.stem import PorterStemmer | |
| nltk.data.path.append("./nltk_data") | |
| class Preprocessing: | |
| def preprocess_query(text): | |
| lemmatizer = WordNetLemmatizer() | |
| stop_words = set(stopwords.words('english')) | |
| text = re.sub(r'\s+', ' ', text) | |
| text = re.sub(r"[^a-zA-Z0-9.,!?':;]", ' ', text) | |
| text = text.lower() | |
| tokens = text.split() | |
| filtered_tokens = [lemmatizer.lemmatize(word) for word in tokens if word not in stop_words] | |
| return ' '.join(filtered_tokens) | |
| def preprocess_content(text): | |
| # Similar preprocessing as preprocess_query, but for content | |
| lemmatizer = WordNetLemmatizer() | |
| stop_words = set(stopwords.words('english')) | |
| text = re.sub(r'\s+', ' ', text) | |
| text = re.sub(r"[^a-zA-Z0-9.,!?':;]", ' ', text) | |
| text = text.lower() | |
| tokens = text.split() | |
| filtered_tokens = [lemmatizer.lemmatize(word) for word in tokens if word not in stop_words] | |
| return ' '.join(filtered_tokens) | |