Spaces:
Runtime error
Runtime error
Download Malicious_Email_Classificator.py from Pmunozac/Clasificator: direct link, hf CLI and curl.
- Browser
- Download file 9.71 kB
-
https://huggingface.co/spaces/Pmunozac/Clasificator/resolve/main/Malicious_Email_Classificator.py
- Command line
-
hf download hf://spaces/Pmunozac/Clasificator/Malicious_Email_Classificator.py
-
curl -L -o Malicious_Email_Classificator.py https://huggingface.co/spaces/Pmunozac/Clasificator/resolve/main/Malicious_Email_Classificator.py
9.71 kB
| import gradio as gr | |
| import csv | |
| from collections import Counter | |
| import re | |
| import urllib | |
| from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer | |
| from sklearn.naive_bayes import MultinomialNB | |
| from sklearn.ensemble import RandomForestClassifier | |
| from sklearn.pipeline import make_pipeline | |
| import numpy as np, pandas as pd | |
| from sklearn.metrics import confusion_matrix, accuracy_score,f1_score | |
| import numpy as np | |
| import nltk | |
| from nltk.stem import WordNetLemmatizer | |
| from nltk.corpus import stopwords | |
| from sentence_transformers import SentenceTransformer | |
| nltk.download('wordnet')#Download wordnet | |
| nltk.download('stopwords') | |
| lemmatizer = WordNetLemmatizer()#start lemmatizer | |
| nltk.download('punkt') | |
| nltk.download('omw-1.4') | |
| title = "Malicious Email Classificator" | |
| ###################################Read the Corpus | |
| def readCorpus(): | |
| text=[] | |
| category=[] | |
| values=(text,category) | |
| with open('Dataset.csv') as file_obj: | |
| #read csv dataset | |
| reader_obj = csv.reader(file_obj) | |
| # Iterate the dataset | |
| for row in reader_obj: | |
| values[0].append(row[0])#0 is the text pos | |
| values[1].append(int(row[1]))#1 is the classification pos: ham=0, Spam=1 | |
| file_obj.close() | |
| return (values[0],values[1])#Tuple with (value,category) | |
| def readCorpus_SpamAssasin(): | |
| text=[] | |
| category=[] | |
| values=(text,category) | |
| with open('Dataset1.csv') as file_obj: | |
| #read csv dataset | |
| reader_obj = csv.reader(file_obj) | |
| # Iterate the dataset | |
| for row in reader_obj: | |
| values[0].append(row[0])#0 is the text pos | |
| values[1].append(int(row[1]))#1 is the classification pos: ham=0, Spam=1 | |
| file_obj.close() | |
| return (values[0],values[1])#Tuple with (value,category) | |
| def readCorpus_Enron(): | |
| text=[] | |
| category=[] | |
| values=(text,category) | |
| with open('Dataset2.csv') as file_obj: | |
| #read csv dataset | |
| reader_obj = csv.reader(file_obj) | |
| # Iterate the dataset | |
| for row in reader_obj: | |
| values[0].append(row[0])#0 is the text pos | |
| values[1].append(int(row[1]))#1 is the classification pos: ham=0, Spam=1 | |
| file_obj.close() | |
| return (values[0],values[1])#Tuple with (value,category) | |
| ######################################Train the model | |
| train_data = readCorpus() | |
| train_data_SpamAssasin = readCorpus_SpamAssasin() | |
| train_data_Enron = readCorpus_Enron() | |
| text_categories = ["ham","spam"] | |
| #BERT embedding model | |
| embeddingModel = SentenceTransformer('all-mpnet-base-v2') | |
| embeddings = embeddingModel.encode(train_data[0]) | |
| embeddings_1 = embeddingModel.encode(train_data_SpamAssasin[0]) | |
| embeddings_2 = embeddingModel.encode(train_data_Enron[0]) | |
| # Build model | |
| # Train model using training dataset | |
| #########################################All Datasets | |
| model1 = make_pipeline(TfidfVectorizer(), MultinomialNB()) | |
| model1.fit(train_data[0], train_data[1]) | |
| model2 = make_pipeline(CountVectorizer(), MultinomialNB()) | |
| model2.fit(train_data[0], train_data[1]) | |
| model3 = make_pipeline(TfidfVectorizer(), RandomForestClassifier()) | |
| model3.fit(train_data[0], train_data[1]) | |
| model4 = make_pipeline(CountVectorizer(), RandomForestClassifier()) | |
| model4.fit(train_data[0], train_data[1]) | |
| model5=RandomForestClassifier() | |
| model5.fit(embeddings, train_data[1]) | |
| #########################################SpamAssasin Dataset | |
| model1_1 = make_pipeline(TfidfVectorizer(), MultinomialNB()) | |
| model1_1.fit(train_data_SpamAssasin[0], train_data_SpamAssasin[1]) | |
| model2_1 = make_pipeline(CountVectorizer(), MultinomialNB()) | |
| model2_1.fit(train_data_SpamAssasin[0], train_data_SpamAssasin[1]) | |
| model3_1 = make_pipeline(TfidfVectorizer(), RandomForestClassifier()) | |
| model3_1.fit(train_data_SpamAssasin[0], train_data_SpamAssasin[1]) | |
| model4_1 = make_pipeline(CountVectorizer(), RandomForestClassifier()) | |
| model4_1.fit(train_data_SpamAssasin[0], train_data_SpamAssasin[1]) | |
| model5_1=RandomForestClassifier() | |
| model5_1.fit(embeddings_1, train_data_SpamAssasin[1]) | |
| #########################################Enron Dataset | |
| model1_2 = make_pipeline(TfidfVectorizer(), MultinomialNB()) | |
| model1_2.fit(train_data_Enron[0], train_data_Enron[1]) | |
| model2_2 = make_pipeline(CountVectorizer(), MultinomialNB()) | |
| model2_2.fit(train_data_Enron[0], train_data_Enron[1]) | |
| model3_2 = make_pipeline(TfidfVectorizer(), RandomForestClassifier()) | |
| model3_2.fit(train_data_Enron[0], train_data_Enron[1]) | |
| model4_2 = make_pipeline(CountVectorizer(), RandomForestClassifier()) | |
| model4_2.fit(train_data_Enron[0], train_data_Enron[1]) | |
| model5_2=RandomForestClassifier() | |
| model5_2.fit(embeddings_2, train_data_Enron[1]) | |
| #######################################Manage text | |
| def getParsedText(text): | |
| Text_NoSpecialCharacters = re.sub('[^A-Za-z0-9]+', ' ', text)#Text_NoSpecialCharacters | |
| Text_NoSpecialCharacters_lower=Text_NoSpecialCharacters.lower()#Text_NoSpecialCharacters_lower | |
| Text_NoSpecialCharacters_lower_tokenized = nltk.word_tokenize(Text_NoSpecialCharacters_lower)#Text_NoSpecialCharacters_lower_tokenized | |
| Text_NoSpecialCharacters_lower_tokenized_NoStopWords= [word for word in Text_NoSpecialCharacters_lower_tokenized if word not in stopwords.words('english')]#Text_NoSpecialCharacters_lower_tokenized_NoStopWords | |
| lemmatized_output = ' '.join([lemmatizer.lemmatize(w) for w in Text_NoSpecialCharacters_lower_tokenized_NoStopWords])#lematized output | |
| return lemmatized_output#finally return a string of lemmatized words | |
| #######################################Classify | |
| def classify(text,chosenModel,chosenDataset): | |
| if (chosenModel == "Naive Bayes + TfidfVectorizer" and chosenDataset == "All Datasets"): | |
| prediction= (np.array(text_categories)[model1.predict([getParsedText(text)])]) | |
| return prediction[0] | |
| elif (chosenModel == "Naive Bayes + CountVectorizer" and chosenDataset == "All Datasets"): | |
| prediction= (np.array(text_categories)[model2.predict([getParsedText(text)])]) | |
| return prediction[0] | |
| elif (chosenModel == "RandomForest + TfidfVectorizer" and chosenDataset == "All Datasets"): | |
| prediction= (np.array(text_categories)[model3.predict([getParsedText(text)])]) | |
| return prediction[0] | |
| elif (chosenModel == "RandomForest + CountVectorizer" and chosenDataset == "All Datasets"): | |
| prediction= (np.array(text_categories)[model4.predict([getParsedText(text)])]) | |
| return prediction[0] | |
| elif (chosenModel == "RandomForest + Bert Embbeding" and chosenDataset == "All Datasets"): | |
| prediction= (np.array(text_categories)[model5.predict([embeddingModel.encode(getParsedText(text))])]) | |
| return prediction[0] ########################################################################################## | |
| elif (chosenModel == "Naive Bayes + TfidfVectorizer" and chosenDataset == "SpamAssasin Dataset"): | |
| prediction= (np.array(text_categories)[model1_1.predict([getParsedText(text)])]) | |
| return prediction[0] | |
| elif (chosenModel == "Naive Bayes + CountVectorizer" and chosenDataset == "SpamAssasin Dataset"): | |
| prediction= (np.array(text_categories)[model2_1.predict([getParsedText(text)])]) | |
| return prediction[0] | |
| elif (chosenModel == "RandomForest + TfidfVectorizer" and chosenDataset == "SpamAssasin Dataset"): | |
| prediction= (np.array(text_categories)[model3_1.predict([getParsedText(text)])]) | |
| return prediction[0] | |
| elif (chosenModel == "RandomForest + CountVectorizer" and chosenDataset == "SpamAssasin Dataset"): | |
| prediction= (np.array(text_categories)[model4_1.predict([getParsedText(text)])]) | |
| return prediction[0] | |
| elif (chosenModel == "RandomForest + Bert Embbeding" and chosenDataset == "SpamAssasin Dataset"): | |
| prediction= (np.array(text_categories)[model5_1.predict([embeddingModel.encode(getParsedText(text))])]) | |
| return prediction[0]############################################################################################# | |
| elif (chosenModel == "Naive Bayes + TfidfVectorizer" and chosenDataset == "Enron Dataset"): | |
| prediction= (np.array(text_categories)[model1_2.predict([getParsedText(text)])]) | |
| return prediction[0] | |
| elif (chosenModel == "Naive Bayes + CountVectorizer" and chosenDataset == "Enron Dataset"): | |
| prediction= (np.array(text_categories)[model2_2.predict([getParsedText(text)])]) | |
| return prediction[0] | |
| elif (chosenModel == "RandomForest + TfidfVectorizer" and chosenDataset == "Enron Dataset"): | |
| prediction= (np.array(text_categories)[model3_2.predict([getParsedText(text)])]) | |
| return prediction[0] | |
| elif (chosenModel == "RandomForest + CountVectorizer" and chosenDataset == "Enron Dataset"): | |
| prediction= (np.array(text_categories)[model4_2.predict([getParsedText(text)])]) | |
| return prediction[0] | |
| elif (chosenModel == "RandomForest + Bert Embbeding" and chosenDataset == "Enron Dataset"): | |
| prediction= (np.array(text_categories)[model5_2.predict([embeddingModel.encode(getParsedText(text))])]) | |
| return prediction[0] | |
| ########################################Launch interface | |
| iface = gr.Interface(classify, [gr.inputs.Textbox(),gr.inputs.Dropdown(["Naive Bayes + TfidfVectorizer","Naive Bayes + CountVectorizer","RandomForest + TfidfVectorizer","RandomForest + CountVectorizer","RandomForest + Bert Embbeding"]),gr.inputs.Dropdown(["All Datasets","SpamAssasin Dataset","Enron Dataset"])],title=title,examples=[["Hello! Do you want to buy some viagra?","Naive Bayes + TfidfVectorizer","All Datasets"],["Hello team, Could you please check? We need to solve this ASAP","RandomForest + TfidfVectorizer","All Datasets"]], outputs="text") | |
| iface.launch() |