Clasificator / Malicious_Email_Classificator.py
Pmunozac's picture
Update Malicious_Email_Classificator.py
ea39e7f
Raw History Blame Contribute Delete
9.71 kB
import gradio as gr
import csv
from collections import Counter
import re
import urllib
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import make_pipeline
import numpy as np, pandas as pd
from sklearn.metrics import confusion_matrix, accuracy_score,f1_score
import numpy as np
import nltk
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
from sentence_transformers import SentenceTransformer
nltk.download('wordnet')#Download wordnet
nltk.download('stopwords')
lemmatizer = WordNetLemmatizer()#start lemmatizer
nltk.download('punkt')
nltk.download('omw-1.4')
title = "Malicious Email Classificator"
###################################Read the Corpus
def readCorpus():
text=[]
category=[]
values=(text,category)
with open('Dataset.csv') as file_obj:
#read csv dataset
reader_obj = csv.reader(file_obj)
# Iterate the dataset
for row in reader_obj:
values[0].append(row[0])#0 is the text pos
values[1].append(int(row[1]))#1 is the classification pos: ham=0, Spam=1
file_obj.close()
return (values[0],values[1])#Tuple with (value,category)
def readCorpus_SpamAssasin():
text=[]
category=[]
values=(text,category)
with open('Dataset1.csv') as file_obj:
#read csv dataset
reader_obj = csv.reader(file_obj)
# Iterate the dataset
for row in reader_obj:
values[0].append(row[0])#0 is the text pos
values[1].append(int(row[1]))#1 is the classification pos: ham=0, Spam=1
file_obj.close()
return (values[0],values[1])#Tuple with (value,category)
def readCorpus_Enron():
text=[]
category=[]
values=(text,category)
with open('Dataset2.csv') as file_obj:
#read csv dataset
reader_obj = csv.reader(file_obj)
# Iterate the dataset
for row in reader_obj:
values[0].append(row[0])#0 is the text pos
values[1].append(int(row[1]))#1 is the classification pos: ham=0, Spam=1
file_obj.close()
return (values[0],values[1])#Tuple with (value,category)
######################################Train the model
train_data = readCorpus()
train_data_SpamAssasin = readCorpus_SpamAssasin()
train_data_Enron = readCorpus_Enron()
text_categories = ["ham","spam"]
#BERT embedding model
embeddingModel = SentenceTransformer('all-mpnet-base-v2')
embeddings = embeddingModel.encode(train_data[0])
embeddings_1 = embeddingModel.encode(train_data_SpamAssasin[0])
embeddings_2 = embeddingModel.encode(train_data_Enron[0])
# Build model
# Train model using training dataset
#########################################All Datasets
model1 = make_pipeline(TfidfVectorizer(), MultinomialNB())
model1.fit(train_data[0], train_data[1])
model2 = make_pipeline(CountVectorizer(), MultinomialNB())
model2.fit(train_data[0], train_data[1])
model3 = make_pipeline(TfidfVectorizer(), RandomForestClassifier())
model3.fit(train_data[0], train_data[1])
model4 = make_pipeline(CountVectorizer(), RandomForestClassifier())
model4.fit(train_data[0], train_data[1])
model5=RandomForestClassifier()
model5.fit(embeddings, train_data[1])
#########################################SpamAssasin Dataset
model1_1 = make_pipeline(TfidfVectorizer(), MultinomialNB())
model1_1.fit(train_data_SpamAssasin[0], train_data_SpamAssasin[1])
model2_1 = make_pipeline(CountVectorizer(), MultinomialNB())
model2_1.fit(train_data_SpamAssasin[0], train_data_SpamAssasin[1])
model3_1 = make_pipeline(TfidfVectorizer(), RandomForestClassifier())
model3_1.fit(train_data_SpamAssasin[0], train_data_SpamAssasin[1])
model4_1 = make_pipeline(CountVectorizer(), RandomForestClassifier())
model4_1.fit(train_data_SpamAssasin[0], train_data_SpamAssasin[1])
model5_1=RandomForestClassifier()
model5_1.fit(embeddings_1, train_data_SpamAssasin[1])
#########################################Enron Dataset
model1_2 = make_pipeline(TfidfVectorizer(), MultinomialNB())
model1_2.fit(train_data_Enron[0], train_data_Enron[1])
model2_2 = make_pipeline(CountVectorizer(), MultinomialNB())
model2_2.fit(train_data_Enron[0], train_data_Enron[1])
model3_2 = make_pipeline(TfidfVectorizer(), RandomForestClassifier())
model3_2.fit(train_data_Enron[0], train_data_Enron[1])
model4_2 = make_pipeline(CountVectorizer(), RandomForestClassifier())
model4_2.fit(train_data_Enron[0], train_data_Enron[1])
model5_2=RandomForestClassifier()
model5_2.fit(embeddings_2, train_data_Enron[1])
#######################################Manage text
def getParsedText(text):
Text_NoSpecialCharacters = re.sub('[^A-Za-z0-9]+', ' ', text)#Text_NoSpecialCharacters
Text_NoSpecialCharacters_lower=Text_NoSpecialCharacters.lower()#Text_NoSpecialCharacters_lower
Text_NoSpecialCharacters_lower_tokenized = nltk.word_tokenize(Text_NoSpecialCharacters_lower)#Text_NoSpecialCharacters_lower_tokenized
Text_NoSpecialCharacters_lower_tokenized_NoStopWords= [word for word in Text_NoSpecialCharacters_lower_tokenized if word not in stopwords.words('english')]#Text_NoSpecialCharacters_lower_tokenized_NoStopWords
lemmatized_output = ' '.join([lemmatizer.lemmatize(w) for w in Text_NoSpecialCharacters_lower_tokenized_NoStopWords])#lematized output
return lemmatized_output#finally return a string of lemmatized words
#######################################Classify
def classify(text,chosenModel,chosenDataset):
if (chosenModel == "Naive Bayes + TfidfVectorizer" and chosenDataset == "All Datasets"):
prediction= (np.array(text_categories)[model1.predict([getParsedText(text)])])
return prediction[0]
elif (chosenModel == "Naive Bayes + CountVectorizer" and chosenDataset == "All Datasets"):
prediction= (np.array(text_categories)[model2.predict([getParsedText(text)])])
return prediction[0]
elif (chosenModel == "RandomForest + TfidfVectorizer" and chosenDataset == "All Datasets"):
prediction= (np.array(text_categories)[model3.predict([getParsedText(text)])])
return prediction[0]
elif (chosenModel == "RandomForest + CountVectorizer" and chosenDataset == "All Datasets"):
prediction= (np.array(text_categories)[model4.predict([getParsedText(text)])])
return prediction[0]
elif (chosenModel == "RandomForest + Bert Embbeding" and chosenDataset == "All Datasets"):
prediction= (np.array(text_categories)[model5.predict([embeddingModel.encode(getParsedText(text))])])
return prediction[0] ##########################################################################################
elif (chosenModel == "Naive Bayes + TfidfVectorizer" and chosenDataset == "SpamAssasin Dataset"):
prediction= (np.array(text_categories)[model1_1.predict([getParsedText(text)])])
return prediction[0]
elif (chosenModel == "Naive Bayes + CountVectorizer" and chosenDataset == "SpamAssasin Dataset"):
prediction= (np.array(text_categories)[model2_1.predict([getParsedText(text)])])
return prediction[0]
elif (chosenModel == "RandomForest + TfidfVectorizer" and chosenDataset == "SpamAssasin Dataset"):
prediction= (np.array(text_categories)[model3_1.predict([getParsedText(text)])])
return prediction[0]
elif (chosenModel == "RandomForest + CountVectorizer" and chosenDataset == "SpamAssasin Dataset"):
prediction= (np.array(text_categories)[model4_1.predict([getParsedText(text)])])
return prediction[0]
elif (chosenModel == "RandomForest + Bert Embbeding" and chosenDataset == "SpamAssasin Dataset"):
prediction= (np.array(text_categories)[model5_1.predict([embeddingModel.encode(getParsedText(text))])])
return prediction[0]#############################################################################################
elif (chosenModel == "Naive Bayes + TfidfVectorizer" and chosenDataset == "Enron Dataset"):
prediction= (np.array(text_categories)[model1_2.predict([getParsedText(text)])])
return prediction[0]
elif (chosenModel == "Naive Bayes + CountVectorizer" and chosenDataset == "Enron Dataset"):
prediction= (np.array(text_categories)[model2_2.predict([getParsedText(text)])])
return prediction[0]
elif (chosenModel == "RandomForest + TfidfVectorizer" and chosenDataset == "Enron Dataset"):
prediction= (np.array(text_categories)[model3_2.predict([getParsedText(text)])])
return prediction[0]
elif (chosenModel == "RandomForest + CountVectorizer" and chosenDataset == "Enron Dataset"):
prediction= (np.array(text_categories)[model4_2.predict([getParsedText(text)])])
return prediction[0]
elif (chosenModel == "RandomForest + Bert Embbeding" and chosenDataset == "Enron Dataset"):
prediction= (np.array(text_categories)[model5_2.predict([embeddingModel.encode(getParsedText(text))])])
return prediction[0]
########################################Launch interface
iface = gr.Interface(classify, [gr.inputs.Textbox(),gr.inputs.Dropdown(["Naive Bayes + TfidfVectorizer","Naive Bayes + CountVectorizer","RandomForest + TfidfVectorizer","RandomForest + CountVectorizer","RandomForest + Bert Embbeding"]),gr.inputs.Dropdown(["All Datasets","SpamAssasin Dataset","Enron Dataset"])],title=title,examples=[["Hello! Do you want to buy some viagra?","Naive Bayes + TfidfVectorizer","All Datasets"],["Hello team, Could you please check? We need to solve this ASAP","RandomForest + TfidfVectorizer","All Datasets"]], outputs="text")
iface.launch()