import gradio as gr import csv from collections import Counter import re import urllib from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer from sklearn.naive_bayes import MultinomialNB from sklearn.ensemble import RandomForestClassifier from sklearn.pipeline import make_pipeline import numpy as np, pandas as pd from sklearn.metrics import confusion_matrix, accuracy_score,f1_score import numpy as np import nltk from nltk.stem import WordNetLemmatizer from nltk.corpus import stopwords from sentence_transformers import SentenceTransformer nltk.download('wordnet')#Download wordnet nltk.download('stopwords') lemmatizer = WordNetLemmatizer()#start lemmatizer nltk.download('punkt') nltk.download('omw-1.4') title = "Malicious Email Classificator" ###################################Read the Corpus def readCorpus(): text=[] category=[] values=(text,category) with open('Dataset.csv') as file_obj: #read csv dataset reader_obj = csv.reader(file_obj) # Iterate the dataset for row in reader_obj: values[0].append(row[0])#0 is the text pos values[1].append(int(row[1]))#1 is the classification pos: ham=0, Spam=1 file_obj.close() return (values[0],values[1])#Tuple with (value,category) def readCorpus_SpamAssasin(): text=[] category=[] values=(text,category) with open('Dataset1.csv') as file_obj: #read csv dataset reader_obj = csv.reader(file_obj) # Iterate the dataset for row in reader_obj: values[0].append(row[0])#0 is the text pos values[1].append(int(row[1]))#1 is the classification pos: ham=0, Spam=1 file_obj.close() return (values[0],values[1])#Tuple with (value,category) def readCorpus_Enron(): text=[] category=[] values=(text,category) with open('Dataset2.csv') as file_obj: #read csv dataset reader_obj = csv.reader(file_obj) # Iterate the dataset for row in reader_obj: values[0].append(row[0])#0 is the text pos values[1].append(int(row[1]))#1 is the classification pos: ham=0, Spam=1 file_obj.close() return (values[0],values[1])#Tuple with (value,category) ######################################Train the model train_data = readCorpus() train_data_SpamAssasin = readCorpus_SpamAssasin() train_data_Enron = readCorpus_Enron() text_categories = ["ham","spam"] #BERT embedding model embeddingModel = SentenceTransformer('all-mpnet-base-v2') embeddings = embeddingModel.encode(train_data[0]) embeddings_1 = embeddingModel.encode(train_data_SpamAssasin[0]) embeddings_2 = embeddingModel.encode(train_data_Enron[0]) # Build model # Train model using training dataset #########################################All Datasets model1 = make_pipeline(TfidfVectorizer(), MultinomialNB()) model1.fit(train_data[0], train_data[1]) model2 = make_pipeline(CountVectorizer(), MultinomialNB()) model2.fit(train_data[0], train_data[1]) model3 = make_pipeline(TfidfVectorizer(), RandomForestClassifier()) model3.fit(train_data[0], train_data[1]) model4 = make_pipeline(CountVectorizer(), RandomForestClassifier()) model4.fit(train_data[0], train_data[1]) model5=RandomForestClassifier() model5.fit(embeddings, train_data[1]) #########################################SpamAssasin Dataset model1_1 = make_pipeline(TfidfVectorizer(), MultinomialNB()) model1_1.fit(train_data_SpamAssasin[0], train_data_SpamAssasin[1]) model2_1 = make_pipeline(CountVectorizer(), MultinomialNB()) model2_1.fit(train_data_SpamAssasin[0], train_data_SpamAssasin[1]) model3_1 = make_pipeline(TfidfVectorizer(), RandomForestClassifier()) model3_1.fit(train_data_SpamAssasin[0], train_data_SpamAssasin[1]) model4_1 = make_pipeline(CountVectorizer(), RandomForestClassifier()) model4_1.fit(train_data_SpamAssasin[0], train_data_SpamAssasin[1]) model5_1=RandomForestClassifier() model5_1.fit(embeddings_1, train_data_SpamAssasin[1]) #########################################Enron Dataset model1_2 = make_pipeline(TfidfVectorizer(), MultinomialNB()) model1_2.fit(train_data_Enron[0], train_data_Enron[1]) model2_2 = make_pipeline(CountVectorizer(), MultinomialNB()) model2_2.fit(train_data_Enron[0], train_data_Enron[1]) model3_2 = make_pipeline(TfidfVectorizer(), RandomForestClassifier()) model3_2.fit(train_data_Enron[0], train_data_Enron[1]) model4_2 = make_pipeline(CountVectorizer(), RandomForestClassifier()) model4_2.fit(train_data_Enron[0], train_data_Enron[1]) model5_2=RandomForestClassifier() model5_2.fit(embeddings_2, train_data_Enron[1]) #######################################Manage text def getParsedText(text): Text_NoSpecialCharacters = re.sub('[^A-Za-z0-9]+', ' ', text)#Text_NoSpecialCharacters Text_NoSpecialCharacters_lower=Text_NoSpecialCharacters.lower()#Text_NoSpecialCharacters_lower Text_NoSpecialCharacters_lower_tokenized = nltk.word_tokenize(Text_NoSpecialCharacters_lower)#Text_NoSpecialCharacters_lower_tokenized Text_NoSpecialCharacters_lower_tokenized_NoStopWords= [word for word in Text_NoSpecialCharacters_lower_tokenized if word not in stopwords.words('english')]#Text_NoSpecialCharacters_lower_tokenized_NoStopWords lemmatized_output = ' '.join([lemmatizer.lemmatize(w) for w in Text_NoSpecialCharacters_lower_tokenized_NoStopWords])#lematized output return lemmatized_output#finally return a string of lemmatized words #######################################Classify def classify(text,chosenModel,chosenDataset): if (chosenModel == "Naive Bayes + TfidfVectorizer" and chosenDataset == "All Datasets"): prediction= (np.array(text_categories)[model1.predict([getParsedText(text)])]) return prediction[0] elif (chosenModel == "Naive Bayes + CountVectorizer" and chosenDataset == "All Datasets"): prediction= (np.array(text_categories)[model2.predict([getParsedText(text)])]) return prediction[0] elif (chosenModel == "RandomForest + TfidfVectorizer" and chosenDataset == "All Datasets"): prediction= (np.array(text_categories)[model3.predict([getParsedText(text)])]) return prediction[0] elif (chosenModel == "RandomForest + CountVectorizer" and chosenDataset == "All Datasets"): prediction= (np.array(text_categories)[model4.predict([getParsedText(text)])]) return prediction[0] elif (chosenModel == "RandomForest + Bert Embbeding" and chosenDataset == "All Datasets"): prediction= (np.array(text_categories)[model5.predict([embeddingModel.encode(getParsedText(text))])]) return prediction[0] ########################################################################################## elif (chosenModel == "Naive Bayes + TfidfVectorizer" and chosenDataset == "SpamAssasin Dataset"): prediction= (np.array(text_categories)[model1_1.predict([getParsedText(text)])]) return prediction[0] elif (chosenModel == "Naive Bayes + CountVectorizer" and chosenDataset == "SpamAssasin Dataset"): prediction= (np.array(text_categories)[model2_1.predict([getParsedText(text)])]) return prediction[0] elif (chosenModel == "RandomForest + TfidfVectorizer" and chosenDataset == "SpamAssasin Dataset"): prediction= (np.array(text_categories)[model3_1.predict([getParsedText(text)])]) return prediction[0] elif (chosenModel == "RandomForest + CountVectorizer" and chosenDataset == "SpamAssasin Dataset"): prediction= (np.array(text_categories)[model4_1.predict([getParsedText(text)])]) return prediction[0] elif (chosenModel == "RandomForest + Bert Embbeding" and chosenDataset == "SpamAssasin Dataset"): prediction= (np.array(text_categories)[model5_1.predict([embeddingModel.encode(getParsedText(text))])]) return prediction[0]############################################################################################# elif (chosenModel == "Naive Bayes + TfidfVectorizer" and chosenDataset == "Enron Dataset"): prediction= (np.array(text_categories)[model1_2.predict([getParsedText(text)])]) return prediction[0] elif (chosenModel == "Naive Bayes + CountVectorizer" and chosenDataset == "Enron Dataset"): prediction= (np.array(text_categories)[model2_2.predict([getParsedText(text)])]) return prediction[0] elif (chosenModel == "RandomForest + TfidfVectorizer" and chosenDataset == "Enron Dataset"): prediction= (np.array(text_categories)[model3_2.predict([getParsedText(text)])]) return prediction[0] elif (chosenModel == "RandomForest + CountVectorizer" and chosenDataset == "Enron Dataset"): prediction= (np.array(text_categories)[model4_2.predict([getParsedText(text)])]) return prediction[0] elif (chosenModel == "RandomForest + Bert Embbeding" and chosenDataset == "Enron Dataset"): prediction= (np.array(text_categories)[model5_2.predict([embeddingModel.encode(getParsedText(text))])]) return prediction[0] ########################################Launch interface iface = gr.Interface(classify, [gr.inputs.Textbox(),gr.inputs.Dropdown(["Naive Bayes + TfidfVectorizer","Naive Bayes + CountVectorizer","RandomForest + TfidfVectorizer","RandomForest + CountVectorizer","RandomForest + Bert Embbeding"]),gr.inputs.Dropdown(["All Datasets","SpamAssasin Dataset","Enron Dataset"])],title=title,examples=[["Hello! Do you want to buy some viagra?","Naive Bayes + TfidfVectorizer","All Datasets"],["Hello team, Could you please check? We need to solve this ASAP","RandomForest + TfidfVectorizer","All Datasets"]], outputs="text") iface.launch()