import os import gradio as gr from functools import partial from utils import * from nets import NETS def get_model_files(dataset): base_path = './models/' dataset_path = { 'MNIST': 'mnist_nets/', 'CIFAR-10': 'cifar_10_nets/', 'CIFAR-100': 'cifar_100_nets/' } full_path = base_path + dataset_path[dataset] result = [f for f in os.listdir(full_path) if f.endswith('.pth')] return result default_epochs = { 'MNIST': 10, 'CIFAR-10': 15, 'CIFAR-100': 20 } with gr.Blocks() as demo: with gr.Tab(label="FGSM Attack"): with gr.Column(): with gr.Row(): with gr.Column(): model_input = gr.Dropdown(label="Public Pretrained Model", choices=[ "AlexNet", "VGG16", "GoogLeNet", "ResNet18", "ResNet50", "Inception_V3", "DenseNet161", "ResNeXt50_32x4d", "MobileNet_V2" ], value="GoogLeNet") eps_input = gr.Slider(label="Epsilon (Level of attack)", minimum=0, maximum=1, step=0.001, value=0.007) image_input = gr.Image(label="Input Image", type="pil") pred_btn = gr.Button() label_output = gr.Label(label="Attacked Prediction") with gr.Row(): with gr.Column(): I1_output = gr.Image(label="Original Image") P1_output = gr.Textbox(label="Original Prediction") C1_output = gr.Textbox(label="Original Confidence") with gr.Column(): I2_output = gr.Image(label="Signed Gradient Image") P2_output = gr.Textbox(label="Signed Gradient Prediction") C2_output = gr.Textbox(label="Signed Gradient Confidence") with gr.Column(): I3_output = gr.Image(label="Perturbed Image") P3_output = gr.Textbox(label="Perturbed Prediction") C3_output = gr.Textbox(label="Perturbed Confidence") pred_btn.click(show_attack, inputs=[model_input, eps_input, image_input], outputs=[ label_output, I1_output, P1_output, C1_output, I2_output, P2_output, C2_output, I3_output, P3_output, C3_output]) gr.Examples([ ["GoogLeNet", 0.007, "attachments/examples/tab1/panda.png"], ["ResNet18", 0.017, "attachments/examples/tab1/duck.jpg"], ["Inception_V3", 0.017, "attachments/examples/tab1/tank.jpg"], ["Inception_V3", 0.007, "attachments/examples/tab1/desk.jpg"], ["AlexNet", 0.011, "attachments/examples/tab1/lion.jpg"], ["DenseNet161", 0.017, "attachments/examples/tab1/cauliflower.jpg"], ], inputs=[model_input, eps_input, image_input]) with gr.Tab(label="Adversarial Training"): for dataset in ["MNIST", "CIFAR-10", "CIFAR-100"]: with gr.Tab(label=dataset): with gr.Column(): with gr.Row(): with gr.Column(): model_input = gr.Dropdown(label="Model", choices=NETS[dataset]) eps_input = gr.Slider(label="Epsilon", minimum=0, maximum=1, step=0.001, value=0.007) alpha_input = gr.Slider(label="Alpha", minimum=0, maximum=1, step=0.01, value=0.5) batch_size_input = gr.Number(label="Batch Size", minimum=1, maximum=256, step=1, value=128) epochs_input = gr.Number(label="Epochs", minimum=1, maximum=100, step=1, value=default_epochs[dataset]) lr_max_input = gr.Slider(label="LR Max", minimum=0.001, maximum=0.1, step=0.001, value=5e-3) train_btn = gr.Button(value="Train") with gr.Column(): train_plot_output = gr.Plot(label="Training Plot") train_acc_output = gr.Number(label="Training Accuracy") val_acc_output = gr.Number(label="Validation Accuracy") # TODO add more results of adversarial training train_btn.click(partial(adversarial_training, dataset=dataset), inputs=[model_input, eps_input, alpha_input, batch_size_input, epochs_input, lr_max_input], outputs=[train_plot_output, train_acc_output, val_acc_output]) if dataset == 'MNIST': gr.Examples([ ['LinearNet', 0, 0, 128, 10, 5e-3], ['LinearNet', 7e-3, 0.5, 128, 10, 5e-3] ], inputs=[model_input, eps_input, alpha_input, batch_size_input, epochs_input, lr_max_input] ) elif dataset == 'CIFAR-10': gr.Examples([ ['CNN', 0, 0, 128, 15, 5e-3], ['CNN', 7e-3, 0.5, 128, 15, 5e-3] ], inputs=[model_input, eps_input, alpha_input, batch_size_input, epochs_input, lr_max_input] ) elif dataset == 'CIFAR-100': gr.Examples([ ['ResNet18', 0, 0, 128, 20, 5e-3], ['ResNet18', 7e-3, 0.5, 128, 20, 5e-3] ], inputs=[model_input, eps_input, alpha_input, batch_size_input, epochs_input, lr_max_input] ) with gr.Tab(label="Comprehensive Robustness Evaluation System using FGSM"): with gr.Tab(label="FGSM Evaluate"): for dataset in ["MNIST", "CIFAR-10", "CIFAR-100"]: with gr.Tab(label=dataset): with gr.Column(): with gr.Row(): with gr.Column(): model_input = gr.Dropdown(label="Model", choices=get_model_files(dataset)) # TODO find a way to update model files eps_input = gr.Slider(label="Epsilon", minimum=0, maximum=1, step=0.001, value=0.007) batch_size_input = gr.Number(label="Batch Size", minimum=1, maximum=256, step=1, value=128) evaluate_btn = gr.Button(value="Run") with gr.Column(): with gr.Row(): eval_acc_before_output = gr.Number(label="Accuracy Before Attack") true_confidence_before_output = gr.Number(label="Correct Confidence Before Attack") false_confidence_before_output = gr.Number(label="Incorrect Confidence Before Attack") with gr.Row(): eval_acc_after_output = gr.Number(label="Accuracy After Attack") true_confidence_after_output = gr.Number(label="Correct Confidence After Attack") false_confidence_after_output = gr.Number(label=" Incorrect Confidence After Attack") CRE_Index_output = gr.Number(label="CRE-Index") evaluate_btn.click(partial(show_evaluation, dataset=dataset), inputs=[model_input, eps_input, batch_size_input], outputs=[ eval_acc_before_output, true_confidence_before_output, false_confidence_before_output, eval_acc_after_output, true_confidence_after_output, false_confidence_after_output, CRE_Index_output ]) if dataset == 'MNIST': gr.Examples([ ['linearnet_epsilon0_alpha0_epochs10.pth', 0.007, 128], ], inputs=[model_input, eps_input, batch_size_input] ) elif dataset == 'CIFAR-10': gr.Examples([ ['cnn_epsilon0_alpha0_epochs15.pth', 0.007, 128], ], inputs=[model_input, eps_input, batch_size_input] ) elif dataset == 'CIFAR-100': gr.Examples([ ['resnet18_epsilon0_alpha0_epochs20.pth', 0.007, 128], ], inputs=[model_input, eps_input, batch_size_input] ) with gr.Tab("Performance over Epsilon"): for dataset in ["MNIST", "CIFAR-10", "CIFAR-100"]: with gr.Tab(label=dataset): with gr.Column(): with gr.Row(): with gr.Column(): model_input = gr.Dropdown(label="Model", choices=get_model_files(dataset)) batch_size_input = gr.Number(label="Batch Size", minimum=1, maximum=256, step=1, value=128) evaluate_btn = gr.Button(value="Run") with gr.Column(): performance_plot_output = gr.Plot(label="Performance Plot") evaluate_btn.click(partial(show_performance, dataset=dataset), inputs=[model_input, batch_size_input], outputs=[performance_plot_output]) if dataset == 'MNIST': gr.Examples([ ['linearnet_epsilon0_alpha0_epochs10.pth', 128], ], inputs=[model_input, batch_size_input] ) elif dataset == 'CIFAR-10': gr.Examples([ ['cnn_epsilon0_alpha0_epochs15.pth', 128], ], inputs=[model_input, batch_size_input] ) elif dataset == 'CIFAR-100': gr.Examples([ ['resnet18_epsilon0_alpha0_epochs20.pth', 128], ], inputs=[model_input, batch_size_input] ) with gr.Tab("Robustness Evaluation"): for dataset in ["MNIST", "CIFAR-10", "CIFAR-100"]: with gr.Tab(label=dataset): with gr.Column(): models_input = gr.Dropdown(label="Models", choices=get_model_files(dataset), multiselect=True) eps_input = gr.Slider(label="Epsilon", minimum=0, maximum=1, step=0.001, value=0.007) batch_size_input = gr.Number(label="Batch Size", minimum=1, maximum=256, step=1, value=128) evaluate_btn = gr.Button(value="Run") robustness_df_output = gr.Dataframe(label="Robustness Ranking") evaluate_btn.click(partial(show_robustness, dataset=dataset), inputs=[models_input, eps_input, batch_size_input], outputs=[robustness_df_output]) if dataset == 'MNIST': gr.Examples([ [[ 'linearnet_epsilon0_alpha0_epochs10.pth', 'rbfnet_epsilon0_alpha0_epochs10.pth', 'mlp_epsilon0_alpha0_epochs10.pth', 'maxoutmlp_epsilon0_alpha0_epochs10.pth', 'cnn_epsilon0_alpha0_epochs10.pth', 'maxoutcnn_epsilon0_alpha0_epochs10.pth', ], 0.007, 128], [[ 'linearnet_epsilon0_alpha0_epochs10.pth', 'rbfnet_epsilon0_alpha0_epochs10.pth', 'mlp_epsilon0_alpha0_epochs10.pth', 'maxoutmlp_epsilon0_alpha0_epochs10.pth', 'cnn_epsilon0_alpha0_epochs10.pth', 'maxoutcnn_epsilon0_alpha0_epochs10.pth', 'linearnet_epsilon0.007_alpha0.5_epochs10.pth', 'rbfnet_epsilon0.007_alpha0.5_epochs10.pth', 'mlp_epsilon0.007_alpha0.5_epochs10.pth', 'maxoutmlp_epsilon0.007_alpha0.5_epochs10.pth', 'cnn_epsilon0.007_alpha0.5_epochs10.pth', 'maxoutcnn_epsilon0.007_alpha0.5_epochs10.pth', ], 0.007, 128], [[ 'linearnet_epsilon0.007_alpha0.5_epochs10.pth', 'rbfnet_epsilon0.007_alpha0.5_epochs10.pth', 'mlp_epsilon0.007_alpha0.5_epochs10.pth', 'maxoutmlp_epsilon0.007_alpha0.5_epochs10.pth', 'cnn_epsilon0.007_alpha0.5_epochs10.pth', 'maxoutcnn_epsilon0.007_alpha0.5_epochs10.pth', ], 0.1, 128] ], inputs=[models_input, eps_input, batch_size_input]) elif dataset == 'CIFAR-10': gr.Examples([ [[ 'cnn_epsilon0_alpha0_epochs15.pth', 'maxoutcnn_epsilon0_alpha0_epochs15.pth' ], 0.007, 128], ], inputs=[models_input, eps_input, batch_size_input]) elif dataset == 'CIFAR-100': gr.Examples([ [[ 'cnn_epsilon0_alpha0_epochs20.pth', 'resnet18_epsilon0_alpha0_epochs20.pth' ], 0.007, 128], ], inputs=[models_input, eps_input, batch_size_input]) if __name__ == '__main__': # if necessary, you can uncomment the following lines to download the pretrained models and labels #get_pretrained_models() # get_imagenet_labels() demo.launch( share=True, # auth=[ # ("admin", "admin"), # ("visitor", "visitor"), # ("tester", "tester"), # ], # auth_message=""" # Welcome to CRES(Comprehensive Robustness Evaluation System using FGSM)! # """, )