{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "id": "8Y-Xb8GkQNJ2" }, "outputs": [], "source": [ "import pandas as pd\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.linear_model import LogisticRegression\n", "from sklearn.tree import DecisionTreeClassifier\n", "from sklearn.ensemble import RandomForestClassifier\n", "from xgboost import XGBClassifier\n", "from sklearn.svm import SVC\n", "from sklearn.neighbors import KNeighborsClassifier\n", "from sklearn.naive_bayes import GaussianNB\n", "from sklearn.neural_network import MLPClassifier\n", "from sklearn.metrics import confusion_matrix, roc_auc_score, roc_curve\n", "import matplotlib.pyplot as plt\n", "\n", "\n", "# Membaca dataset\n", "df = pd.read_csv('./cm1.csv')\n", "\n", "# Memisahkan fitur dan target\n", "X = df.drop(['defects'], axis=1)\n", "y = df['defects']\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "id": "Ua-r1TS4QZut" }, "outputs": [], "source": [ "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "A1I3pNGqQbaX", "outputId": "8a10c6c2-78a6-4f4c-c297-ba45363fff74" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Shape of X_train: (398, 21)\n", "Shape of X_test: (100, 21)\n", "Shape of y_train: (398,)\n", "Shape of y_test: (100,)\n" ] } ], "source": [ "print(f\"Shape of X_train: {X_train.shape}\")\n", "print(f\"Shape of X_test: {X_test.shape}\")\n", "print(f\"Shape of y_train: {y_train.shape}\")\n", "print(f\"Shape of y_test: {y_test.shape}\")" ] }, { "cell_type": "code", "source": [ "models = {\n", " 'Logistic Regression': LogisticRegression(max_iter=10000),\n", " 'Decision Tree': DecisionTreeClassifier(),\n", " 'Random Forest': RandomForestClassifier(),\n", " 'XGBoost': XGBClassifier(),\n", " 'KNN': KNeighborsClassifier(),\n", " 'Naive Bayes': GaussianNB(),\n", " 'Neural Network': MLPClassifier(max_iter=10000),\n", " 'SVM' : SVC(probability=True)\n", "}" ], "metadata": { "id": "fnDfC42o5aSz" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "def evaluate_model(model, X_train, y_train, X_test, y_test):\n", " model.fit(X_train, y_train)\n", " y_pred = model.predict(X_test)\n", " y_pred_proba = model.predict_proba(X_test)[:, 1]\n", " cm = confusion_matrix(y_test, y_pred)\n", " roc_auc = roc_auc_score(y_test, y_pred_proba)\n", " fpr, tpr, _ = roc_curve(y_test, y_pred_proba)\n", " lift_chart = create_lift_chart(model, X_test, y_test)\n", " return cm, roc_auc, fpr, tpr, lift_chart" ], "metadata": { "id": "TJkU4yxB5oIC" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "def create_lift_chart(model, X, actual_target):\n", " avg_tgt = actual_target.sum() / len(actual_target)\n", " df_data = X.copy()\n", " X_data = df_data.copy()\n", " df_data['Actual'] = actual_target\n", " df_data['Predict'] = model.predict(X_data)\n", " y_Prob = pd.DataFrame(model.predict_proba(X_data))\n", " df_data['Prob_1'] = list(y_Prob[1])\n", " df_data.sort_values(by=['Prob_1'], ascending=False, inplace=True)\n", " df_data.reset_index(drop=True, inplace=True)\n", " decile_size = int(len(df_data) / 10)\n", " df_data['Decile'] = pd.Series(range(len(df_data))) // decile_size\n", " output_df = pd.DataFrame()\n", " grouped = df_data.groupby('Decile', as_index=False)\n", " output_df['Max_Scr'] = grouped.max().Prob_1\n", " output_df['Min_Scr'] = grouped.min().Prob_1\n", " output_df['Actual'] = grouped.sum().Actual\n", " output_df['Total'] = grouped.count().Actual\n", " output_df[\"Population_perc\"] = (output_df[\"Total\"] / len(actual_target)) * 100\n", " output_df['Per_Events'] = (output_df['Actual'] / output_df['Actual'].sum()) * 100\n", " output_df['Gain_Percentage'] = output_df.Per_Events.cumsum()\n", " output_df[\"Cumulative_Population\"] = output_df.Population_perc.cumsum()\n", " output_df[\"Lift\"] = output_df[\"Gain_Percentage\"] / output_df[\"Cumulative_Population\"]\n", " return output_df\n" ], "metadata": { "id": "1tlKVc7r5pi2" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "evaluation_results_no_resampling = {}\n", "\n", "for model_name, model in models.items():\n", " cm, roc_auc, fpr, tpr, lift_chart = evaluate_model(model, X_train, y_train, X_test, y_test)\n", " evaluation_results_no_resampling[model_name] = {\n", " 'Confusion Matrix': cm,\n", " 'ROC AUC': roc_auc,\n", " 'FPR': fpr,\n", " 'TPR': tpr,\n", " 'Lift Chart': lift_chart\n", " }\n", " print(f\"Model: {model_name}\")\n", " print(f\"Confusion Matrix:\\n{cm}\")\n", " print(f\"ROC AUC: {roc_auc}\")\n", " print()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "vYspKy2a5rgC", "outputId": "d3bf4516-8af4-4774-c30e-41f8b08fc8dd" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Model: Logistic Regression\n", "Confusion Matrix:\n", "[[87 1]\n", " [12 0]]\n", "ROC AUC: 0.4337121212121212\n", "\n", "Model: Decision Tree\n", "Confusion Matrix:\n", "[[82 6]\n", " [ 8 4]]\n", "ROC AUC: 0.6325757575757576\n", "\n", "Model: Random Forest\n", "Confusion Matrix:\n", "[[87 1]\n", " [12 0]]\n", "ROC AUC: 0.712594696969697\n", "\n", "Model: XGBoost\n", "Confusion Matrix:\n", "[[85 3]\n", " [11 1]]\n", "ROC AUC: 0.8087121212121212\n", "\n", "Model: KNN\n", "Confusion Matrix:\n", "[[88 0]\n", " [12 0]]\n", "ROC AUC: 0.625\n", "\n", "Model: Naive Bayes\n", "Confusion Matrix:\n", "[[86 2]\n", " [10 2]]\n", "ROC AUC: 0.7793560606060606\n", "\n", "Model: Neural Network\n", "Confusion Matrix:\n", "[[88 0]\n", " [12 0]]\n", "ROC AUC: 0.3001893939393939\n", "\n", "Model: SVM\n", "Confusion Matrix:\n", "[[88 0]\n", " [12 0]]\n", "ROC AUC: 0.7679924242424242\n", "\n" ] } ] }, { "cell_type": "code", "source": [ "import openpyxl\n", "from openpyxl import Workbook\n", "from openpyxl.utils.dataframe import dataframe_to_rows\n", "\n", "# Membuat workbook baru\n", "wb = Workbook()\n", "ws = wb.active\n", "ws.title = \"Evaluation Results\"\n", "\n", "# Menuliskan header\n", "headers = [\"Method\", \"Model\", \"TP\", \"TN\", \"FP\", \"FN\", \"ROC AUC\"]\n", "ws.append(headers)\n", "\n", "# Menuliskan hasil evaluasi\n", "for model_name, results in evaluation_results_no_resampling.items():\n", " cm = results['Confusion Matrix']\n", " tp = cm[1, 1]\n", " tn = cm[0, 0]\n", " fp = cm[0, 1]\n", " fn = cm[1, 0]\n", " roc_auc = results['ROC AUC']\n", " ws.append([model_name, tp, tn, fp, fn, roc_auc])\n", "\n", "# # Menyimpan workbook ke file\n", "# wb.save(\"evaluation_results.xlsx\")\n", "\n", "# Menyimpan Lift Chart ke dalam Sheet terpisah\n", "for model_name, results in evaluation_results_no_resampling.items():\n", " lift_df = results['Lift Chart']\n", " ws = wb.create_sheet(title=f\"{model_name}_LiftChart\")\n", " for r in dataframe_to_rows(lift_df, index=True, header=True):\n", " ws.append(r)\n", "\n", "wb.save(\"Evaluasi tanpa sampling.xlsx\")\n" ], "metadata": { "id": "eyGjSHDZ5tTF" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [], "metadata": { "id": "01ORp8yR6KYS" }, "execution_count": null, "outputs": [] } ], "metadata": { "colab": { "provenance": [] }, "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "name": "python" } }, "nbformat": 4, "nbformat_minor": 0 }