{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "BI-LSTM"
      ],
      "metadata": {
        "id": "2GvF-KbdzFop"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "kFdyXq8axVn_"
      },
      "outputs": [],
      "source": [
        "import time\n",
        "import joblib\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "from tensorflow.keras.layers import Input\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.models import load_model\n",
        "from tensorflow.keras.layers import (\n",
        "    Embedding,\n",
        "    LSTM,\n",
        "    Bidirectional,\n",
        "    Dense,\n",
        "    Dropout,\n",
        "    SpatialDropout1D\n",
        ")\n",
        "\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from tensorflow.keras.callbacks import (\n",
        "    EarlyStopping,\n",
        "    ModelCheckpoint,\n",
        "    ReduceLROnPlateau\n",
        ")\n",
        "\n",
        "from sklearn.metrics import (\n",
        "    accuracy_score,\n",
        "    precision_score,\n",
        "    recall_score,\n",
        "    f1_score,\n",
        "    roc_auc_score,\n",
        "    confusion_matrix,\n",
        "    classification_report,\n",
        "    ConfusionMatrixDisplay\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x_train = joblib.load(\"/content/x_train.pkl\")\n",
        "x_test = joblib.load(\"/content/x_test.pkl\")\n",
        "\n",
        "y_train = joblib.load(\"/content/y_train.pkl\")\n",
        "y_test = joblib.load(\"/content/y_test.pkl\")"
      ],
      "metadata": {
        "id": "pIXRs43wBh84"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "l2WxznjtBkgr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Tokenizer"
      ],
      "metadata": {
        "id": "ogRzzmKpC0a7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "VOCAB_SIZE = 50000\n",
        "\n",
        "tokenizer = Tokenizer(\n",
        "    num_words=VOCAB_SIZE,\n",
        "    oov_token=\"<OOV>\"\n",
        ")\n",
        "tokenizer.fit_on_texts(x_train)"
      ],
      "metadata": {
        "id": "tm_isCNBBkdz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x_tr_seq = tokenizer.texts_to_sequences(x_train)\n",
        "x_te_seq = tokenizer.texts_to_sequences(x_test)"
      ],
      "metadata": {
        "id": "Y5RctTB6Bkbg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "seq_len = [len(seq) for seq in x_tr_seq]\n",
        "\n",
        "print(\"Maximum Length :\", max(seq_len))\n",
        "print(\"Minimum Length :\", min(seq_len))\n",
        "print(\"Average Length :\", np.mean(seq_len))\n",
        "print(\"Median Length  :\", np.median(seq_len))\n",
        "print(\"95th Percentile:\", np.percentile(seq_len, 95))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OZYMPSTRBkWm",
        "outputId": "585f89c8-06b0-42cd-9285-583ee8caaf5f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Maximum Length : 56\n",
            "Minimum Length : 1\n",
            "Average Length : 7.173818624180827\n",
            "Median Length  : 7.0\n",
            "95th Percentile: 14.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "FlLYJ9UvBkUY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Padding"
      ],
      "metadata": {
        "id": "IjnszFIkDIn_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "MAX_LENGTH = 20\n",
        "\n",
        "x_tr_pad = pad_sequences(\n",
        "    x_tr_seq,\n",
        "    maxlen=MAX_LENGTH,\n",
        "    padding=\"post\",\n",
        "    truncating=\"post\"\n",
        ")\n",
        "\n",
        "x_te_pad = pad_sequences(\n",
        "    x_te_seq,\n",
        "    maxlen=MAX_LENGTH,\n",
        "    padding=\"post\",\n",
        "    truncating=\"post\"\n",
        ")"
      ],
      "metadata": {
        "id": "rOy2bfMNBkSA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(x_tr_pad.shape)\n",
        "print(x_te_pad.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RB9IQR7IBkPj",
        "outputId": "facfe3df-b49c-4d11-972e-19454893a613"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(1274150, 20)\n",
            "(318538, 20)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "KtndHIvRBkNT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Model"
      ],
      "metadata": {
        "id": "RuKmFeapDcyW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "VOCAB_SIZE = 50000\n",
        "EMBEDDING_DIM = 128\n",
        "\n",
        "model = Sequential([\n",
        "\n",
        "    Input(shape=(MAX_LENGTH,)),\n",
        "\n",
        "    Embedding(\n",
        "        input_dim=VOCAB_SIZE,\n",
        "        output_dim=EMBEDDING_DIM\n",
        "    ),\n",
        "\n",
        "    SpatialDropout1D(0.2),\n",
        "\n",
        "    # Bidirectional(\n",
        "    #     LSTM(\n",
        "    #         256,\n",
        "    #         return_sequences=True,\n",
        "    #         dropout=0.2,\n",
        "    #         recurrent_dropout=0.2\n",
        "    #     )\n",
        "    # ),\n",
        "\n",
        "    Bidirectional(\n",
        "        LSTM(\n",
        "            128,\n",
        "            return_sequences=True,\n",
        "            dropout=0.2,\n",
        "            recurrent_dropout=0.2\n",
        "        )\n",
        "    ),\n",
        "\n",
        "    Bidirectional(\n",
        "        LSTM(\n",
        "            64,\n",
        "            dropout=0.2,\n",
        "            recurrent_dropout=0.2\n",
        "        )\n",
        "    ),\n",
        "\n",
        "    Dense(128, activation=\"relu\"),\n",
        "\n",
        "    Dropout(0.4),\n",
        "\n",
        "    Dense(64, activation=\"relu\"),\n",
        "\n",
        "    Dropout(0.3),\n",
        "\n",
        "    Dense(1, activation=\"sigmoid\")\n",
        "])"
      ],
      "metadata": {
        "id": "Wpml-BiPBkLC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.layers"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k-D_8ZhHBkIi",
        "outputId": "4f47def4-0010-4176-a3a6-58a80455683c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<Embedding name=embedding, built=True>,\n",
              " <SpatialDropout1D name=spatial_dropout1d, built=True>,\n",
              " <Bidirectional name=bidirectional, built=True>,\n",
              " <Bidirectional name=bidirectional_1, built=True>,\n",
              " <Dense name=dense, built=True>,\n",
              " <Dropout name=dropout, built=True>,\n",
              " <Dense name=dense_1, built=True>,\n",
              " <Dropout name=dropout_1, built=True>,\n",
              " <Dense name=dense_2, built=True>]"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 429
        },
        "id": "_bERNuVXBkGQ",
        "outputId": "5ef21e5d-6384-42d9-ed27-a2e96adc91e6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1mModel: \"sequential\"\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"sequential\"</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
              "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                   \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape          \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
              "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
              "│ embedding (\u001b[38;5;33mEmbedding\u001b[0m)           │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m20\u001b[0m, \u001b[38;5;34m128\u001b[0m)        │     \u001b[38;5;34m6,400,000\u001b[0m │\n",
              "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
              "│ spatial_dropout1d               │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m20\u001b[0m, \u001b[38;5;34m128\u001b[0m)        │             \u001b[38;5;34m0\u001b[0m │\n",
              "│ (\u001b[38;5;33mSpatialDropout1D\u001b[0m)              │                        │               │\n",
              "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
              "│ bidirectional (\u001b[38;5;33mBidirectional\u001b[0m)   │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m20\u001b[0m, \u001b[38;5;34m256\u001b[0m)        │       \u001b[38;5;34m263,168\u001b[0m │\n",
              "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
              "│ bidirectional_1 (\u001b[38;5;33mBidirectional\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m)            │       \u001b[38;5;34m164,352\u001b[0m │\n",
              "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
              "│ dense (\u001b[38;5;33mDense\u001b[0m)                   │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m)            │        \u001b[38;5;34m16,512\u001b[0m │\n",
              "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
              "│ dropout (\u001b[38;5;33mDropout\u001b[0m)               │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m)            │             \u001b[38;5;34m0\u001b[0m │\n",
              "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
              "│ dense_1 (\u001b[38;5;33mDense\u001b[0m)                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m64\u001b[0m)             │         \u001b[38;5;34m8,256\u001b[0m │\n",
              "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
              "│ dropout_1 (\u001b[38;5;33mDropout\u001b[0m)             │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m64\u001b[0m)             │             \u001b[38;5;34m0\u001b[0m │\n",
              "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
              "│ dense_2 (\u001b[38;5;33mDense\u001b[0m)                 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)              │            \u001b[38;5;34m65\u001b[0m │\n",
              "└─────────────────────────────────┴────────────────────────┴───────────────┘\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
              "┃<span style=\"font-weight: bold\"> Layer (type)                    </span>┃<span style=\"font-weight: bold\"> Output Shape           </span>┃<span style=\"font-weight: bold\">       Param # </span>┃\n",
              "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
              "│ embedding (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>)           │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">20</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)        │     <span style=\"color: #00af00; text-decoration-color: #00af00\">6,400,000</span> │\n",
              "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
              "│ spatial_dropout1d               │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">20</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)        │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
              "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">SpatialDropout1D</span>)              │                        │               │\n",
              "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
              "│ bidirectional (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>)   │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">20</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)        │       <span style=\"color: #00af00; text-decoration-color: #00af00\">263,168</span> │\n",
              "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
              "│ bidirectional_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)            │       <span style=\"color: #00af00; text-decoration-color: #00af00\">164,352</span> │\n",
              "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
              "│ dense (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                   │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)            │        <span style=\"color: #00af00; text-decoration-color: #00af00\">16,512</span> │\n",
              "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
              "│ dropout (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>)               │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)            │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
              "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
              "│ dense_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)             │         <span style=\"color: #00af00; text-decoration-color: #00af00\">8,256</span> │\n",
              "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
              "│ dropout_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>)             │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)             │             <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
              "├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
              "│ dense_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)              │            <span style=\"color: #00af00; text-decoration-color: #00af00\">65</span> │\n",
              "└─────────────────────────────────┴────────────────────────┴───────────────┘\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m6,852,353\u001b[0m (26.14 MB)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">6,852,353</span> (26.14 MB)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m6,852,353\u001b[0m (26.14 MB)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">6,852,353</span> (26.14 MB)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Compile"
      ],
      "metadata": {
        "id": "vP_QyXh9GqZm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model.compile(\n",
        "    optimizer=\"adam\",\n",
        "    loss=\"binary_crossentropy\",\n",
        "    metrics=[\n",
        "        \"accuracy\",\n",
        "        tf.keras.metrics.Precision(name=\"precision\"),\n",
        "        tf.keras.metrics.Recall(name=\"recall\")\n",
        "    ]\n",
        ")"
      ],
      "metadata": {
        "id": "VSf0ZfXiGlqQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Call Backs"
      ],
      "metadata": {
        "id": "ITZVgIfDHFyt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "early_stop = EarlyStopping(\n",
        "    monitor=\"val_loss\",\n",
        "    patience=3,\n",
        "    restore_best_weights=True,\n",
        "    verbose=1\n",
        ")\n",
        "\n",
        "checkpoint = ModelCheckpoint(\n",
        "    \"best_model.keras\",\n",
        "    monitor=\"val_accuracy\",\n",
        "    save_best_only=True,\n",
        "    verbose=1\n",
        ")\n",
        "\n",
        "reduce_lr = ReduceLROnPlateau(\n",
        "    monitor=\"val_loss\",\n",
        "    factor=0.5,\n",
        "    patience=2,\n",
        "    min_lr=1e-6,\n",
        "    verbose=1\n",
        ")"
      ],
      "metadata": {
        "id": "-GW_N9ESBkDw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "start = time.time()\n",
        "\n",
        "history = model.fit(\n",
        "    x_tr_pad,\n",
        "    y_train,\n",
        "    validation_split=0.2,\n",
        "    epochs=10,\n",
        "    batch_size=512,\n",
        "    callbacks=[\n",
        "        early_stop,\n",
        "        checkpoint,\n",
        "        reduce_lr\n",
        "    ],\n",
        "    verbose=1\n",
        ")\n",
        "end = time.time()\n",
        "print(f\"Training Time: {(end-start)/60:.2f} Minutes\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cn-TBueIBkBZ",
        "outputId": "473b8514-a2ab-4c86-9a63-4af9e8b10852"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "\u001b[1m1991/1991\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 262ms/step - accuracy: 0.7457 - loss: 0.5080 - precision: 0.7454 - recall: 0.7463\n",
            "Epoch 1: val_accuracy improved from None to 0.79230, saving model to best_model.keras\n",
            "\n",
            "Epoch 1: finished saving model to best_model.keras\n",
            "\u001b[1m1991/1991\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m559s\u001b[0m 273ms/step - accuracy: 0.7743 - loss: 0.4743 - precision: 0.7738 - recall: 0.7751 - val_accuracy: 0.7923 - val_loss: 0.4418 - val_precision: 0.7898 - val_recall: 0.7969 - learning_rate: 0.0010\n",
            "Epoch 2/10\n",
            "\u001b[1m1991/1991\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 269ms/step - accuracy: 0.8029 - loss: 0.4278 - precision: 0.8023 - recall: 0.8029\n",
            "Epoch 2: val_accuracy improved from 0.79230 to 0.79612, saving model to best_model.keras\n",
            "\n",
            "Epoch 2: finished saving model to best_model.keras\n",
            "\u001b[1m1991/1991\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m558s\u001b[0m 280ms/step - accuracy: 0.8016 - loss: 0.4297 - precision: 0.8008 - recall: 0.8029 - val_accuracy: 0.7961 - val_loss: 0.4393 - val_precision: 0.7915 - val_recall: 0.8043 - learning_rate: 0.0010\n",
            "Epoch 3/10\n",
            "\u001b[1m1991/1991\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 269ms/step - accuracy: 0.8149 - loss: 0.4060 - precision: 0.8145 - recall: 0.8154\n",
            "Epoch 3: val_accuracy did not improve from 0.79612\n",
            "\u001b[1m1991/1991\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m556s\u001b[0m 279ms/step - accuracy: 0.8127 - loss: 0.4096 - precision: 0.8114 - recall: 0.8147 - val_accuracy: 0.7947 - val_loss: 0.4425 - val_precision: 0.7975 - val_recall: 0.7903 - learning_rate: 0.0010\n",
            "Epoch 4/10\n",
            "\u001b[1m1991/1991\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 268ms/step - accuracy: 0.8252 - loss: 0.3862 - precision: 0.8237 - recall: 0.8270\n",
            "Epoch 4: val_accuracy did not improve from 0.79612\n",
            "\n",
            "Epoch 4: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
            "\u001b[1m1991/1991\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m555s\u001b[0m 279ms/step - accuracy: 0.8217 - loss: 0.3914 - precision: 0.8203 - recall: 0.8239 - val_accuracy: 0.7938 - val_loss: 0.4519 - val_precision: 0.7874 - val_recall: 0.8054 - learning_rate: 0.0010\n",
            "Epoch 5/10\n",
            "\u001b[1m1991/1991\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 264ms/step - accuracy: 0.8375 - loss: 0.3598 - precision: 0.8340 - recall: 0.8429\n",
            "Epoch 5: val_accuracy did not improve from 0.79612\n",
            "\u001b[1m1991/1991\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m546s\u001b[0m 274ms/step - accuracy: 0.8364 - loss: 0.3621 - precision: 0.8330 - recall: 0.8415 - val_accuracy: 0.7903 - val_loss: 0.4834 - val_precision: 0.7876 - val_recall: 0.7953 - learning_rate: 5.0000e-04\n",
            "Epoch 5: early stopping\n",
            "Restoring model weights from the end of the best epoch: 2.\n",
            "Training Time: 46.24 Minutes\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "6O8Hl-naBj-x"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Testing"
      ],
      "metadata": {
        "id": "_kKIm0clfAZM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(type(x_test))\n",
        "print(type(y_test))\n",
        "\n",
        "print(x_test.dtype)\n",
        "print(y_test.dtype)\n",
        "\n",
        "print(x_test.shape)\n",
        "print(y_test.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pq4bF7wDf6dX",
        "outputId": "0f2c1ebf-8051-4449-8baf-c4ce5e4b56d2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.series.Series'>\n",
            "<class 'pandas.core.series.Series'>\n",
            "object\n",
            "int64\n",
            "(318538,)\n",
            "(318538,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "best_model = load_model(\"best_model.keras\")"
      ],
      "metadata": {
        "id": "SCFEFxARfHsT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_loss, test_acc, test_precision, test_recall = best_model.evaluate(\n",
        "    x_te_pad,\n",
        "    y_test,\n",
        "    verbose=1\n",
        ")\n",
        "\n",
        "print(f\"Test Loss      : {test_loss:.4f}\")\n",
        "print(f\"Test Accuracy  : {test_acc*100:.2f}%\")\n",
        "print(f\"Test Precision : {test_precision:.4f}\")\n",
        "print(f\"Test Recall    : {test_recall:.4f}\")"
      ],
      "metadata": {
        "id": "V71yXa-LBj8h",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0d74ecfd-e911-4640-fd20-5690c0808fb7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1m9955/9955\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m448s\u001b[0m 45ms/step - accuracy: 0.7952 - loss: 0.4403 - precision: 0.7899 - recall: 0.8044\n",
            "Test Loss      : 0.4403\n",
            "Test Accuracy  : 79.52%\n",
            "Test Precision : 0.7899\n",
            "Test Recall    : 0.8044\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "I361MlblBj6N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "8rl-w5pzBj3t"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ARsryAAeBj1N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "AozltyAXBjyv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "IfCPEOHKBjwt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "QKzz9FZIBjpB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "fS-kTIwIBjkD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "FVOSL4D8BjfD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ZhvWK-roBjdP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "n0uol65ABja8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "4ghmilJ4BjYy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ay3mBuUBBjW7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "qU5U8OZDBjUz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "1BfFr-AABjS_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "0zmPqy3SBjQr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Rx8b8k4WBjOt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "E6Nav-ICBjMh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "9L24Om0KBjKd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "50T6tv4ABjID"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "zBNkJmqQBh6v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "MpjGr40eBh2M"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "AXoO32F1Bhzw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "XdV0DbASBhaB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "vhY7nM5pBhU6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "039SsO_fBhRc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Kw645uYbBhOy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "BVwYUv5yBhMX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "mwu80S3uBhJd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "cTm9C2KoBhG6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Kg4nAIe9BhE1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "SHHKeeb_Bg_M"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "sizuaIQXBgyy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "SeegWePpBgsq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "-f3y2AW5Bged"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "EZL72rVRBgcX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ASt5fXo1Bgag"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "M2seqoVhBgYB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "EQ3m9BnjBgVW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "eV0hZzVyBgQY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "lmQ7dMk5BgOg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "BbuTJBQhBgMb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "T78sI_qmBgKL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "j_cUswYFBgIX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "NZ4_pGSBBgGU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "CHW_1VO7BgD_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "tXzWbdpzBgB2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "n7lQ602yBf_2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "T439C-n1Bf9y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "VKy8EAAVBf7k"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "N6su_PWbBf5q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "oktQw2VxBf3F"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "GBZqBxOABf0s"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}