Neural_Network_Hyperparameter_Tuning.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/manasik29/Neural-Networks/blob/main/Neural_Network_Hyperparameter_Tuning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "gl0MVt2lqU6e"
      },
      "outputs": [],
      "source": [
        "# Importing the necessary packages\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import keras\n",
        "from sklearn.preprocessing import StandardScaler"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "6JELDa1oqU6j"
      },
      "outputs": [],
      "source": [
        "# load pima indians dataset\n",
        "dataset = np.loadtxt(\"/content/pima-indians-diabetes.data.csv\", delimiter=\",\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "4B7q_P31qU6m"
      },
      "outputs": [],
      "source": [
        "X = dataset[:,0:8]\n",
        "y = dataset[:,8]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "LW45iQzBqU6m"
      },
      "outputs": [],
      "source": [
        "# Standardization\n",
        "a = StandardScaler()\n",
        "a.fit(X)\n",
        "X_standardized = a.transform(X)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "sL-tinNxqU6o",
        "outputId": "7fb555b5-b92f-4d87-d6d5-d1a982b4851d"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                  0             1             2             3             4  \\\n",
              "count  7.680000e+02  7.680000e+02  7.680000e+02  7.680000e+02  7.680000e+02   \n",
              "mean  -8.789266e-17 -9.251859e-18  1.503427e-17  8.211024e-17 -4.972874e-17   \n",
              "std    1.000652e+00  1.000652e+00  1.000652e+00  1.000652e+00  1.000652e+00   \n",
              "min   -1.141852e+00 -3.783654e+00 -3.572597e+00 -1.288212e+00 -6.928906e-01   \n",
              "25%   -8.448851e-01 -6.852363e-01 -3.673367e-01 -1.288212e+00 -6.928906e-01   \n",
              "50%   -2.509521e-01 -1.218877e-01  1.496408e-01  1.545332e-01 -4.280622e-01   \n",
              "75%    6.399473e-01  6.057709e-01  5.632228e-01  7.190857e-01  4.120079e-01   \n",
              "max    3.906578e+00  2.444478e+00  2.734528e+00  4.921866e+00  6.652839e+00   \n",
              "\n",
              "                  5             6             7  \n",
              "count  7.680000e+02  7.680000e+02  7.680000e+02  \n",
              "mean   2.960595e-15  1.905883e-15  2.081668e-16  \n",
              "std    1.000652e+00  1.000652e+00  1.000652e+00  \n",
              "min   -4.060474e+00 -1.189553e+00 -1.041549e+00  \n",
              "25%   -5.955785e-01 -6.889685e-01 -7.862862e-01  \n",
              "50%    9.419788e-04 -3.001282e-01 -3.608474e-01  \n",
              "75%    5.847705e-01  4.662269e-01  6.602056e-01  \n",
              "max    4.455807e+00  5.883565e+00  4.063716e+00  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8d5c95af-996b-48c4-b6be-ea7b7a25a772\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>7.680000e+02</td>\n",
              "      <td>7.680000e+02</td>\n",
              "      <td>7.680000e+02</td>\n",
              "      <td>7.680000e+02</td>\n",
              "      <td>7.680000e+02</td>\n",
              "      <td>7.680000e+02</td>\n",
              "      <td>7.680000e+02</td>\n",
              "      <td>7.680000e+02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>-8.789266e-17</td>\n",
              "      <td>-9.251859e-18</td>\n",
              "      <td>1.503427e-17</td>\n",
              "      <td>8.211024e-17</td>\n",
              "      <td>-4.972874e-17</td>\n",
              "      <td>2.960595e-15</td>\n",
              "      <td>1.905883e-15</td>\n",
              "      <td>2.081668e-16</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>1.000652e+00</td>\n",
              "      <td>1.000652e+00</td>\n",
              "      <td>1.000652e+00</td>\n",
              "      <td>1.000652e+00</td>\n",
              "      <td>1.000652e+00</td>\n",
              "      <td>1.000652e+00</td>\n",
              "      <td>1.000652e+00</td>\n",
              "      <td>1.000652e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>-1.141852e+00</td>\n",
              "      <td>-3.783654e+00</td>\n",
              "      <td>-3.572597e+00</td>\n",
              "      <td>-1.288212e+00</td>\n",
              "      <td>-6.928906e-01</td>\n",
              "      <td>-4.060474e+00</td>\n",
              "      <td>-1.189553e+00</td>\n",
              "      <td>-1.041549e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>-8.448851e-01</td>\n",
              "      <td>-6.852363e-01</td>\n",
              "      <td>-3.673367e-01</td>\n",
              "      <td>-1.288212e+00</td>\n",
              "      <td>-6.928906e-01</td>\n",
              "      <td>-5.955785e-01</td>\n",
              "      <td>-6.889685e-01</td>\n",
              "      <td>-7.862862e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>-2.509521e-01</td>\n",
              "      <td>-1.218877e-01</td>\n",
              "      <td>1.496408e-01</td>\n",
              "      <td>1.545332e-01</td>\n",
              "      <td>-4.280622e-01</td>\n",
              "      <td>9.419788e-04</td>\n",
              "      <td>-3.001282e-01</td>\n",
              "      <td>-3.608474e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>6.399473e-01</td>\n",
              "      <td>6.057709e-01</td>\n",
              "      <td>5.632228e-01</td>\n",
              "      <td>7.190857e-01</td>\n",
              "      <td>4.120079e-01</td>\n",
              "      <td>5.847705e-01</td>\n",
              "      <td>4.662269e-01</td>\n",
              "      <td>6.602056e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>3.906578e+00</td>\n",
              "      <td>2.444478e+00</td>\n",
              "      <td>2.734528e+00</td>\n",
              "      <td>4.921866e+00</td>\n",
              "      <td>6.652839e+00</td>\n",
              "      <td>4.455807e+00</td>\n",
              "      <td>5.883565e+00</td>\n",
              "      <td>4.063716e+00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8d5c95af-996b-48c4-b6be-ea7b7a25a772')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8d5c95af-996b-48c4-b6be-ea7b7a25a772 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8d5c95af-996b-48c4-b6be-ea7b7a25a772');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "pd.DataFrame(X_standardized).describe()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OhSC8_hTqU6p"
      },
      "source": [
        "#### Tuning of Hyperparameters :- Batch Size and Epochs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "kp_4DRy0qU6u"
      },
      "outputs": [],
      "source": [
        "# Importing the necessary packages\n",
        "from sklearn.model_selection import GridSearchCV, KFold\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense\n",
        "from keras.wrappers.scikit_learn import KerasClassifier\n",
        "from tensorflow.keras.optimizers import Adam"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "FcxYE6HvqU6v"
      },
      "outputs": [],
      "source": [
        "# create model\n",
        "def create_model():\n",
        "    model = Sequential()\n",
        "    model.add(Dense(12, input_dim=8, kernel_initializer='uniform', activation='relu'))\n",
        "    model.add(Dense(8, kernel_initializer='uniform', activation='relu'))\n",
        "    model.add(Dense(1, kernel_initializer='uniform', activation='sigmoid'))\n",
        "    \n",
        "    adam=Adam(lr=0.01)\n",
        "    model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])\n",
        "    return model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "M-AOSsbyqU6w",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1d258431-6606-409c-8bd9-3f71968750b7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:2: DeprecationWarning: KerasClassifier is deprecated, use Sci-Keras (https://github.com/adriangb/scikeras) instead. See https://www.adriangb.com/scikeras/stable/migration.html for help migrating.\n",
            "  \n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 5 folds for each of 9 candidates, totalling 45 fits\n",
            "[CV 1/5; 1/9] START batch_size=10, epochs=10....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 1/9] END .....batch_size=10, epochs=10;, score=0.727 total time=   2.5s\n",
            "[CV 2/5; 1/9] START batch_size=10, epochs=10....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 1/9] END .....batch_size=10, epochs=10;, score=0.675 total time=   1.5s\n",
            "[CV 3/5; 1/9] START batch_size=10, epochs=10....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 1/9] END .....batch_size=10, epochs=10;, score=0.747 total time=   2.0s\n",
            "[CV 4/5; 1/9] START batch_size=10, epochs=10....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 1/9] END .....batch_size=10, epochs=10;, score=0.830 total time=   1.5s\n",
            "[CV 5/5; 1/9] START batch_size=10, epochs=10....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 1/9] END .....batch_size=10, epochs=10;, score=0.765 total time=   1.7s\n",
            "[CV 1/5; 2/9] START batch_size=10, epochs=50....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 2/9] END .....batch_size=10, epochs=50;, score=0.766 total time=   5.8s\n",
            "[CV 2/5; 2/9] START batch_size=10, epochs=50....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 2/9] END .....batch_size=10, epochs=50;, score=0.714 total time=   4.4s\n",
            "[CV 3/5; 2/9] START batch_size=10, epochs=50....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 2/9] END .....batch_size=10, epochs=50;, score=0.760 total time=   5.8s\n",
            "[CV 4/5; 2/9] START batch_size=10, epochs=50....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 2/9] END .....batch_size=10, epochs=50;, score=0.804 total time=   5.8s\n",
            "[CV 5/5; 2/9] START batch_size=10, epochs=50....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 2/9] END .....batch_size=10, epochs=50;, score=0.771 total time=   4.4s\n",
            "[CV 1/5; 3/9] START batch_size=10, epochs=100...................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 3/9] END ....batch_size=10, epochs=100;, score=0.747 total time=  10.9s\n",
            "[CV 2/5; 3/9] START batch_size=10, epochs=100...................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 3/9] END ....batch_size=10, epochs=100;, score=0.675 total time=   8.2s\n",
            "[CV 3/5; 3/9] START batch_size=10, epochs=100...................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 3/9] END ....batch_size=10, epochs=100;, score=0.740 total time=  11.0s\n",
            "[CV 4/5; 3/9] START batch_size=10, epochs=100...................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 3/9] END ....batch_size=10, epochs=100;, score=0.824 total time=   8.0s\n",
            "[CV 5/5; 3/9] START batch_size=10, epochs=100...................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 3/9] END ....batch_size=10, epochs=100;, score=0.719 total time=  21.2s\n",
            "[CV 1/5; 4/9] START batch_size=20, epochs=10....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 4/9] END .....batch_size=20, epochs=10;, score=0.753 total time=   1.4s\n",
            "[CV 2/5; 4/9] START batch_size=20, epochs=10....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 4/9] END .....batch_size=20, epochs=10;, score=0.675 total time=   1.1s\n",
            "[CV 3/5; 4/9] START batch_size=20, epochs=10....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 4/9] END .....batch_size=20, epochs=10;, score=0.773 total time=   1.1s\n",
            "[CV 4/5; 4/9] START batch_size=20, epochs=10....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 4/9] END .....batch_size=20, epochs=10;, score=0.837 total time=   1.3s\n",
            "[CV 5/5; 4/9] START batch_size=20, epochs=10....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 4/9] END .....batch_size=20, epochs=10;, score=0.771 total time=   1.1s\n",
            "[CV 1/5; 5/9] START batch_size=20, epochs=50....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 5/9] END .....batch_size=20, epochs=50;, score=0.740 total time=   3.3s\n",
            "[CV 2/5; 5/9] START batch_size=20, epochs=50....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 5/9] END .....batch_size=20, epochs=50;, score=0.682 total time=   2.6s\n",
            "[CV 3/5; 5/9] START batch_size=20, epochs=50....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 5/9] END .....batch_size=20, epochs=50;, score=0.786 total time=   3.3s\n",
            "[CV 4/5; 5/9] START batch_size=20, epochs=50....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 5/9] END .....batch_size=20, epochs=50;, score=0.817 total time=   2.7s\n",
            "[CV 5/5; 5/9] START batch_size=20, epochs=50....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 5/9] END .....batch_size=20, epochs=50;, score=0.765 total time=   3.3s\n",
            "[CV 1/5; 6/9] START batch_size=20, epochs=100...................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 6/9] END ....batch_size=20, epochs=100;, score=0.753 total time=   4.5s\n",
            "[CV 2/5; 6/9] START batch_size=20, epochs=100...................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 6/9] END ....batch_size=20, epochs=100;, score=0.714 total time=   5.8s\n",
            "[CV 3/5; 6/9] START batch_size=20, epochs=100...................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 6/9] END ....batch_size=20, epochs=100;, score=0.747 total time=   4.9s\n",
            "[CV 4/5; 6/9] START batch_size=20, epochs=100...................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 6/9] END ....batch_size=20, epochs=100;, score=0.824 total time=   4.6s\n",
            "[CV 5/5; 6/9] START batch_size=20, epochs=100...................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 6/9] END ....batch_size=20, epochs=100;, score=0.752 total time=   5.9s\n",
            "[CV 1/5; 7/9] START batch_size=40, epochs=10....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 7/9] END .....batch_size=40, epochs=10;, score=0.766 total time=   1.1s\n",
            "[CV 2/5; 7/9] START batch_size=40, epochs=10....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 7/9] END .....batch_size=40, epochs=10;, score=0.727 total time=   1.2s\n",
            "[CV 3/5; 7/9] START batch_size=40, epochs=10....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 7/9] END .....batch_size=40, epochs=10;, score=0.766 total time=   1.5s\n",
            "[CV 4/5; 7/9] START batch_size=40, epochs=10....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 7/9] END .....batch_size=40, epochs=10;, score=0.850 total time=   1.0s\n",
            "[CV 5/5; 7/9] START batch_size=40, epochs=10....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 7/9] END .....batch_size=40, epochs=10;, score=0.765 total time=   1.0s\n",
            "[CV 1/5; 8/9] START batch_size=40, epochs=50....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 8/9] END .....batch_size=40, epochs=50;, score=0.773 total time=   1.8s\n",
            "[CV 2/5; 8/9] START batch_size=40, epochs=50....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 8/9] END .....batch_size=40, epochs=50;, score=0.721 total time=   1.7s\n",
            "[CV 3/5; 8/9] START batch_size=40, epochs=50....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 8/9] END .....batch_size=40, epochs=50;, score=0.786 total time=   1.8s\n",
            "[CV 4/5; 8/9] START batch_size=40, epochs=50....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 8/9] END .....batch_size=40, epochs=50;, score=0.824 total time=   2.4s\n",
            "[CV 5/5; 8/9] START batch_size=40, epochs=50....................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 8/9] END .....batch_size=40, epochs=50;, score=0.778 total time=   1.8s\n",
            "[CV 1/5; 9/9] START batch_size=40, epochs=100...................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 9/9] END ....batch_size=40, epochs=100;, score=0.753 total time=   2.7s\n",
            "[CV 2/5; 9/9] START batch_size=40, epochs=100...................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 9/9] END ....batch_size=40, epochs=100;, score=0.688 total time=   3.3s\n",
            "[CV 3/5; 9/9] START batch_size=40, epochs=100...................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 9/9] END ....batch_size=40, epochs=100;, score=0.747 total time=   2.7s\n",
            "[CV 4/5; 9/9] START batch_size=40, epochs=100...................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 9/9] END ....batch_size=40, epochs=100;, score=0.837 total time=   2.8s\n",
            "[CV 5/5; 9/9] START batch_size=40, epochs=100...................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 9/9] END ....batch_size=40, epochs=100;, score=0.758 total time=   2.7s\n"
          ]
        }
      ],
      "source": [
        "# Create the model\n",
        "model = KerasClassifier(build_fn = create_model,verbose = 0)\n",
        "# Define the grid search parameters\n",
        "batch_size = [10,20,40]\n",
        "epochs = [10,50,100]\n",
        "# Make a dictionary of the grid search parameters\n",
        "param_grid = dict(batch_size = batch_size,epochs = epochs)\n",
        "# Build and fit the GridSearchCV\n",
        "grid = GridSearchCV(estimator = model,param_grid = param_grid,cv = KFold(),verbose = 10)\n",
        "grid_result = grid.fit(X_standardized,y)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "D92_CScOqU6x",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7673a909-be36-43a7-d6b9-b5cbd8feea56"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best : 0.7761055946350097, using {'batch_size': 40, 'epochs': 50}\n",
            "0.748824393749237,0.050444602014764015 with: {'batch_size': 10, 'epochs': 10}\n",
            "0.7630846261978149,0.02876327886566644 with: {'batch_size': 10, 'epochs': 50}\n",
            "0.7409642815589905,0.04825766172878011 with: {'batch_size': 10, 'epochs': 100}\n",
            "0.7618283629417419,0.05168157501896805 with: {'batch_size': 20, 'epochs': 10}\n",
            "0.7578983187675477,0.045632472221432706 with: {'batch_size': 20, 'epochs': 50}\n",
            "0.7578898310661316,0.035761963482688446 with: {'batch_size': 20, 'epochs': 100}\n",
            "0.7748238682746887,0.0402827184477866 with: {'batch_size': 40, 'epochs': 10}\n",
            "0.7761055946350097,0.03291370837378032 with: {'batch_size': 40, 'epochs': 50}\n",
            "0.7566165924072266,0.04727402874982167 with: {'batch_size': 40, 'epochs': 100}\n"
          ]
        }
      ],
      "source": [
        "# Summarize the results\n",
        "print('Best : {}, using {}'.format(grid_result.best_score_,grid_result.best_params_))\n",
        "means = grid_result.cv_results_['mean_test_score']\n",
        "stds = grid_result.cv_results_['std_test_score']\n",
        "params = grid_result.cv_results_['params']\n",
        "for mean, stdev, param in zip(means, stds, params):\n",
        "  print('{},{} with: {}'.format(mean, stdev, param))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7BLxpKh9qU6y"
      },
      "source": [
        "#### Tuning of Hyperparameters:- Learning rate and Drop out rate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "S3eawWxjqU6z",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "11b6105f-f189-4475-f41e-f433ea9abb79"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:19: DeprecationWarning: KerasClassifier is deprecated, use Sci-Keras (https://github.com/adriangb/scikeras) instead. See https://www.adriangb.com/scikeras/stable/migration.html for help migrating.\n",
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 5 folds for each of 9 candidates, totalling 45 fits\n",
            "[CV 1/5; 1/9] START dropout_rate=0.0, learning_rate=0.001.......................\n",
            "[CV 1/5; 1/9] END dropout_rate=0.0, learning_rate=0.001;, score=0.766 total time=   1.8s\n",
            "[CV 2/5; 1/9] START dropout_rate=0.0, learning_rate=0.001.......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 1/9] END dropout_rate=0.0, learning_rate=0.001;, score=0.727 total time=   2.0s\n",
            "[CV 3/5; 1/9] START dropout_rate=0.0, learning_rate=0.001.......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 1/9] END dropout_rate=0.0, learning_rate=0.001;, score=0.773 total time=   1.7s\n",
            "[CV 4/5; 1/9] START dropout_rate=0.0, learning_rate=0.001.......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 1/9] END dropout_rate=0.0, learning_rate=0.001;, score=0.856 total time=   2.3s\n",
            "[CV 5/5; 1/9] START dropout_rate=0.0, learning_rate=0.001.......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 1/9] END dropout_rate=0.0, learning_rate=0.001;, score=0.752 total time=   2.0s\n",
            "[CV 1/5; 2/9] START dropout_rate=0.0, learning_rate=0.01........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 2/9] END dropout_rate=0.0, learning_rate=0.01;, score=0.779 total time=   1.7s\n",
            "[CV 2/5; 2/9] START dropout_rate=0.0, learning_rate=0.01........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 2/9] END dropout_rate=0.0, learning_rate=0.01;, score=0.688 total time=   1.7s\n",
            "[CV 3/5; 2/9] START dropout_rate=0.0, learning_rate=0.01........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 2/9] END dropout_rate=0.0, learning_rate=0.01;, score=0.760 total time=   1.7s\n",
            "[CV 4/5; 2/9] START dropout_rate=0.0, learning_rate=0.01........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 2/9] END dropout_rate=0.0, learning_rate=0.01;, score=0.824 total time=   2.0s\n",
            "[CV 5/5; 2/9] START dropout_rate=0.0, learning_rate=0.01........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 2/9] END dropout_rate=0.0, learning_rate=0.01;, score=0.778 total time=   1.8s\n",
            "[CV 1/5; 3/9] START dropout_rate=0.0, learning_rate=0.1.........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 3/9] END dropout_rate=0.0, learning_rate=0.1;, score=0.708 total time=   2.0s\n",
            "[CV 2/5; 3/9] START dropout_rate=0.0, learning_rate=0.1.........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 3/9] END dropout_rate=0.0, learning_rate=0.1;, score=0.649 total time=   2.0s\n",
            "[CV 3/5; 3/9] START dropout_rate=0.0, learning_rate=0.1.........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 3/9] END dropout_rate=0.0, learning_rate=0.1;, score=0.747 total time=   1.7s\n",
            "[CV 4/5; 3/9] START dropout_rate=0.0, learning_rate=0.1.........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 3/9] END dropout_rate=0.0, learning_rate=0.1;, score=0.791 total time=   2.0s\n",
            "[CV 5/5; 3/9] START dropout_rate=0.0, learning_rate=0.1.........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 3/9] END dropout_rate=0.0, learning_rate=0.1;, score=0.712 total time=   2.4s\n",
            "[CV 1/5; 4/9] START dropout_rate=0.1, learning_rate=0.001.......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 4/9] END dropout_rate=0.1, learning_rate=0.001;, score=0.760 total time=   2.1s\n",
            "[CV 2/5; 4/9] START dropout_rate=0.1, learning_rate=0.001.......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 4/9] END dropout_rate=0.1, learning_rate=0.001;, score=0.734 total time=   1.8s\n",
            "[CV 3/5; 4/9] START dropout_rate=0.1, learning_rate=0.001.......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 4/9] END dropout_rate=0.1, learning_rate=0.001;, score=0.766 total time=   1.8s\n",
            "[CV 4/5; 4/9] START dropout_rate=0.1, learning_rate=0.001.......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 4/9] END dropout_rate=0.1, learning_rate=0.001;, score=0.856 total time=   2.1s\n",
            "[CV 5/5; 4/9] START dropout_rate=0.1, learning_rate=0.001.......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 4/9] END dropout_rate=0.1, learning_rate=0.001;, score=0.778 total time=   2.1s\n",
            "[CV 1/5; 5/9] START dropout_rate=0.1, learning_rate=0.01........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 5/9] END dropout_rate=0.1, learning_rate=0.01;, score=0.753 total time=   1.9s\n",
            "[CV 2/5; 5/9] START dropout_rate=0.1, learning_rate=0.01........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 5/9] END dropout_rate=0.1, learning_rate=0.01;, score=0.688 total time=   1.8s\n",
            "[CV 3/5; 5/9] START dropout_rate=0.1, learning_rate=0.01........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 5/9] END dropout_rate=0.1, learning_rate=0.01;, score=0.773 total time=   1.8s\n",
            "[CV 4/5; 5/9] START dropout_rate=0.1, learning_rate=0.01........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 5/9] END dropout_rate=0.1, learning_rate=0.01;, score=0.817 total time=   1.8s\n",
            "[CV 5/5; 5/9] START dropout_rate=0.1, learning_rate=0.01........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 5/9] END dropout_rate=0.1, learning_rate=0.01;, score=0.771 total time=   2.2s\n",
            "[CV 1/5; 6/9] START dropout_rate=0.1, learning_rate=0.1.........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 6/9] END dropout_rate=0.1, learning_rate=0.1;, score=0.695 total time=   1.8s\n",
            "[CV 2/5; 6/9] START dropout_rate=0.1, learning_rate=0.1.........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 6/9] END dropout_rate=0.1, learning_rate=0.1;, score=0.714 total time=   2.1s\n",
            "[CV 3/5; 6/9] START dropout_rate=0.1, learning_rate=0.1.........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 6/9] END dropout_rate=0.1, learning_rate=0.1;, score=0.766 total time=   2.1s\n",
            "[CV 4/5; 6/9] START dropout_rate=0.1, learning_rate=0.1.........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 6/9] END dropout_rate=0.1, learning_rate=0.1;, score=0.778 total time=   2.0s\n",
            "[CV 5/5; 6/9] START dropout_rate=0.1, learning_rate=0.1.........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 6/9] END dropout_rate=0.1, learning_rate=0.1;, score=0.771 total time=   2.0s\n",
            "[CV 1/5; 7/9] START dropout_rate=0.2, learning_rate=0.001.......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 7/9] END dropout_rate=0.2, learning_rate=0.001;, score=0.766 total time=   1.8s\n",
            "[CV 2/5; 7/9] START dropout_rate=0.2, learning_rate=0.001.......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 7/9] END dropout_rate=0.2, learning_rate=0.001;, score=0.721 total time=   1.8s\n",
            "[CV 3/5; 7/9] START dropout_rate=0.2, learning_rate=0.001.......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 7/9] END dropout_rate=0.2, learning_rate=0.001;, score=0.747 total time=   1.8s\n",
            "[CV 4/5; 7/9] START dropout_rate=0.2, learning_rate=0.001.......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 7/9] END dropout_rate=0.2, learning_rate=0.001;, score=0.830 total time=   2.0s\n",
            "[CV 5/5; 7/9] START dropout_rate=0.2, learning_rate=0.001.......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 7/9] END dropout_rate=0.2, learning_rate=0.001;, score=0.752 total time=   2.1s\n",
            "[CV 1/5; 8/9] START dropout_rate=0.2, learning_rate=0.01........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 8/9] END dropout_rate=0.2, learning_rate=0.01;, score=0.766 total time=   1.8s\n",
            "[CV 2/5; 8/9] START dropout_rate=0.2, learning_rate=0.01........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 8/9] END dropout_rate=0.2, learning_rate=0.01;, score=0.714 total time=   2.1s\n",
            "[CV 3/5; 8/9] START dropout_rate=0.2, learning_rate=0.01........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 8/9] END dropout_rate=0.2, learning_rate=0.01;, score=0.779 total time=   1.8s\n",
            "[CV 4/5; 8/9] START dropout_rate=0.2, learning_rate=0.01........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 8/9] END dropout_rate=0.2, learning_rate=0.01;, score=0.830 total time=   1.8s\n",
            "[CV 5/5; 8/9] START dropout_rate=0.2, learning_rate=0.01........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 8/9] END dropout_rate=0.2, learning_rate=0.01;, score=0.771 total time=   2.0s\n",
            "[CV 1/5; 9/9] START dropout_rate=0.2, learning_rate=0.1.........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 9/9] END dropout_rate=0.2, learning_rate=0.1;, score=0.727 total time=   2.0s\n",
            "[CV 2/5; 9/9] START dropout_rate=0.2, learning_rate=0.1.........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 9/9] END dropout_rate=0.2, learning_rate=0.1;, score=0.610 total time=   1.8s\n",
            "[CV 3/5; 9/9] START dropout_rate=0.2, learning_rate=0.1.........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 9/9] END dropout_rate=0.2, learning_rate=0.1;, score=0.734 total time=   2.1s\n",
            "[CV 4/5; 9/9] START dropout_rate=0.2, learning_rate=0.1.........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 9/9] END dropout_rate=0.2, learning_rate=0.1;, score=0.745 total time=   2.0s\n",
            "[CV 5/5; 9/9] START dropout_rate=0.2, learning_rate=0.1.........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 9/9] END dropout_rate=0.2, learning_rate=0.1;, score=0.732 total time=   2.0s\n"
          ]
        }
      ],
      "source": [
        "from keras.layers import Dropout\n",
        "\n",
        "# Defining the model\n",
        "\n",
        "def create_model(learning_rate,dropout_rate):\n",
        "    model = Sequential()\n",
        "    model.add(Dense(12,input_dim = 8,kernel_initializer = 'normal',activation = 'relu'))\n",
        "    model.add(Dropout(dropout_rate))\n",
        "    model.add(Dense(8,input_dim = 8,kernel_initializer = 'normal',activation = 'relu'))\n",
        "    model.add(Dropout(dropout_rate))\n",
        "    model.add(Dense(1,activation = 'sigmoid'))\n",
        "    \n",
        "    adam = Adam(lr = learning_rate)\n",
        "    model.compile(loss = 'binary_crossentropy',optimizer = adam,metrics = ['accuracy'])\n",
        "    return model\n",
        "\n",
        "# Create the model\n",
        "\n",
        "model = KerasClassifier(build_fn = create_model,verbose = 0,batch_size = 40,epochs = 50)\n",
        "\n",
        "# Define the grid search parameters\n",
        "\n",
        "learning_rate = [0.001,0.01,0.1]\n",
        "dropout_rate = [0.0,0.1,0.2]\n",
        "\n",
        "# Make a dictionary of the grid search parameters\n",
        "\n",
        "param_grids = dict(learning_rate = learning_rate,dropout_rate = dropout_rate)\n",
        "\n",
        "# Build and fit the GridSearchCV\n",
        "\n",
        "grid = GridSearchCV(estimator = model,param_grid = param_grids,cv = KFold(),verbose = 10)\n",
        "grid_result = grid.fit(X_standardized,y)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "OBufCxTgqU62",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "58064433-e500-4280-c008-7ea2016c0b43"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best : 0.778745436668396, using {'dropout_rate': 0.1, 'learning_rate': 0.001}\n",
            "0.7748153805732727,0.0435904349559179 with: {'dropout_rate': 0.0, 'learning_rate': 0.001}\n",
            "0.7657159805297852,0.044039512244529164 with: {'dropout_rate': 0.0, 'learning_rate': 0.01}\n",
            "0.7214328289031983,0.0467396920596142 with: {'dropout_rate': 0.0, 'learning_rate': 0.1}\n",
            "0.778745436668396,0.04133383936360968 with: {'dropout_rate': 0.1, 'learning_rate': 0.001}\n",
            "0.7605041980743408,0.04176162527997297 with: {'dropout_rate': 0.1, 'learning_rate': 0.01}\n",
            "0.7448688626289368,0.0336947697087545 with: {'dropout_rate': 0.1, 'learning_rate': 0.1}\n",
            "0.7630931258201599,0.036562035140510644 with: {'dropout_rate': 0.2, 'learning_rate': 0.001}\n",
            "0.7722094893455506,0.03684635241369448 with: {'dropout_rate': 0.2, 'learning_rate': 0.01}\n",
            "0.7097105622291565,0.050004045552829286 with: {'dropout_rate': 0.2, 'learning_rate': 0.1}\n"
          ]
        }
      ],
      "source": [
        "# Summarize the results\n",
        "print('Best : {}, using {}'.format(grid_result.best_score_,grid_result.best_params_))\n",
        "means = grid_result.cv_results_['mean_test_score']\n",
        "stds = grid_result.cv_results_['std_test_score']\n",
        "params = grid_result.cv_results_['params']\n",
        "for mean, stdev, param in zip(means, stds, params):\n",
        "  print('{},{} with: {}'.format(mean, stdev, param))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N2sQfLMuqU63"
      },
      "source": [
        "#### Tuning of Hyperparameters:- Activation Function and Kernel Initializer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "2o56jukkqU66",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1e8b7aee-2163-4441-8dbc-dbe6dcafe981"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:17: DeprecationWarning: KerasClassifier is deprecated, use Sci-Keras (https://github.com/adriangb/scikeras) instead. See https://www.adriangb.com/scikeras/stable/migration.html for help migrating.\n",
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 5 folds for each of 12 candidates, totalling 60 fits\n",
            "[CV 1/5; 1/12] START activation_function=softmax, init=uniform..................\n",
            "[CV 1/5; 1/12] END activation_function=softmax, init=uniform;, score=0.662 total time=   2.1s\n",
            "[CV 2/5; 1/12] START activation_function=softmax, init=uniform..................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 1/12] END activation_function=softmax, init=uniform;, score=0.584 total time=   2.1s\n",
            "[CV 3/5; 1/12] START activation_function=softmax, init=uniform..................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 1/12] END activation_function=softmax, init=uniform;, score=0.630 total time=   1.9s\n",
            "[CV 4/5; 1/12] START activation_function=softmax, init=uniform..................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 1/12] END activation_function=softmax, init=uniform;, score=0.830 total time=   2.1s\n",
            "[CV 5/5; 1/12] START activation_function=softmax, init=uniform..................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 1/12] END activation_function=softmax, init=uniform;, score=0.732 total time=   2.0s\n",
            "[CV 1/5; 2/12] START activation_function=softmax, init=normal...................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 2/12] END activation_function=softmax, init=normal;, score=0.675 total time=   2.1s\n",
            "[CV 2/5; 2/12] START activation_function=softmax, init=normal...................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 2/12] END activation_function=softmax, init=normal;, score=0.584 total time=   2.1s\n",
            "[CV 3/5; 2/12] START activation_function=softmax, init=normal...................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 2/12] END activation_function=softmax, init=normal;, score=0.630 total time=   2.1s\n",
            "[CV 4/5; 2/12] START activation_function=softmax, init=normal...................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 2/12] END activation_function=softmax, init=normal;, score=0.745 total time=   2.0s\n",
            "[CV 5/5; 2/12] START activation_function=softmax, init=normal...................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 2/12] END activation_function=softmax, init=normal;, score=0.647 total time=   2.2s\n",
            "[CV 1/5; 3/12] START activation_function=softmax, init=zero.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 3/12] END activation_function=softmax, init=zero;, score=0.740 total time=   1.8s\n",
            "[CV 2/5; 3/12] START activation_function=softmax, init=zero.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 3/12] END activation_function=softmax, init=zero;, score=0.584 total time=   2.1s\n",
            "[CV 3/5; 3/12] START activation_function=softmax, init=zero.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 3/12] END activation_function=softmax, init=zero;, score=0.630 total time=   1.9s\n",
            "[CV 4/5; 3/12] START activation_function=softmax, init=zero.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 3/12] END activation_function=softmax, init=zero;, score=0.804 total time=   2.1s\n",
            "[CV 5/5; 3/12] START activation_function=softmax, init=zero.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 3/12] END activation_function=softmax, init=zero;, score=0.647 total time=   1.9s\n",
            "[CV 1/5; 4/12] START activation_function=relu, init=uniform.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 4/12] END activation_function=relu, init=uniform;, score=0.740 total time=   1.8s\n",
            "[CV 2/5; 4/12] START activation_function=relu, init=uniform.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 4/12] END activation_function=relu, init=uniform;, score=0.701 total time=   1.8s\n",
            "[CV 3/5; 4/12] START activation_function=relu, init=uniform.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 4/12] END activation_function=relu, init=uniform;, score=0.740 total time=   2.0s\n",
            "[CV 4/5; 4/12] START activation_function=relu, init=uniform.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 4/12] END activation_function=relu, init=uniform;, score=0.837 total time=   1.8s\n",
            "[CV 5/5; 4/12] START activation_function=relu, init=uniform.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 4/12] END activation_function=relu, init=uniform;, score=0.771 total time=   2.4s\n",
            "[CV 1/5; 5/12] START activation_function=relu, init=normal......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 5/12] END activation_function=relu, init=normal;, score=0.760 total time=   2.1s\n",
            "[CV 2/5; 5/12] START activation_function=relu, init=normal......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 5/12] END activation_function=relu, init=normal;, score=0.734 total time=   2.1s\n",
            "[CV 3/5; 5/12] START activation_function=relu, init=normal......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 5/12] END activation_function=relu, init=normal;, score=0.773 total time=   2.1s\n",
            "[CV 4/5; 5/12] START activation_function=relu, init=normal......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 5/12] END activation_function=relu, init=normal;, score=0.863 total time=   1.8s\n",
            "[CV 5/5; 5/12] START activation_function=relu, init=normal......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 5/12] END activation_function=relu, init=normal;, score=0.765 total time=   2.0s\n",
            "[CV 1/5; 6/12] START activation_function=relu, init=zero........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 6/12] END activation_function=relu, init=zero;, score=0.649 total time=   1.8s\n",
            "[CV 2/5; 6/12] START activation_function=relu, init=zero........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 6/12] END activation_function=relu, init=zero;, score=0.584 total time=   2.0s\n",
            "[CV 3/5; 6/12] START activation_function=relu, init=zero........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 6/12] END activation_function=relu, init=zero;, score=0.630 total time=   1.8s\n",
            "[CV 4/5; 6/12] START activation_function=relu, init=zero........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 6/12] END activation_function=relu, init=zero;, score=0.745 total time=   1.8s\n",
            "[CV 5/5; 6/12] START activation_function=relu, init=zero........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 6/12] END activation_function=relu, init=zero;, score=0.647 total time=   2.4s\n",
            "[CV 1/5; 7/12] START activation_function=tanh, init=uniform.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 7/12] END activation_function=tanh, init=uniform;, score=0.766 total time=   1.8s\n",
            "[CV 2/5; 7/12] START activation_function=tanh, init=uniform.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 7/12] END activation_function=tanh, init=uniform;, score=0.734 total time=   1.8s\n",
            "[CV 3/5; 7/12] START activation_function=tanh, init=uniform.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 7/12] END activation_function=tanh, init=uniform;, score=0.760 total time=   1.9s\n",
            "[CV 4/5; 7/12] START activation_function=tanh, init=uniform.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 7/12] END activation_function=tanh, init=uniform;, score=0.850 total time=   2.0s\n",
            "[CV 5/5; 7/12] START activation_function=tanh, init=uniform.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 7/12] END activation_function=tanh, init=uniform;, score=0.745 total time=   2.0s\n",
            "[CV 1/5; 8/12] START activation_function=tanh, init=normal......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 8/12] END activation_function=tanh, init=normal;, score=0.753 total time=   1.8s\n",
            "[CV 2/5; 8/12] START activation_function=tanh, init=normal......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 8/12] END activation_function=tanh, init=normal;, score=0.734 total time=   2.1s\n",
            "[CV 3/5; 8/12] START activation_function=tanh, init=normal......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 8/12] END activation_function=tanh, init=normal;, score=0.760 total time=   1.8s\n",
            "[CV 4/5; 8/12] START activation_function=tanh, init=normal......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 8/12] END activation_function=tanh, init=normal;, score=0.850 total time=   1.8s\n",
            "[CV 5/5; 8/12] START activation_function=tanh, init=normal......................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 8/12] END activation_function=tanh, init=normal;, score=0.758 total time=   1.9s\n",
            "[CV 1/5; 9/12] START activation_function=tanh, init=zero........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 9/12] END activation_function=tanh, init=zero;, score=0.649 total time=   2.3s\n",
            "[CV 2/5; 9/12] START activation_function=tanh, init=zero........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 9/12] END activation_function=tanh, init=zero;, score=0.584 total time=   1.8s\n",
            "[CV 3/5; 9/12] START activation_function=tanh, init=zero........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 9/12] END activation_function=tanh, init=zero;, score=0.630 total time=   1.8s\n",
            "[CV 4/5; 9/12] START activation_function=tanh, init=zero........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 9/12] END activation_function=tanh, init=zero;, score=0.745 total time=   2.1s\n",
            "[CV 5/5; 9/12] START activation_function=tanh, init=zero........................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 9/12] END activation_function=tanh, init=zero;, score=0.647 total time=   2.1s\n",
            "[CV 1/5; 10/12] START activation_function=linear, init=uniform..................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 10/12] END activation_function=linear, init=uniform;, score=0.766 total time=   2.0s\n",
            "[CV 2/5; 10/12] START activation_function=linear, init=uniform..................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 10/12] END activation_function=linear, init=uniform;, score=0.714 total time=   1.8s\n",
            "[CV 3/5; 10/12] START activation_function=linear, init=uniform..................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 10/12] END activation_function=linear, init=uniform;, score=0.766 total time=   1.7s\n",
            "[CV 4/5; 10/12] START activation_function=linear, init=uniform..................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 10/12] END activation_function=linear, init=uniform;, score=0.830 total time=   2.0s\n",
            "[CV 5/5; 10/12] START activation_function=linear, init=uniform..................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 10/12] END activation_function=linear, init=uniform;, score=0.752 total time=   1.8s\n",
            "[CV 1/5; 11/12] START activation_function=linear, init=normal...................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 11/12] END activation_function=linear, init=normal;, score=0.773 total time=   2.2s\n",
            "[CV 2/5; 11/12] START activation_function=linear, init=normal...................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 11/12] END activation_function=linear, init=normal;, score=0.721 total time=   1.8s\n",
            "[CV 3/5; 11/12] START activation_function=linear, init=normal...................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 11/12] END activation_function=linear, init=normal;, score=0.760 total time=   2.1s\n",
            "[CV 4/5; 11/12] START activation_function=linear, init=normal...................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 11/12] END activation_function=linear, init=normal;, score=0.830 total time=   2.0s\n",
            "[CV 5/5; 11/12] START activation_function=linear, init=normal...................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 11/12] END activation_function=linear, init=normal;, score=0.758 total time=   2.0s\n",
            "[CV 1/5; 12/12] START activation_function=linear, init=zero.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 12/12] END activation_function=linear, init=zero;, score=0.649 total time=   2.0s\n",
            "[CV 2/5; 12/12] START activation_function=linear, init=zero.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 12/12] END activation_function=linear, init=zero;, score=0.584 total time=   2.0s\n",
            "[CV 3/5; 12/12] START activation_function=linear, init=zero.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 12/12] END activation_function=linear, init=zero;, score=0.630 total time=   2.1s\n",
            "[CV 4/5; 12/12] START activation_function=linear, init=zero.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 12/12] END activation_function=linear, init=zero;, score=0.745 total time=   2.1s\n",
            "[CV 5/5; 12/12] START activation_function=linear, init=zero.....................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 12/12] END activation_function=linear, init=zero;, score=0.647 total time=   1.8s\n"
          ]
        }
      ],
      "source": [
        "# Defining the model\n",
        "\n",
        "def create_model(activation_function,init):\n",
        "    model = Sequential()\n",
        "    model.add(Dense(8,input_dim = 8,kernel_initializer = init,activation = activation_function))\n",
        "    model.add(Dropout(0.1))\n",
        "    model.add(Dense(4,input_dim = 8,kernel_initializer = init,activation = activation_function))\n",
        "    model.add(Dropout(0.1))\n",
        "    model.add(Dense(1,activation = 'sigmoid'))\n",
        "    \n",
        "    adam = Adam(lr = 0.001)\n",
        "    model.compile(loss = 'binary_crossentropy',optimizer = adam,metrics = ['accuracy'])\n",
        "    return model\n",
        "\n",
        "# Create the model\n",
        "\n",
        "model = KerasClassifier(build_fn = create_model,verbose = 0,batch_size = 40,epochs = 50)\n",
        "\n",
        "# Define the grid search parameters\n",
        "activation_function = ['softmax','relu','tanh','linear']\n",
        "init = ['uniform','normal','zero']\n",
        "\n",
        "# Make a dictionary of the grid search parameters\n",
        "param_grids = dict(activation_function = activation_function,init = init)\n",
        "\n",
        "# Build and fit the GridSearchCV\n",
        "\n",
        "grid = GridSearchCV(estimator = model,param_grid = param_grids,cv = KFold(),verbose = 10)\n",
        "grid_result = grid.fit(X_standardized,y)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "NnF7_mcbqU67",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "08c41394-cc90-4ddc-87d5-417863c3419f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best : 0.77873694896698, using {'activation_function': 'relu', 'init': 'normal'}\n",
            "0.6877429723739624,0.08589286105338516 with: {'activation_function': 'softmax', 'init': 'uniform'}\n",
            "0.6563534498214721,0.0532885160025794 with: {'activation_function': 'softmax', 'init': 'normal'}\n",
            "0.6811051726341247,0.07964446189013208 with: {'activation_function': 'softmax', 'init': 'zero'}\n",
            "0.7579322814941406,0.045161838209097464 with: {'activation_function': 'relu', 'init': 'uniform'}\n",
            "0.77873694896698,0.04398544227302887 with: {'activation_function': 'relu', 'init': 'normal'}\n",
            "0.6511586427688598,0.05244526932680711 with: {'activation_function': 'relu', 'init': 'zero'}\n",
            "0.7709022998809815,0.040978502646467956 with: {'activation_function': 'tanh', 'init': 'uniform'}\n",
            "0.7709192752838134,0.04045334234875855 with: {'activation_function': 'tanh', 'init': 'normal'}\n",
            "0.6511586427688598,0.05244526932680711 with: {'activation_function': 'tanh', 'init': 'zero'}\n",
            "0.7656905174255371,0.037375792829326994 with: {'activation_function': 'linear', 'init': 'uniform'}\n",
            "0.7682964086532593,0.035408110721954086 with: {'activation_function': 'linear', 'init': 'normal'}\n",
            "0.6511586427688598,0.05244526932680711 with: {'activation_function': 'linear', 'init': 'zero'}\n"
          ]
        }
      ],
      "source": [
        "# Summarize the results\n",
        "print('Best : {}, using {}'.format(grid_result.best_score_,grid_result.best_params_))\n",
        "means = grid_result.cv_results_['mean_test_score']\n",
        "stds = grid_result.cv_results_['std_test_score']\n",
        "params = grid_result.cv_results_['params']\n",
        "for mean, stdev, param in zip(means, stds, params):\n",
        "  print('{},{} with: {}'.format(mean, stdev, param))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mvKVnnH9qU68"
      },
      "source": [
        "#### Tuning of Hyperparameter :-Number of Neurons in activation layer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "BYq59GbfqU69",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b3b8639e-9d94-4145-b193-56b713c7d8b1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:17: DeprecationWarning: KerasClassifier is deprecated, use Sci-Keras (https://github.com/adriangb/scikeras) instead. See https://www.adriangb.com/scikeras/stable/migration.html for help migrating.\n",
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 5 folds for each of 9 candidates, totalling 45 fits\n",
            "[CV 1/5; 1/9] START neuron1=4, neuron2=2........................................\n",
            "[CV 1/5; 1/9] END .........neuron1=4, neuron2=2;, score=0.760 total time=   2.2s\n",
            "[CV 2/5; 1/9] START neuron1=4, neuron2=2........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 1/9] END .........neuron1=4, neuron2=2;, score=0.721 total time=   2.1s\n",
            "[CV 3/5; 1/9] START neuron1=4, neuron2=2........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 1/9] END .........neuron1=4, neuron2=2;, score=0.753 total time=   2.1s\n",
            "[CV 4/5; 1/9] START neuron1=4, neuron2=2........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 1/9] END .........neuron1=4, neuron2=2;, score=0.837 total time=   2.0s\n",
            "[CV 5/5; 1/9] START neuron1=4, neuron2=2........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 1/9] END .........neuron1=4, neuron2=2;, score=0.758 total time=   2.0s\n",
            "[CV 1/5; 2/9] START neuron1=4, neuron2=4........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 2/9] END .........neuron1=4, neuron2=4;, score=0.760 total time=   2.1s\n",
            "[CV 2/5; 2/9] START neuron1=4, neuron2=4........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 2/9] END .........neuron1=4, neuron2=4;, score=0.747 total time=   1.8s\n",
            "[CV 3/5; 2/9] START neuron1=4, neuron2=4........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 2/9] END .........neuron1=4, neuron2=4;, score=0.740 total time=   2.1s\n",
            "[CV 4/5; 2/9] START neuron1=4, neuron2=4........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 2/9] END .........neuron1=4, neuron2=4;, score=0.843 total time=   2.1s\n",
            "[CV 5/5; 2/9] START neuron1=4, neuron2=4........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 2/9] END .........neuron1=4, neuron2=4;, score=0.752 total time=   1.8s\n",
            "[CV 1/5; 3/9] START neuron1=4, neuron2=8........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 3/9] END .........neuron1=4, neuron2=8;, score=0.753 total time=   1.8s\n",
            "[CV 2/5; 3/9] START neuron1=4, neuron2=8........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 3/9] END .........neuron1=4, neuron2=8;, score=0.740 total time=   2.2s\n",
            "[CV 3/5; 3/9] START neuron1=4, neuron2=8........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 3/9] END .........neuron1=4, neuron2=8;, score=0.747 total time=   1.8s\n",
            "[CV 4/5; 3/9] START neuron1=4, neuron2=8........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 3/9] END .........neuron1=4, neuron2=8;, score=0.830 total time=   2.0s\n",
            "[CV 5/5; 3/9] START neuron1=4, neuron2=8........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 3/9] END .........neuron1=4, neuron2=8;, score=0.758 total time=   2.1s\n",
            "[CV 1/5; 4/9] START neuron1=8, neuron2=2........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 4/9] END .........neuron1=8, neuron2=2;, score=0.734 total time=   1.8s\n",
            "[CV 2/5; 4/9] START neuron1=8, neuron2=2........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 4/9] END .........neuron1=8, neuron2=2;, score=0.747 total time=   2.1s\n",
            "[CV 3/5; 4/9] START neuron1=8, neuron2=2........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 4/9] END .........neuron1=8, neuron2=2;, score=0.747 total time=   2.1s\n",
            "[CV 4/5; 4/9] START neuron1=8, neuron2=2........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 4/9] END .........neuron1=8, neuron2=2;, score=0.830 total time=   1.9s\n",
            "[CV 5/5; 4/9] START neuron1=8, neuron2=2........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 4/9] END .........neuron1=8, neuron2=2;, score=0.752 total time=   2.1s\n",
            "[CV 1/5; 5/9] START neuron1=8, neuron2=4........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 5/9] END .........neuron1=8, neuron2=4;, score=0.766 total time=   3.4s\n",
            "[CV 2/5; 5/9] START neuron1=8, neuron2=4........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 5/9] END .........neuron1=8, neuron2=4;, score=0.714 total time=   2.4s\n",
            "[CV 3/5; 5/9] START neuron1=8, neuron2=4........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 5/9] END .........neuron1=8, neuron2=4;, score=0.747 total time=   2.4s\n",
            "[CV 4/5; 5/9] START neuron1=8, neuron2=4........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 5/9] END .........neuron1=8, neuron2=4;, score=0.843 total time=   2.0s\n",
            "[CV 5/5; 5/9] START neuron1=8, neuron2=4........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 5/9] END .........neuron1=8, neuron2=4;, score=0.771 total time=   2.1s\n",
            "[CV 1/5; 6/9] START neuron1=8, neuron2=8........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 6/9] END .........neuron1=8, neuron2=8;, score=0.766 total time=   2.0s\n",
            "[CV 2/5; 6/9] START neuron1=8, neuron2=8........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 6/9] END .........neuron1=8, neuron2=8;, score=0.714 total time=   1.9s\n",
            "[CV 3/5; 6/9] START neuron1=8, neuron2=8........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 6/9] END .........neuron1=8, neuron2=8;, score=0.747 total time=   1.9s\n",
            "[CV 4/5; 6/9] START neuron1=8, neuron2=8........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 6/9] END .........neuron1=8, neuron2=8;, score=0.856 total time=   2.1s\n",
            "[CV 5/5; 6/9] START neuron1=8, neuron2=8........................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 6/9] END .........neuron1=8, neuron2=8;, score=0.758 total time=   2.1s\n",
            "[CV 1/5; 7/9] START neuron1=16, neuron2=2.......................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 7/9] END ........neuron1=16, neuron2=2;, score=0.734 total time=   1.9s\n",
            "[CV 2/5; 7/9] START neuron1=16, neuron2=2.......................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 7/9] END ........neuron1=16, neuron2=2;, score=0.714 total time=   1.9s\n",
            "[CV 3/5; 7/9] START neuron1=16, neuron2=2.......................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 7/9] END ........neuron1=16, neuron2=2;, score=0.760 total time=   2.1s\n",
            "[CV 4/5; 7/9] START neuron1=16, neuron2=2.......................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 7/9] END ........neuron1=16, neuron2=2;, score=0.843 total time=   2.2s\n",
            "[CV 5/5; 7/9] START neuron1=16, neuron2=2.......................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 7/9] END ........neuron1=16, neuron2=2;, score=0.758 total time=   1.9s\n",
            "[CV 1/5; 8/9] START neuron1=16, neuron2=4.......................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 8/9] END ........neuron1=16, neuron2=4;, score=0.747 total time=   2.1s\n",
            "[CV 2/5; 8/9] START neuron1=16, neuron2=4.......................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 8/9] END ........neuron1=16, neuron2=4;, score=0.721 total time=   1.8s\n",
            "[CV 3/5; 8/9] START neuron1=16, neuron2=4.......................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 8/9] END ........neuron1=16, neuron2=4;, score=0.740 total time=   2.1s\n",
            "[CV 4/5; 8/9] START neuron1=16, neuron2=4.......................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 8/9] END ........neuron1=16, neuron2=4;, score=0.856 total time=   2.1s\n",
            "[CV 5/5; 8/9] START neuron1=16, neuron2=4.......................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 8/9] END ........neuron1=16, neuron2=4;, score=0.758 total time=   2.1s\n",
            "[CV 1/5; 9/9] START neuron1=16, neuron2=8.......................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 1/5; 9/9] END ........neuron1=16, neuron2=8;, score=0.753 total time=   2.0s\n",
            "[CV 2/5; 9/9] START neuron1=16, neuron2=8.......................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 2/5; 9/9] END ........neuron1=16, neuron2=8;, score=0.721 total time=   1.8s\n",
            "[CV 3/5; 9/9] START neuron1=16, neuron2=8.......................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 3/5; 9/9] END ........neuron1=16, neuron2=8;, score=0.773 total time=   1.9s\n",
            "[CV 4/5; 9/9] START neuron1=16, neuron2=8.......................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 4/5; 9/9] END ........neuron1=16, neuron2=8;, score=0.843 total time=   1.8s\n",
            "[CV 5/5; 9/9] START neuron1=16, neuron2=8.......................................\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CV 5/5; 9/9] END ........neuron1=16, neuron2=8;, score=0.745 total time=   2.5s\n"
          ]
        }
      ],
      "source": [
        "# Defining the model\n",
        "\n",
        "def create_model(neuron1,neuron2):\n",
        "    model = Sequential()\n",
        "    model.add(Dense(neuron1,input_dim = 8,kernel_initializer = 'normal',activation = 'relu'))\n",
        "    model.add(Dropout(0.1))\n",
        "    model.add(Dense(neuron2,input_dim = neuron1,kernel_initializer = 'normal',activation = 'relu'))\n",
        "    model.add(Dropout(0.1))\n",
        "    model.add(Dense(1,activation = 'sigmoid'))\n",
        "    \n",
        "    adam = Adam(lr = 0.001)\n",
        "    model.compile(loss = 'binary_crossentropy',optimizer = adam,metrics = ['accuracy'])\n",
        "    return model\n",
        "\n",
        "# Create the model\n",
        "\n",
        "model = KerasClassifier(build_fn = create_model,verbose = 0,batch_size = 40,epochs = 50)\n",
        "\n",
        "# Define the grid search parameters\n",
        "\n",
        "neuron1 = [4,8,16]\n",
        "neuron2 = [2,4,8]\n",
        "\n",
        "# Make a dictionary of the grid search parameters\n",
        "\n",
        "param_grids = dict(neuron1 = neuron1,neuron2 = neuron2)\n",
        "\n",
        "# Build and fit the GridSearchCV\n",
        "\n",
        "grid = GridSearchCV(estimator = model,param_grid = param_grids,cv = KFold(),verbose = 10)\n",
        "grid_result = grid.fit(X_standardized,y)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "BjNJt-YjqU6-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "459f009e-5302-4248-b085-05f05f486c1d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best : 0.7683303713798523, using {'neuron1': 8, 'neuron2': 4}\n",
            "0.7657074928283691,0.038189463682198346 with: {'neuron1': 4, 'neuron2': 2}\n",
            "0.7683049082756043,0.03795301604333752 with: {'neuron1': 4, 'neuron2': 4}\n",
            "0.7656990170478821,0.032744069340224405 with: {'neuron1': 4, 'neuron2': 8}\n",
            "0.7617944359779358,0.03464738547125577 with: {'neuron1': 8, 'neuron2': 2}\n",
            "0.7683303713798523,0.042415416859472206 with: {'neuron1': 8, 'neuron2': 4}\n",
            "0.7683303713798523,0.04736392197016323 with: {'neuron1': 8, 'neuron2': 8}\n",
            "0.7618198871612549,0.04399241961772094 with: {'neuron1': 16, 'neuron2': 2}\n",
            "0.764434278011322,0.04746505613137959 with: {'neuron1': 16, 'neuron2': 4}\n",
            "0.7669977068901062,0.04155692682007004 with: {'neuron1': 16, 'neuron2': 8}\n"
          ]
        }
      ],
      "source": [
        "# Summarize the results\n",
        "print('Best : {}, using {}'.format(grid_result.best_score_,grid_result.best_params_))\n",
        "means = grid_result.cv_results_['mean_test_score']\n",
        "stds = grid_result.cv_results_['std_test_score']\n",
        "params = grid_result.cv_results_['params']\n",
        "for mean, stdev, param in zip(means, stds, params):\n",
        "  print('{},{} with: {}'.format(mean, stdev, param))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pH9CbVQ9qU6_"
      },
      "source": [
        "#### Training model with optimum values of Hyperparameters"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "nVwz6faPqU7A",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6651e925-9bd8-4d02-c067-4aa51eea6ad9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:19: DeprecationWarning: KerasClassifier is deprecated, use Sci-Keras (https://github.com/adriangb/scikeras) instead. See https://www.adriangb.com/scikeras/stable/migration.html for help migrating.\n",
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.7916666666666666\n"
          ]
        }
      ],
      "source": [
        "from sklearn.metrics import classification_report, accuracy_score\n",
        "\n",
        "# Defining the model\n",
        "\n",
        "def create_model():\n",
        "    model = Sequential()\n",
        "    model.add(Dense(8,input_dim = 8,kernel_initializer = 'normal',activation = 'relu'))\n",
        "    model.add(Dropout(0.1))\n",
        "    model.add(Dense(4,input_dim = 8,kernel_initializer = 'normal',activation = 'relu'))\n",
        "    model.add(Dropout(0.1))\n",
        "    model.add(Dense(1,activation = 'sigmoid'))\n",
        "    \n",
        "    adam = Adam(lr = 0.001) #sgd = SGD(lr=learning_rate, momentum=momentum, decay=decay_rate, nesterov=False)\n",
        "    model.compile(loss = 'binary_crossentropy',optimizer = adam,metrics = ['accuracy'])\n",
        "    return model\n",
        "\n",
        "# Create the model\n",
        "\n",
        "model = KerasClassifier(build_fn = create_model,verbose = 0,batch_size = 40,epochs = 50)\n",
        "\n",
        "# Fitting the model\n",
        "\n",
        "model.fit(X_standardized,y)\n",
        "\n",
        "# Predicting using trained model\n",
        "\n",
        "y_predict = model.predict(X_standardized)\n",
        "\n",
        "# Printing the metrics\n",
        "print(accuracy_score(y,y_predict))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_FBFA_7mqU7C"
      },
      "source": [
        "# Hyperparameters all at once"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2QYSeHXOqU7D"
      },
      "source": [
        "\n",
        "The hyperparameter optimization was carried out by taking 2 hyperparameters at once. We may have missed the best values. The performance can be further improved by finding the optimum values of hyperparameters all at once given by the code snippet below.\n",
        "#### This process is computationally expensive."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fNlN7YsGqU7D"
      },
      "outputs": [],
      "source": [
        "def create_model(learning_rate,dropout_rate,activation_function,init,neuron1,neuron2):\n",
        "    model = Sequential()\n",
        "    model.add(Dense(neuron1,input_dim = 8,kernel_initializer = init,activation = activation_function))\n",
        "    model.add(Dropout(dropout_rate))\n",
        "    model.add(Dense(neuron2,input_dim = neuron1,kernel_initializer = init,activation = activation_function))\n",
        "    model.add(Dropout(dropout_rate))\n",
        "    model.add(Dense(1,activation = 'sigmoid'))\n",
        "    \n",
        "    adam = Adam(lr = learning_rate)\n",
        "    model.compile(loss = 'binary_crossentropy',optimizer = adam,metrics = ['accuracy'])\n",
        "    return model\n",
        "\n",
        "# Create the model\n",
        "\n",
        "model = KerasClassifier(build_fn = create_model,verbose = 0)\n",
        "\n",
        "# Define the grid search parameters\n",
        "\n",
        "batch_size = [10,20,40]\n",
        "epochs = [10,50,100]\n",
        "learning_rate = [0.001,0.01,0.1]\n",
        "dropout_rate = [0.0,0.1,0.2]\n",
        "activation_function = ['softmax','relu','tanh','linear']\n",
        "init = ['uniform','normal','zero']\n",
        "neuron1 = [4,8,16]\n",
        "neuron2 = [2,4,8]\n",
        "\n",
        "# Make a dictionary of the grid search parameters\n",
        "\n",
        "param_grids = dict(batch_size = batch_size,epochs = epochs,learning_rate = learning_rate,dropout_rate = dropout_rate,\n",
        "                   activation_function = activation_function,init = init,neuron1 = neuron1,neuron2 = neuron2)\n",
        "\n",
        "# Build and fit the GridSearchCV\n",
        "\n",
        "grid = GridSearchCV(estimator = model,param_grid = param_grids,cv = KFold(),verbose = 10)\n",
        "grid_result = grid.fit(X_standardized,y)\n",
        "\n",
        "# Summarize the results\n",
        "print('Best : {}, using {}'.format(grid_result.best_score_,grid_result.best_params_))\n",
        "means = grid_result.cv_results_['mean_test_score']\n",
        "stds = grid_result.cv_results_['std_test_score']\n",
        "params = grid_result.cv_results_['params']\n",
        "for mean, stdev, param in zip(means, stds, params):\n",
        "  print('{},{} with: {}'.format(mean, stdev, param))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "weZL6A-TqU7F"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    },
    "colab": {
      "name": "Neural Network_Hyperparameter Tuning.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}