Hierarchical_Attention.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Hierarchical_Attention.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/readthroughmyglasses/FYP_QUORA/blob/master/Hierarchical_Attention.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eQpfcYmpE0oO",
        "colab_type": "code",
        "outputId": "a2f53d0d-e501-4f77-9ae8-043b0c49fdd4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "# drive.mount(\"/content/drive\", force_remount=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ojgdc2hiYVcn",
        "colab_type": "text"
      },
      "source": [
        "**Setting the path for data**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sH05NoD4Fk4P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "path = '/content/drive/My Drive/FINAL DESTINATION/DATA/train.csv'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ola3E_IzQQVi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# data_folder = '/content/drive/My Drive/FINAL DESTINATION/DATA/'\n",
        "data_folder = '/content/drive/My Drive/DS/DATA/'\n",
        "requirements = '/content/drive/My Drive/FINAL DESTINATION/DATA/requirements.txt'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a0IHmesHpz4q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "input_folder = '/content/drive/My Drive/DS'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Th14zuZ8Dolh",
        "colab_type": "code",
        "outputId": "91ebb052-b26a-4276-ec18-a4269ee7808a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "#!python -m spacy download en_vectors_web_lg\n",
        "!pip install utils\n",
        "\n",
        "!pip install jellyfish\n",
        "!pip install testfixtures\n",
        "!pip install vocabulary\n",
        "\n",
        "!pip freeze > '/content/drive/My Drive/FINAL DESTINATION/DATA/requirements.txt'\n",
        "\n",
        "!pip install hyperas"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: utils in /usr/local/lib/python3.6/dist-packages (1.0.1)\n",
            "Requirement already satisfied: jellyfish in /usr/local/lib/python3.6/dist-packages (0.7.2)\n",
            "Requirement already satisfied: testfixtures in /usr/local/lib/python3.6/dist-packages (6.12.0)\n",
            "Requirement already satisfied: vocabulary in /usr/local/lib/python3.6/dist-packages (1.0.4)\n",
            "Requirement already satisfied: mock==2.0.0 in /usr/local/lib/python3.6/dist-packages (from vocabulary) (2.0.0)\n",
            "Requirement already satisfied: requests==2.13.0 in /usr/local/lib/python3.6/dist-packages (from vocabulary) (2.13.0)\n",
            "Requirement already satisfied: pbr>=0.11 in /usr/local/lib/python3.6/dist-packages (from mock==2.0.0->vocabulary) (5.4.4)\n",
            "Requirement already satisfied: six>=1.9 in /usr/local/lib/python3.6/dist-packages (from mock==2.0.0->vocabulary) (1.12.0)\n",
            "/bin/bash: /content/drive/My Drive/FINAL DESTINATION/DATA/requirements.txt: No such file or directory\n",
            "Requirement already satisfied: hyperas in /usr/local/lib/python3.6/dist-packages (0.4.1)\n",
            "Requirement already satisfied: jupyter in /usr/local/lib/python3.6/dist-packages (from hyperas) (1.0.0)\n",
            "Requirement already satisfied: nbformat in /usr/local/lib/python3.6/dist-packages (from hyperas) (5.0.4)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.6/dist-packages (from hyperas) (0.3)\n",
            "Requirement already satisfied: hyperopt in /usr/local/lib/python3.6/dist-packages (from hyperas) (0.1.2)\n",
            "Requirement already satisfied: keras in /usr/local/lib/python3.6/dist-packages (from hyperas) (2.2.5)\n",
            "Requirement already satisfied: nbconvert in /usr/local/lib/python3.6/dist-packages (from hyperas) (5.6.1)\n",
            "Requirement already satisfied: qtconsole in /usr/local/lib/python3.6/dist-packages (from jupyter->hyperas) (4.6.0)\n",
            "Requirement already satisfied: jupyter-console in /usr/local/lib/python3.6/dist-packages (from jupyter->hyperas) (5.2.0)\n",
            "Requirement already satisfied: notebook in /usr/local/lib/python3.6/dist-packages (from jupyter->hyperas) (5.2.2)\n",
            "Requirement already satisfied: ipywidgets in /usr/local/lib/python3.6/dist-packages (from jupyter->hyperas) (7.5.1)\n",
            "Requirement already satisfied: ipykernel in /usr/local/lib/python3.6/dist-packages (from jupyter->hyperas) (4.6.1)\n",
            "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /usr/local/lib/python3.6/dist-packages (from nbformat->hyperas) (2.6.0)\n",
            "Requirement already satisfied: jupyter-core in /usr/local/lib/python3.6/dist-packages (from nbformat->hyperas) (4.6.1)\n",
            "Requirement already satisfied: traitlets>=4.1 in /usr/local/lib/python3.6/dist-packages (from nbformat->hyperas) (4.3.3)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.6/dist-packages (from nbformat->hyperas) (0.2.0)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.6/dist-packages (from hyperopt->hyperas) (2.4)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from hyperopt->hyperas) (1.4.1)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from hyperopt->hyperas) (0.16.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from hyperopt->hyperas) (1.17.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from hyperopt->hyperas) (1.12.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from hyperopt->hyperas) (4.28.1)\n",
            "Requirement already satisfied: pymongo in /usr/local/lib/python3.6/dist-packages (from hyperopt->hyperas) (3.10.1)\n",
            "Requirement already satisfied: keras-applications>=1.0.8 in /usr/local/lib/python3.6/dist-packages (from keras->hyperas) (1.0.8)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from keras->hyperas) (3.13)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras->hyperas) (2.8.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from keras->hyperas) (1.1.0)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.6/dist-packages (from nbconvert->hyperas) (0.6.0)\n",
            "Requirement already satisfied: jinja2>=2.4 in /usr/local/lib/python3.6/dist-packages (from nbconvert->hyperas) (2.11.1)\n",
            "Requirement already satisfied: bleach in /usr/local/lib/python3.6/dist-packages (from nbconvert->hyperas) (3.1.0)\n",
            "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.6/dist-packages (from nbconvert->hyperas) (1.4.2)\n",
            "Requirement already satisfied: testpath in /usr/local/lib/python3.6/dist-packages (from nbconvert->hyperas) (0.4.4)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.6/dist-packages (from nbconvert->hyperas) (2.1.3)\n",
            "Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.6/dist-packages (from nbconvert->hyperas) (0.8.4)\n",
            "Requirement already satisfied: jupyter-client>=4.1 in /usr/local/lib/python3.6/dist-packages (from qtconsole->jupyter->hyperas) (5.3.4)\n",
            "Requirement already satisfied: ipython in /usr/local/lib/python3.6/dist-packages (from jupyter-console->jupyter->hyperas) (5.5.0)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from jupyter-console->jupyter->hyperas) (1.0.18)\n",
            "Requirement already satisfied: terminado>=0.3.3; sys_platform != \"win32\" in /usr/local/lib/python3.6/dist-packages (from notebook->jupyter->hyperas) (0.8.3)\n",
            "Requirement already satisfied: tornado>=4 in /usr/local/lib/python3.6/dist-packages (from notebook->jupyter->hyperas) (4.5.3)\n",
            "Requirement already satisfied: widgetsnbextension~=3.5.0 in /usr/local/lib/python3.6/dist-packages (from ipywidgets->jupyter->hyperas) (3.5.1)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.6/dist-packages (from traitlets>=4.1->nbformat->hyperas) (4.4.1)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from jinja2>=2.4->nbconvert->hyperas) (1.1.1)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.6/dist-packages (from bleach->nbconvert->hyperas) (0.5.1)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from jupyter-client>=4.1->qtconsole->jupyter->hyperas) (2.6.1)\n",
            "Requirement already satisfied: pyzmq>=13 in /usr/local/lib/python3.6/dist-packages (from jupyter-client>=4.1->qtconsole->jupyter->hyperas) (17.0.0)\n",
            "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.6/dist-packages (from ipython->jupyter-console->jupyter->hyperas) (4.8.0)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.6/dist-packages (from ipython->jupyter-console->jupyter->hyperas) (0.7.5)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.6/dist-packages (from ipython->jupyter-console->jupyter->hyperas) (0.8.1)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.6/dist-packages (from ipython->jupyter-console->jupyter->hyperas) (45.1.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from prompt-toolkit<2.0.0,>=1.0.0->jupyter-console->jupyter->hyperas) (0.1.8)\n",
            "Requirement already satisfied: ptyprocess; os_name != \"nt\" in /usr/local/lib/python3.6/dist-packages (from terminado>=0.3.3; sys_platform != \"win32\"->notebook->jupyter->hyperas) (0.6.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "otFJQJBtFxJ6",
        "colab_type": "code",
        "outputId": "57a8ccd9-0ae1-4075-fbaf-2263d17c5cda",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 552
        }
      },
      "source": [
        "import io\n",
        "import seaborn as sns\n",
        "import spacy\n",
        "\n",
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from math import*\n",
        "\n",
        "import sys\n",
        "import os \n",
        "import re\n",
        "import string\n",
        "\n",
        "import jellyfish\n",
        "import gensim\n",
        "from gensim.models import *\n",
        "from gensim.parsing.preprocessing import preprocess_string, strip_tags, strip_multiple_whitespaces, remove_stopwords, stem_text\n",
        "from gensim.test.test_doc2vec import ConcatenatedDoc2Vec\n",
        "\n",
        "import nltk\n",
        "from nltk.tokenize import RegexpTokenizer\n",
        "from nltk.corpus import stopwords\n",
        "from nltk import word_tokenize\n",
        "from nltk.stem.porter import PorterStemmer\n",
        "nltk.download('stopwords')\n",
        "nltk.download('universal_tagset')\n",
        "nltk.download('punkt')\n",
        "\n",
        "import utils\n",
        "import vocabulary\n",
        "import sklearn\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn import metrics\n",
        "from sklearn.metrics import classification_report, log_loss\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.metrics import recall_score\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.metrics import roc_curve, auc\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from importlib import reload\n",
        "from textblob import TextBlob\n",
        "reload(utils)\n",
        "\n",
        "import time\n",
        "import timeit\n",
        "import itertools\n",
        "\n",
        "import keras\n",
        "from keras import layers, Model, models, initializers\n",
        "from keras.engine.topology import Layer\n",
        "from keras import backend as K\n",
        "from keras.optimizers import Adam,RMSprop, Nadam, SGD\n",
        "from keras.callbacks import EarlyStopping\n",
        "\n",
        "import sys\n",
        "sys.path.append('/content/drive/My Drive/FINAL DESTINATION/DATA/src/utils/')\n",
        "from plotting import plot_roc\n",
        "# import src.utils.dl_utils\n",
        "# from src.utils.dl_utils import *\n",
        "\n",
        "from dl_utils import init_session, data, plot_optimization_history\n",
        "\n",
        "import tensorflow as tf\n",
        "from keras.backend.tensorflow_backend import set_session\n",
        "\n",
        "from hyperopt import Trials, STATUS_OK, tpe\n",
        "from hyperas import optim\n",
        "from hyperas.distributions import choice, uniform, loguniform, lognormal, normal\n",
        "import gc\n",
        "\n",
        "import pickle"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package universal_tagset to /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/universal_tagset.zip.\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "error",
          "ename": "ModuleNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-5-0c59f0654b26>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     61\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive/My Drive/FINAL DESTINATION/DATA/src/utils/'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mplotting\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mplot_roc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     64\u001b[0m \u001b[0;31m# import src.utils.dl_utils\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m \u001b[0;31m# from src.utils.dl_utils import *\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'plotting'",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HgaIkTTJHgwZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dtypes = {'id': 'int64','qid1': 'int64','qid2': 'int64','question1': 'object','question2': 'object','is_duplicate': 'int64'}\n",
        "df = pd.read_csv('/content/drive/My Drive/FINAL DESTINATION/DATA/train.csv', sep=',', header=0, usecols=dtypes.keys(), skipinitialspace=True, skip_blank_lines=True, encoding='utf-8')\n",
        "df = df.set_index('id')\n",
        "df = df.dropna()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n8j6Fq0uHgzv",
        "colab_type": "code",
        "outputId": "9726c42c-797d-48a5-9838-9cc3b242f208",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        }
      },
      "source": [
        "X = df.drop(columns=['is_duplicate'])\n",
        "y = df['is_duplicate']\n",
        "X_train, X_test, y_train, y_test =  train_test_split(X, y, test_size=0.5, random_state=42)\n",
        "X_train.info()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 202143 entries, 79039 to 121959\n",
            "Data columns (total 4 columns):\n",
            "qid1         202143 non-null int64\n",
            "qid2         202143 non-null int64\n",
            "question1    202143 non-null object\n",
            "question2    202143 non-null object\n",
            "dtypes: int64(2), object(2)\n",
            "memory usage: 7.7+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m9htQ4TPJUAT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pickle\n",
        "pickle.dump(X_train, open(input_folder+'X_train.p', 'wb'))\n",
        "pickle.dump(y_train, open(input_folder+'y_train.p', 'wb'))\n",
        "pickle.dump(X_test, open(input_folder+'X_test.p', 'wb'))\n",
        "pickle.dump(y_test, open(input_folder+'y_test.p', 'wb'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9gfNI6p-M6aA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "del X,y,df,X_train,X_test, y_train, y_test"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y5iTvXWYM6k7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train = pickle.load(open(input_folder+'X_train.p', 'rb'))\n",
        "X_test = pickle.load(open(input_folder+'X_test.p', 'rb'))\n",
        "y_train = pickle.load(open(input_folder+'y_train.p', 'rb'))\n",
        "y_test = pickle.load(open(input_folder+'y_test.p', 'rb'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ezIVBHWeZLcp",
        "colab_type": "code",
        "outputId": "218b458d-44a9-4a80-c5d4-ab1de2edabd0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        }
      },
      "source": [
        "!python -m spacy download en"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: en_core_web_sm==2.1.0 from https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.1.0/en_core_web_sm-2.1.0.tar.gz#egg=en_core_web_sm==2.1.0 in /usr/local/lib/python3.6/dist-packages (2.1.0)\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the model via spacy.load('en_core_web_sm')\n",
            "\u001b[38;5;2m✔ Linking successful\u001b[0m\n",
            "/usr/local/lib/python3.6/dist-packages/en_core_web_sm -->\n",
            "/usr/local/lib/python3.6/dist-packages/spacy/data/en\n",
            "You can now load the model via spacy.load('en')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1W83bbATN9-_",
        "colab_type": "code",
        "outputId": "d5580d35-5f12-45bc-e818-afaf51bcbf88",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "!python -m spacy download en_core_web_lg"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting en_core_web_lg==2.1.0\n",
            "\u001b[?25l  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-2.1.0/en_core_web_lg-2.1.0.tar.gz (826.9MB)\n",
            "\u001b[K     |████████████████████████████████| 826.9MB 1.1MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: en-core-web-lg\n",
            "  Building wheel for en-core-web-lg (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for en-core-web-lg: filename=en_core_web_lg-2.1.0-cp36-none-any.whl size=828255078 sha256=73c4a58433a69395e6f25605ffa125a7fdd838b9aa671f9ecc1d0c018d1d90c3\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-64bj8kvd/wheels/b4/d7/70/426d313a459f82ed5e06cc36a50e2bb2f0ec5cb31d8e0bdf09\n",
            "Successfully built en-core-web-lg\n",
            "Installing collected packages: en-core-web-lg\n",
            "Successfully installed en-core-web-lg-2.1.0\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the model via spacy.load('en_core_web_lg')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-ZAOjObXTwxx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import en_core_web_lg\n",
        "nlp = en_core_web_lg.load()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q8Uy0HuTN-Fj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def create_dataset(nlp, X, num_oov, max_length, norm_vectors = True):\n",
        "    len_q1 = X['question1'].size\n",
        "    sents = pd.concat([X['question1'], X['question2']]).values\n",
        "    \n",
        "    # the extra +1 is for a zero vector represting NULL for padding\n",
        "    num_vectors = max(lex.rank for lex in nlp.vocab) + 2 \n",
        "    \n",
        "    # create random vectors for OOV tokens\n",
        "    oov = np.random.normal(size=(num_oov, nlp.vocab.vectors_length))\n",
        "    oov = oov / oov.sum(axis=1, keepdims=True)\n",
        "    \n",
        "    vectors = np.zeros((num_vectors + num_oov, nlp.vocab.vectors_length), dtype='float32')\n",
        "    vectors[num_vectors:, ] = oov\n",
        "    for lex in nlp.vocab:\n",
        "        if lex.has_vector and lex.vector_norm > 0:\n",
        "            vectors[lex.rank + 1] = lex.vector / lex.vector_norm if norm_vectors == True else lex.vector\n",
        "            \n",
        "    sents_as_ids = []\n",
        "    for sent in sents:\n",
        "        doc = nlp(sent)\n",
        "        word_ids = []\n",
        "        \n",
        "        for i, token in enumerate(doc):\n",
        "            # skip odd spaces from tokenizer\n",
        "            if token.has_vector and token.vector_norm == 0:\n",
        "                continue\n",
        "                \n",
        "            if i > max_length:\n",
        "                break\n",
        "                \n",
        "            if token.has_vector:\n",
        "                word_ids.append(token.rank + 1)\n",
        "            else:\n",
        "                # if we don't have a vector, pick an OOV entry\n",
        "                word_ids.append(token.rank % num_oov + num_vectors) \n",
        "                \n",
        "        # there must be a simpler way of generating padded arrays from lists...\n",
        "        word_id_vec = np.zeros((max_length), dtype='int')\n",
        "        clipped_len = min(max_length, len(word_ids))\n",
        "        word_id_vec[:clipped_len] = word_ids[:clipped_len]\n",
        "        sents_as_ids.append(word_id_vec)        \n",
        "        \n",
        "    return vectors, np.array(sents_as_ids[:len_q1]), np.array(sents_as_ids[len_q1:])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JqIL0zLcN-Lh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "def create_embedding(vectors, max_length, projected_dim):\n",
        "    return models.Sequential([\n",
        "        layers.Embedding(\n",
        "            vectors.shape[0],\n",
        "            vectors.shape[1],\n",
        "            input_length=max_length,\n",
        "            weights=[vectors],\n",
        "            trainable=False),\n",
        "        \n",
        "        layers.TimeDistributed(\n",
        "            layers.Dense(projected_dim,\n",
        "                         activation=None,\n",
        "                         use_bias=False))\n",
        "    ])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I1efLRTgR6Yz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "w2v, q1_train_w2v, q2_train_w2v = create_dataset(nlp, X_train, 100, 50, True)\n",
        "\n",
        "pickle.dump(w2v, open(data_folder+'w2v.p', 'wb'))\n",
        "pickle.dump(q1_train_w2v, open(data_folder+'q1_train_w2v.p', 'wb'))\n",
        "pickle.dump(q2_train_w2v, open(data_folder+'q2_train_w2v.p', 'wb'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9EfYuyZGR6iz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "_, q1_test_w2v, q2_test_w2v = create_dataset(nlp, X_test, 100, 50, True)\n",
        "\n",
        "\n",
        "pickle.dump(q1_test_w2v, open(data_folder+'q1_test_w2v.p', 'wb'))\n",
        "pickle.dump(q2_test_w2v, open(data_folder+'q2_test_w2v.p', 'wb'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k5RBYiTyR6wp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "w2v = pickle.load(open(data_folder+'w2v.p', 'rb'))\n",
        "q1_train_w2v = pickle.load(open(data_folder+'q1_train_w2v.p', 'rb'))\n",
        "q2_train_w2v = pickle.load(open(data_folder+'q2_train_w2v.p', 'rb'))\n",
        "q1_test_w2v = pickle.load(open(data_folder+'q1_test_w2v.p', 'rb'))\n",
        "q2_test_w2v = pickle.load(open(data_folder+'q2_test_w2v.p', 'rb'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dWHyN4sTwWLs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.callbacks import ModelCheckpoint"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ArUeouF4nVjC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.layers import Layer, RNN\n",
        "\n",
        "class Attention_Layer(Layer):\n",
        "\n",
        "    def __init__(self, output_dim):\n",
        "        self.output_dim = output_dim\n",
        "        super(Attention_Layer, self).__init__()\n",
        "\n",
        "    def build(self, input_shape):\n",
        "        # Create a trainable weight variable for this layer.\n",
        "        self.W = self.add_weight(name='W', \n",
        "                                      shape=(input_shape[-1], self.output_dim),\n",
        "                                      initializer='uniform',\n",
        "                                      trainable=True)\n",
        "        self.b = self.add_weight(name='b', \n",
        "                                      shape=(self.output_dim,),\n",
        "                                      initializer='uniform',\n",
        "                                      trainable=True)\n",
        "        self.u = self.add_weight(name='u', \n",
        "                                      shape=(self.output_dim,1),\n",
        "                                      initializer='uniform',\n",
        "                                      trainable=True)\n",
        "        super(Attention_Layer, self).build(input_shape)  # Be sure to call this at the end\n",
        "\n",
        "    def call(self, h_it):        \n",
        "        u_it = K.tanh(K.bias_add(K.dot(h_it, self.W), self.b))\n",
        "        att_weights = K.dot(u_it, self.u)\n",
        "        exp_weights = K.exp(att_weights)\n",
        "        sum_weights = K.sum(exp_weights, axis=1, keepdims=True)\n",
        "        alpha_it = exp_weights/sum_weights\n",
        "        return K.sum(h_it*alpha_it, axis=1)\n",
        "\n",
        "    def compute_output_shape(self, input_shape):\n",
        "        return (input_shape[0], input_shape[-1])\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l7qqdf35n9mw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.callbacks import LambdaCallback\n",
        "def build_hier_model(vectors, optimizer, learn_rate, dropout_rate1, dropout_rate2, max_length=50, num_hidden=200, num_classes=1, \n",
        "                projected_dim=200):\n",
        "    # optimizer=params['optimizer']\n",
        "    # learn_rate=params['learn_rate']\n",
        "    # dropout_rate1=params['dropout_rate1']\n",
        "    # dropout_rate2=params['dropout_rate1']\n",
        "    #K.clear_session()\n",
        "    # input    \n",
        "    model_input = layers.Input(shape=(2, max_length), dtype='int32')\n",
        "    \n",
        "    # embeddings (projected)\n",
        "    embed = create_embedding(vectors, max_length, projected_dim)\n",
        "    \n",
        "    # step 1: word encoder\n",
        "    word_sequence_input = layers.Input(shape=(max_length,), dtype='int32')\n",
        "    h_w = layers.Bidirectional(layers.GRU(num_hidden, dropout=dropout_rate1, return_sequences=True))(embed(word_sequence_input))\n",
        "    \n",
        "    # step 2: word attention\n",
        "    s_w = Attention_Layer(num_hidden)(h_w)\n",
        "    word_encode_attend = Model(word_sequence_input, s_w)\n",
        "    \n",
        "    # step 3: sentence encoder\n",
        "    sent_encode_attend = layers.TimeDistributed(word_encode_attend)(model_input)\n",
        "    h = layers.Bidirectional(layers.GRU(num_hidden, dropout=dropout_rate2, return_sequences=True))(sent_encode_attend)\n",
        "    \n",
        "    # step 4: sentence attention\n",
        "    v = Attention_Layer(num_hidden)(h)\n",
        "\n",
        "    out = layers.Dense(num_classes, activation='sigmoid', use_bias=True)(v)\n",
        "    \n",
        "    model = Model(model_input, out)\n",
        "    \n",
        "    if optimizer == 'sgd':\n",
        "        opt = SGD(lr=learn_rate)\n",
        "    elif optimizer == 'adam':\n",
        "        opt = Adam(lr=learn_rate)\n",
        "    elif optimizer == 'rmsprop':\n",
        "        opt = RMSprop(lr=learn_rate)\n",
        "    else:\n",
        "        opt = Nadam(lr=learn_rate)\n",
        "    \n",
        "    model.compile(optimizer=opt,\n",
        "                  loss='binary_crossentropy',\n",
        "                  metrics=['accuracy'])\n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q05CJzVgojTl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from hyperopt import fmin, tpe, hp, Trials\n",
        "from hyperopt import STATUS_OK\n",
        "\n",
        "x_train, y_train, x_test, y_test = data()\n",
        "w2v = pickle.load(open(data_folder+'w2v.p', 'rb'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u8QGtYdGSCSm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.callbacks import LearningRateScheduler\n",
        "# function for exponential decay (used for laerning rate)\n",
        "def model2_exp_decay(init_lr, k):\n",
        "    def _exp_decay(epoch):\n",
        "        lrate = init_lr * np.exp(-k*epoch)\n",
        "        return lrate\n",
        "    return _exp_decay"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MIDYDlroSbq2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "54BIwcbIBvRI",
        "colab_type": "code",
        "outputId": "05c22486-eb53-4d96-d544-49c17033a953",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "checkpoint_path='/content/drive/My Drive/DS/DATA/model2-{epoch:02d}-{val_acc:.2f}.hdf5'\n",
        "\n",
        "x_train\n",
        "y_train"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "337571    0\n",
              "79039     0\n",
              "12796     1\n",
              "26184     0\n",
              "214725    0\n",
              "         ..\n",
              "259178    0\n",
              "365838    1\n",
              "131932    1\n",
              "146867    0\n",
              "121958    1\n",
              "Name: is_duplicate, Length: 202145, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tUbAkeUQ3K2g",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# optimization function which will be called by hyperopt for each combination of variable parameters\n",
        "def opt_deep_fn(params):\n",
        "    K.clear_session()\n",
        "    # this is needed to free up gpu memory after every evaluation\n",
        "    gc.collect()\n",
        "    model = build_hier_model(vectors=w2v, optimizer='adam', \n",
        "                             learn_rate= 0.01, dropout_rate1=params['dropout_rate1'], \n",
        "                             dropout_rate2=params['dropout_rate2'], max_length=50, projected_dim=200, \n",
        "                             num_classes=1, num_hidden=200)\n",
        "    print_weights = LambdaCallback(on_epoch_end = lambda batch,logs:print(model.layers[0].get_weights()))\n",
        "    early_stopping = EarlyStopping(monitor='val_loss', patience=10)\n",
        "    learn_rate = LearningRateScheduler(model2_exp_decay(params['learn_rate'], params['decay']))\n",
        "    model_checkpoint = ModelCheckpoint(checkpoint_path, monitor='val_loss', save_best_only=True, mode='min')\n",
        "\n",
        "    result = model.fit(x_train, y_train.values, batch_size=128, epochs=20,\n",
        "      validation_split=0.33, callbacks=[early_stopping, learn_rate,model_checkpoint,print_weights],verbose=2)\n",
        "    validation_acc = np.amax(result.history['val_acc']) \n",
        "    print('Best validation accuracy of eval run:', validation_acc)\n",
        "    #print('history of eval run:', result.history)\n",
        "    return {'loss': -validation_acc, 'status': STATUS_OK,\n",
        "            'train_acc': np.amax(result.history['acc'])}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ABFCFf7V3LDr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# search space for all variable hyperparameters\n",
        "search_deep_space = {\n",
        "                'learn_rate':hp.loguniform('learn_rate',-6,-3),\n",
        "                'dropout_rate1':hp.uniform('dropout_rate1',0, 1),\n",
        "                'dropout_rate2':hp.uniform('dropout_rate2',0, 1),\n",
        "                'decay':hp.uniform('decay',0, 1)\n",
        "                }"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_h--1VVu3LOP",
        "colab_type": "code",
        "outputId": "b4e29978-7cda-4ce0-a33b-3282cba7a100",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "init_session()\n",
        "model2_run2_trials=Trials()\n",
        "\n",
        "# hyperopt optimization\n",
        "model2_run2_best = fmin(opt_deep_fn,\n",
        "                        search_deep_space,\n",
        "                          algo=tpe.suggest,\n",
        "                          max_evals=10,\n",
        "                           trials=model2_run2_trials)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Device mapping:\n",
            "/job:localhost/replica:0/task:0/device:XLA_CPU:0 -> device: XLA_CPU device\n",
            "/job:localhost/replica:0/task:0/device:XLA_GPU:0 -> device: XLA_GPU device\n",
            "/job:localhost/replica:0/task:0/device:GPU:0 -> device: 0, name: Tesla T4, pci bus id: 0000:00:04.0, compute capability: 7.5\n",
            "\n",
            "Train on 135437 samples, validate on 66708 samples\n",
            "Epoch 1/20\n",
            " - 196s - loss: 0.6659 - acc: 0.6274 - val_loss: 0.6597 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 2/20\n",
            " - 197s - loss: 0.6616 - acc: 0.6315 - val_loss: 0.6612 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 3/20\n",
            " - 192s - loss: 0.6641 - acc: 0.6281 - val_loss: 0.6594 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 4/20\n",
            " - 197s - loss: 0.6629 - acc: 0.6316 - val_loss: 0.6751 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 5/20\n",
            " - 193s - loss: 0.6627 - acc: 0.6313 - val_loss: 0.6888 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 6/20\n",
            " - 194s - loss: 0.6623 - acc: 0.6314 - val_loss: 0.6904 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 7/20\n",
            " - 192s - loss: 0.6632 - acc: 0.6301 - val_loss: 0.6709 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 8/20\n",
            " - 192s - loss: 0.6620 - acc: 0.6315 - val_loss: 0.6601 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 9/20\n",
            " - 195s - loss: 0.6616 - acc: 0.6326 - val_loss: 0.6606 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 10/20\n",
            " - 193s - loss: 0.6611 - acc: 0.6326 - val_loss: 0.6590 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 11/20\n",
            " - 198s - loss: 0.6598 - acc: 0.6328 - val_loss: 0.6595 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 12/20\n",
            " - 194s - loss: 0.6596 - acc: 0.6328 - val_loss: 0.6780 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 13/20\n",
            " - 194s - loss: 0.6594 - acc: 0.6328 - val_loss: 0.6607 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 14/20\n",
            " - 192s - loss: 0.6590 - acc: 0.6328 - val_loss: 0.6594 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 15/20\n",
            " - 192s - loss: 0.6590 - acc: 0.6328 - val_loss: 0.6590 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 16/20\n",
            " - 193s - loss: 0.6594 - acc: 0.6328 - val_loss: 0.6598 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 17/20\n",
            " - 200s - loss: 0.6586 - acc: 0.6328 - val_loss: 0.6591 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 18/20\n",
            " - 194s - loss: 0.6585 - acc: 0.6328 - val_loss: 0.6599 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 19/20\n",
            " - 194s - loss: 0.6584 - acc: 0.6328 - val_loss: 0.6604 - val_acc: 0.6299\n",
            "\n",
            "[]\n",
            "Epoch 20/20\n",
            " - 195s - loss: 0.6587 - acc: 0.6328 - val_loss: 0.6599 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Best validation accuracy of eval run:\n",
            "0.6300293817832944\n",
            "Train on 135437 samples, validate on 66708 samples\n",
            "Epoch 1/20\n",
            " - 202s - loss: 0.6607 - acc: 0.6325 - val_loss: 0.6591 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 2/20\n",
            " - 198s - loss: 0.6592 - acc: 0.6328 - val_loss: 0.6620 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 3/20\n",
            " - 205s - loss: 0.6586 - acc: 0.6328 - val_loss: 0.6595 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 4/20\n",
            " - 198s - loss: 0.6584 - acc: 0.6328 - val_loss: 0.6594 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 5/20\n",
            " - 197s - loss: 0.6592 - acc: 0.6328 - val_loss: 0.6648 - val_acc: 0.6300\n",
            "\n",
            "[]\n",
            "Epoch 6/20\n",
            " 10%|█         | 1/10 [1:22:00<9:46:17, 3908.65s/it, best loss: -0.6300293817832944]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-45-422288981881>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m                           \u001b[0malgo\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtpe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msuggest\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m                           \u001b[0mmax_evals\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m                            trials=model2_run2_trials)\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mfmin\u001b[0;34m(fn, space, algo, max_evals, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin, points_to_evaluate, max_queue_len, show_progressbar)\u001b[0m\n\u001b[1;32m    386\u001b[0m             \u001b[0mcatch_eval_exceptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcatch_eval_exceptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    387\u001b[0m             \u001b[0mreturn_argmin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreturn_argmin\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 388\u001b[0;31m             \u001b[0mshow_progressbar\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshow_progressbar\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    389\u001b[0m         )\n\u001b[1;32m    390\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/base.py\u001b[0m in \u001b[0;36mfmin\u001b[0;34m(self, fn, space, algo, max_evals, rstate, verbose, pass_expr_memo_ctrl, catch_eval_exceptions, return_argmin, show_progressbar)\u001b[0m\n\u001b[1;32m    637\u001b[0m             \u001b[0mcatch_eval_exceptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcatch_eval_exceptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    638\u001b[0m             \u001b[0mreturn_argmin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreturn_argmin\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 639\u001b[0;31m             show_progressbar=show_progressbar)\n\u001b[0m\u001b[1;32m    640\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    641\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mfmin\u001b[0;34m(fn, space, algo, max_evals, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin, points_to_evaluate, max_queue_len, show_progressbar)\u001b[0m\n\u001b[1;32m    405\u001b[0m                     show_progressbar=show_progressbar)\n\u001b[1;32m    406\u001b[0m     \u001b[0mrval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcatch_eval_exceptions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcatch_eval_exceptions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 407\u001b[0;31m     \u001b[0mrval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexhaust\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    408\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mreturn_argmin\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    409\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mtrials\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmin\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mexhaust\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    260\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mexhaust\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    261\u001b[0m         \u001b[0mn_done\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 262\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_evals\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mn_done\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mblock_until_done\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masynchronous\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    263\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrefresh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    264\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, N, block_until_done)\u001b[0m\n\u001b[1;32m    225\u001b[0m                     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    226\u001b[0m                         \u001b[0;31m# -- loop over trials and do the jobs directly\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 227\u001b[0;31m                         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mserial_evaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    228\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    229\u001b[0m                     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mserial_evaluate\u001b[0;34m(self, N)\u001b[0m\n\u001b[1;32m    139\u001b[0m                 \u001b[0mctrl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbase\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCtrl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcurrent_trial\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 141\u001b[0;31m                     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdomain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspec\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mctrl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    142\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    143\u001b[0m                     \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'job exception: %s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/hyperopt/base.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, config, ctrl, attach_attachments)\u001b[0m\n\u001b[1;32m    842\u001b[0m                 \u001b[0mmemo\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmemo\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    843\u001b[0m                 print_node_on_error=self.rec_eval_print_node_on_error)\n\u001b[0;32m--> 844\u001b[0;31m             \u001b[0mrval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpyll_rval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    845\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    846\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumber\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-43-8c6f309068e4>\u001b[0m in \u001b[0;36mopt_deep_fn\u001b[0;34m(params)\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m     result = model.fit(x_train, y_train.values, batch_size=128, epochs=20,\n\u001b[0;32m---> 15\u001b[0;31m       validation_split=0.33, callbacks=[early_stopping, learn_rate,model_checkpoint,print_weights],verbose=2)\n\u001b[0m\u001b[1;32m     16\u001b[0m     \u001b[0mvalidation_acc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mamax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'val_acc'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Best validation accuracy of eval run:'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_acc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m   1176\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1177\u001b[0m                                         \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_steps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1178\u001b[0;31m                                         validation_freq=validation_freq)\n\u001b[0m\u001b[1;32m   1179\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1180\u001b[0m     def evaluate(self,\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, fit_function, fit_inputs, out_labels, batch_size, epochs, verbose, callbacks, val_function, val_inputs, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps, validation_freq)\u001b[0m\n\u001b[1;32m    202\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 204\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfit_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    205\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    206\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2977\u001b[0m                     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2978\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2979\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2980\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2981\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2935\u001b[0m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2936\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2937\u001b[0;31m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2938\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2939\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1470\u001b[0m         ret = tf_session.TF_SessionRunCallable(self._session._session,\n\u001b[1;32m   1471\u001b[0m                                                \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1472\u001b[0;31m                                                run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1473\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1474\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "raLqtygJpGmX",
        "colab_type": "code",
        "outputId": "dd1e883f-08d8-4b96-ab8e-5932a6afb609",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 163
        }
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-32-72cef00d1cc6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mweights\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mbiases\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6u_R0WMfU76a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "checkpointer = ModelCheckpoint(filepath='keras_weights.hdf5', \n",
        "                                   verbose=1, \n",
        "                                   save_best_only=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B4bjm5rSUrEY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(\"Best performing model hyper-parameters:\")\n",
        "print(model2_run2_best)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DlYf6H1WX3Q_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "K.clear_session()\n",
        "m2 = build_hier_model(vectors=w2v, optimizer='rmsprop', \n",
        "                             learn_rate=0.0, dropout_rate1=  , \n",
        "                             dropout_rate2=  , max_length=50, projected_dim=200, \n",
        "                             num_classes=1, num_hidden=200)\n",
        "#early_stopping = EarlyStopping(monitor='val_loss', patience=4)\n",
        "m2.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pEARDnp2X3Ww",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn_rate = LearningRateScheduler(model2_exp_decay(  ,  ))\n",
        "result = m2.fit(x_train, y_train, batch_size= 128, epochs=40,\n",
        "  validation_split=0.1, callbacks=[learn_rate])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X8CwkEQ6Sbx9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SIvAGBIgSb7V",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}