as2-model-vgg-input-features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.6 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.7\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import os,shutil\n",
    "import numpy as np\n",
    "import cv2\n",
    "import pickle\n",
    "import pandas as pd\n",
    "\n",
    "from tqdm import tqdm_notebook\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Conv2D, Flatten, BatchNormalization, MaxPooling2D, Dropout, Activation\n",
    "from keras.layers.core import Flatten, Dense, Dropout\n",
    "from keras.layers.convolutional import Convolution2D, MaxPooling2D, ZeroPadding2D\n",
    "from keras.initializers import glorot_normal\n",
    "from keras.optimizers import Adam,  SGD\n",
    "from keras.losses import binary_crossentropy\n",
    "from keras.utils import to_categorical\n",
    "from keras.applications.vgg19 import VGG19\n",
    "from keras.preprocessing import image\n",
    "from keras.applications.vgg19 import preprocess_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import load_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "cur_dir = os.getcwd()\n",
    "\n",
    "dataset_dir = os.path.join(cur_dir,\"Dataset\")\n",
    "\n",
    "labels_dir = os.path.join(dataset_dir,\"labels\")\n",
    "frames_dir = os.path.join(dataset_dir,\"frames\")\n",
    "\n",
    "n_lecs = len(os.listdir(labels_dir))\n",
    "\n",
    "listOfLectures = []\n",
    "\n",
    "for lec in sorted(os.listdir(frames_dir)):\n",
    "    if lec.startswith('.'):\n",
    "        continue\n",
    "    frame_list = []\n",
    "    lec_path = os.path.join(frames_dir,lec)\n",
    "    for frame in sorted(os.listdir(lec_path)):\n",
    "        frame_list.append([os.path.join(lec_path,frame),])\n",
    "    listOfLectures.append(frame_list)\n",
    "\n",
    "\n",
    "lec_num = 0\n",
    "for csv in sorted(os.listdir(labels_dir)):\n",
    "    csv_path = os.path.join(labels_dir,csv)\n",
    "    with open(csv_path,'r') as input:\n",
    "        data = input.read()\n",
    "        data = data.split()\n",
    "        for i in range(len(data)):\n",
    "            listOfLectures[lec_num][i].append(int(data[i]))\n",
    "    lec_num+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, None, None, 3)     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, None, None, 64)    1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, None, None, 64)    36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, None, None, 64)    0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, None, None, 128)   73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, None, None, 128)   147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, None, None, 128)   0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, None, None, 256)   295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, None, None, 256)   590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, None, None, 256)   590080    \n",
      "_________________________________________________________________\n",
      "block3_conv4 (Conv2D)        (None, None, None, 256)   590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, None, None, 256)   0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, None, None, 512)   1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv4 (Conv2D)        (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, None, None, 512)   0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv4 (Conv2D)        (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, None, None, 512)   0         \n",
      "=================================================================\n",
      "Total params: 20,024,384\n",
      "Trainable params: 20,024,384\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = VGG19(weights='imagenet', include_top=False)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_________________________________________________________________\n",
    "Layer (type)                 Output Shape              Param #   \n",
    "=================================================================\n",
    "input_1 (InputLayer)         (None, None, None, 3)     0         \n",
    "_________________________________________________________________\n",
    "block1_conv1 (Conv2D)        (None, None, None, 64)    1792      \n",
    "_________________________________________________________________\n",
    "block1_conv2 (Conv2D)        (None, None, None, 64)    36928     \n",
    "_________________________________________________________________\n",
    "block1_pool (MaxPooling2D)   (None, None, None, 64)    0         \n",
    "_________________________________________________________________\n",
    "block2_conv1 (Conv2D)        (None, None, None, 128)   73856     \n",
    "_________________________________________________________________\n",
    "block2_conv2 (Conv2D)        (None, None, None, 128)   147584    \n",
    "_________________________________________________________________\n",
    "block2_pool (MaxPooling2D)   (None, None, None, 128)   0         \n",
    "_________________________________________________________________\n",
    "block3_conv1 (Conv2D)        (None, None, None, 256)   295168    \n",
    "_________________________________________________________________\n",
    "block3_conv2 (Conv2D)        (None, None, None, 256)   590080    \n",
    "_________________________________________________________________\n",
    "block3_conv3 (Conv2D)        (None, None, None, 256)   590080    \n",
    "_________________________________________________________________\n",
    "block3_conv4 (Conv2D)        (None, None, None, 256)   590080    \n",
    "_________________________________________________________________\n",
    "block3_pool (MaxPooling2D)   (None, None, None, 256)   0         \n",
    "_________________________________________________________________\n",
    "block4_conv1 (Conv2D)        (None, None, None, 512)   1180160   \n",
    "_________________________________________________________________\n",
    "block4_conv2 (Conv2D)        (None, None, None, 512)   2359808   \n",
    "_________________________________________________________________\n",
    "block4_conv3 (Conv2D)        (None, None, None, 512)   2359808   \n",
    "_________________________________________________________________\n",
    "block4_conv4 (Conv2D)        (None, None, None, 512)   2359808   \n",
    "_________________________________________________________________\n",
    "block4_pool (MaxPooling2D)   (None, None, None, 512)   0         \n",
    "_________________________________________________________________\n",
    "block5_conv1 (Conv2D)        (None, None, None, 512)   2359808   \n",
    "_________________________________________________________________\n",
    "block5_conv2 (Conv2D)        (None, None, None, 512)   2359808   \n",
    "_________________________________________________________________\n",
    "block5_conv3 (Conv2D)        (None, None, None, 512)   2359808   \n",
    "_________________________________________________________________\n",
    "block5_conv4 (Conv2D)        (None, None, None, 512)   2359808   \n",
    "_________________________________________________________________\n",
    "block5_pool (MaxPooling2D)   (None, None, None, 512)   0         \n",
    "=================================================================\n",
    "Total params: 20,024,384\n",
    "Trainable params: 20,024,384\n",
    "Non-trainable params: 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for heavy model architectures, .h5 file is unsupported.\n",
    "weigh = model.get_weights()\n",
    "pklfile= \"model.pkl\"\n",
    "fpkl= open(pklfile, 'wb')    #Python 3     \n",
    "pickle.dump(weigh, fpkl, protocol= pickle.HIGHEST_PROTOCOL)\n",
    "fpkl.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_model = VGG19(weights=None, include_top=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load('vgg-full.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Unknown layer:name",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-0173f2c62ef3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'f1_0.8.h5'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/engine/saving.py\u001b[0m in \u001b[0;36mload_model\u001b[0;34m(filepath, custom_objects, compile)\u001b[0m\n\u001b[1;32m    259\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'No model found in config file.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    260\u001b[0m         \u001b[0mmodel_config\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_config\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'utf-8'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 261\u001b[0;31m         \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_from_config\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_config\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcustom_objects\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcustom_objects\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    262\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m         \u001b[0;31m# set weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/engine/saving.py\u001b[0m in \u001b[0;36mmodel_from_config\u001b[0;34m(config, custom_objects)\u001b[0m\n\u001b[1;32m    333\u001b[0m                         '`Sequential.from_config(config)`?')\n\u001b[1;32m    334\u001b[0m     \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdeserialize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 335\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mdeserialize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcustom_objects\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcustom_objects\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    336\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    337\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/layers/__init__.py\u001b[0m in \u001b[0;36mdeserialize\u001b[0;34m(config, custom_objects)\u001b[0m\n\u001b[1;32m     53\u001b[0m                                     \u001b[0mmodule_objects\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mglobs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m                                     \u001b[0mcustom_objects\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcustom_objects\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m                                     printable_module_name='layer')\n\u001b[0m",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/utils/generic_utils.py\u001b[0m in \u001b[0;36mdeserialize_keras_object\u001b[0;34m(identifier, module_objects, custom_objects, printable_module_name)\u001b[0m\n\u001b[1;32m    143\u001b[0m                     \u001b[0mconfig\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'config'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m                     custom_objects=dict(list(_GLOBAL_CUSTOM_OBJECTS.items()) +\n\u001b[0;32m--> 145\u001b[0;31m                                         list(custom_objects.items())))\n\u001b[0m\u001b[1;32m    146\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mCustomObjectScope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcustom_objects\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_config\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'config'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/engine/sequential.py\u001b[0m in \u001b[0;36mfrom_config\u001b[0;34m(cls, config, custom_objects)\u001b[0m\n\u001b[1;32m    290\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mconf\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    291\u001b[0m             layer = layer_module.deserialize(conf,\n\u001b[0;32m--> 292\u001b[0;31m                                              custom_objects=custom_objects)\n\u001b[0m\u001b[1;32m    293\u001b[0m             \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlayer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    294\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/layers/__init__.py\u001b[0m in \u001b[0;36mdeserialize\u001b[0;34m(config, custom_objects)\u001b[0m\n\u001b[1;32m     53\u001b[0m                                     \u001b[0mmodule_objects\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mglobs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m                                     \u001b[0mcustom_objects\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcustom_objects\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m                                     printable_module_name='layer')\n\u001b[0m",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/utils/generic_utils.py\u001b[0m in \u001b[0;36mdeserialize_keras_object\u001b[0;34m(identifier, module_objects, custom_objects, printable_module_name)\u001b[0m\n\u001b[1;32m    163\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mfn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m                 raise ValueError('Unknown ' + printable_module_name +\n\u001b[0;32m--> 165\u001b[0;31m                                  ':' + function_name)\n\u001b[0m\u001b[1;32m    166\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Unknown layer:name"
     ]
    }
   ],
   "source": [
    "model = load_model('f1_0.8.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_model = Model(inputs=model.input, outputs=model.get_layer('block3_pool').output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_img_ft(img_path):\n",
    "    img = image.load_img(img_path)\n",
    "    x = image.img_to_array(img)\n",
    "    x = np.expand_dims(x, axis=0)\n",
    "    x = preprocess_input(x)\n",
    "    features = feature_model.predict(x)\n",
    "    features = np.squeeze(features)\n",
    "    return features "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Window size is the size of window on either side. Using window_size = 2 will cause 5 frames to be stacked for one input example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def load_data_fast(listOfLectures, resolution, window_size):\n",
    "def load_data_fast(listOfLectures, window_size):\n",
    "    X_train = []\n",
    "    Y_train = []\n",
    "    for i in tqdm_notebook(range(len(listOfLectures))):\n",
    "        im_saved = []\n",
    "        idx = 0 \n",
    "        for j in tqdm_notebook(range(window_size,len(listOfLectures[i])-window_size)):\n",
    "            if idx>10:\n",
    "                break\n",
    "            if j == window_size:\n",
    "#                 im_saved = [cv2.resize(cv2.imread(listOfLectures[i][j+k][0], 0), resolution) \n",
    "#                        for k in range(-window_size, window_size+1)]\n",
    "                im_saved = [get_img_ft(listOfLectures[i][j+k][0]) \n",
    "                       for k in range(-window_size, window_size+1)]\n",
    "            else:\n",
    "                ims = im_saved[1:]\n",
    "                ims.append(get_img_ft(listOfLectures[i][j+window_size][0]))\n",
    "                im_saved = ims\n",
    "            x = np.stack(im_saved,axis=3)\n",
    "            x = x.reshape(x.shape[0],x.shape[1],x.shape[2]*x.shape[3])\n",
    "            X_train.append(x)\n",
    "            Y_train.append(listOfLectures[i][j][1])\n",
    "            idx+=1\n",
    "        break\n",
    "    X_train = np.array(X_train)\n",
    "    Y_train = np.array(Y_train)\n",
    "    return X_train, Y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a2a25e48c78423a813234ef1a98b670",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=35), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0824b386a7a94b63b46fe94a7f2b4d7d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=523), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "resolution = (60,80)\n",
    "window_size = 2\n",
    "# X, Y = load_data_fast(listOfLectures, resolution, window_size)\n",
    "X_train, Y_train = load_data_fast(listOfLectures, window_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30, 40, 1280)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_shape = X_train[0].shape\n",
    "input_shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./XY_train_80_60_5.pkl\", \"wb\") as pickle_in:\n",
    "    pickle.dump((X_train, Y_train), pickle_in)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(\"./XY_train_80_60_5.pkl\", \"rb\") as pickle_in:\n",
    "#     X_train, Y_train = pickle.load(pickle_in)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_train = to_categorical(Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11, 2)\n",
      "(11, 30, 40, 1280)\n"
     ]
    }
   ],
   "source": [
    "print(Y_train.shape)\n",
    "print(X_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, Y_train, Y_test = train_test_split(X_train, Y_train, test_size=0.1, random_state=42, stratify=Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(X_train.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'Y_train' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-8d398b765086>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY_train\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'Y_train' is not defined"
     ]
    }
   ],
   "source": [
    "np.sum(Y_train[:, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import resample\n",
    "X_train_pos = X_train[Y_train[:, 1] == 1]\n",
    "Y_train_pos = Y_train[Y_train[:, 1] == 1]\n",
    "X_train_ups, Y_train_ups =  resample(X_train_pos, Y_train_pos, \n",
    "                                 replace=True,     # sample with replacement\n",
    "                                 n_samples=25117,    # to match majority class\n",
    "                                 random_state=123)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_final = np.concatenate((X_train[Y_train[:, 1] == 0], X_train_ups), axis=0)\n",
    "Y_train_final = np.concatenate((Y_train[Y_train[:, 1] == 0], Y_train_ups), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(X_train_final.shape)\n",
    "print(Y_train_final.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 5\n",
    "num_classes = 2\n",
    "# batch_size = 100\n",
    "learning_rate = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_model(): # (30, 40, 1280)\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(64, kernel_size=3, activation='relu', input_shape=(60,80,3), padding='valid'))\n",
    "    model.add(Conv2D(64, kernel_size=3, activation='relu', padding='valid'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2), strides=None, padding='valid'))\n",
    "    \n",
    "    model.add(Conv2D(32, kernel_size=3, activation='relu'))\n",
    "    model.add(Conv2D(32, kernel_size=3, activation='relu'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2), strides=None, padding='valid'))\n",
    "    \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(2, activation='softmax'))\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = gen_model(input_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optim = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, amsgrad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='binary_crossentropy', optimizer=optim, metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "history = model.fit(X_train_final, Y_train_final, epochs=5, verbose=1, validation_data=(X_test, Y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = model.predict(X_test)\n",
    "preds = np.argmax(preds, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(np.argmax(Y_test, axis=1), preds))\n",
    "print(f1_score(np.argmax(Y_test, axis=1), preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('vgg_b3_pool.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0b3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}