CNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "author=\"Jumabek Alikhanov\"\n",
    "date = 'Nov 22,2019'"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset can be downloaded from https://www.unb.ca/cic/datasets/ids-2017.html or you can directly below shell\n",
    "You need to just unzip and put it same directory with this CNN.ipynb file"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download dataset directly and unzip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!gdown --id 1-t3RdDpmqMs4ABt9oobSapeNYTZJ9tpu\n",
    "!unzip MachineLearningCSV.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from os.path import join\n",
    "import glob\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "from preprocessing import balance_data, normalize,load_data\n",
    "%load_ext autoreload"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set dataset path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "dataroot = 'MachineLearningCVE/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MachineLearningCVE/*.pcap_ISCX.csv\n",
      "there are 2830743 flow records with 79 feature dimension\n",
      "stripped column names\n",
      "dropped bad columns\n",
      "There are 0 nan entries\n",
      "converted to numeric\n"
     ]
    }
   ],
   "source": [
    "%autoreload 2\n",
    "X,y = load_data(dataroot) # reads csv file and returns np array of X,y -> of shape (N,D) and (N,1)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Imbalance\n",
    "1. It is curucial to adress this issue in order to get decent performance\n",
    "2. It also affects evaluation, we should calculate  `balanced accuracy`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "X = normalize(X)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set random seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED = 42"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### X and y dataset shapes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2830743, 76)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(X.shape)\n",
    "print(y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "%autoreload 2\n",
    "from models import Classifier\n",
    "\n",
    "def ensure_dir(dir_path):\n",
    "    if not os.path.exists(dir_path):\n",
    "        os.makedirs(dir_path)\n",
    "\n",
    "def getClassifier(args,runs_dir=None):\n",
    "    \n",
    "    (method,optim,lr,reg,batch_size,input_dim,num_class,num_epochs) = args\n",
    "    if runs_dir is not None:\n",
    "        ensure_dir(runs_dir)\n",
    "    \n",
    "    clf = Classifier(method,input_dim,num_class,lr=lr,reg=reg,num_epochs=num_epochs,\n",
    "                        batch_size=batch_size,runs_dir=runs_dir)\n",
    "    return clf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------\n",
      "Fold #0\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "from sklearn import metrics\n",
    "K=5\n",
    "skf = StratifiedKFold(n_splits=K,random_state=SEED)\n",
    "for fold_index, (train_index,test_index) in enumerate(skf.split(X,y)):# runs only once \n",
    "        print('---------------------------------------------')\n",
    "        print('Fold #{}'.format(fold_index))    \n",
    "        X_train = X[train_index]\n",
    "        y_train = y[train_index]\n",
    "        X_test = X[test_index]\n",
    "        y_test = y[test_index]\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best epoch 0, best batch 0\n",
      "bst acc  -1\n",
      "Epoch [1/60], Step [50/398], Loss: 0.2503\n",
      "Epoch [1/60], Step [100/398], Loss: 0.2391\n",
      "Epoch [1/60], Step [150/398], Loss: 0.2070\n",
      "Epoch [1/60], Step [200/398], Loss: 0.1962\n",
      "Epoch [1/60], Step [250/398], Loss: 0.1917\n",
      "Epoch [1/60], Step [300/398], Loss: 0.1778\n",
      "Epoch [1/60], Step [350/398], Loss: 0.1740\n",
      "Epoch [2/60], Step [1/398], Loss: 0.1879\n",
      "Epoch [2/60], Step [51/398], Loss: 0.1908\n",
      "Epoch [2/60], Step [101/398], Loss: 0.1637\n",
      "Epoch [2/60], Step [151/398], Loss: 0.1576\n",
      "Epoch [2/60], Step [201/398], Loss: 0.1688\n",
      "Epoch [2/60], Step [251/398], Loss: 0.1634\n",
      "Epoch [2/60], Step [301/398], Loss: 0.1684\n",
      "Epoch [2/60], Step [351/398], Loss: 0.1641\n",
      "Epoch [3/60], Step [2/398], Loss: 0.1745\n",
      "Epoch [3/60], Step [52/398], Loss: 0.1554\n",
      "Epoch [3/60], Step [102/398], Loss: 0.1523\n",
      "Epoch [3/60], Step [152/398], Loss: 0.1639\n",
      "Epoch [3/60], Step [202/398], Loss: 0.1669\n",
      "Epoch [3/60], Step [252/398], Loss: 0.1681\n",
      "Epoch [3/60], Step [302/398], Loss: 0.1625\n",
      "Epoch [3/60], Step [352/398], Loss: 0.1552\n",
      "Epoch [4/60], Step [3/398], Loss: 0.1509\n",
      "Epoch [4/60], Step [53/398], Loss: 0.1496\n",
      "Epoch [4/60], Step [103/398], Loss: 0.1517\n",
      "Epoch [4/60], Step [153/398], Loss: 0.1449\n",
      "Epoch [4/60], Step [203/398], Loss: 0.1522\n",
      "Epoch [4/60], Step [253/398], Loss: 0.1496\n",
      "Epoch [4/60], Step [303/398], Loss: 0.1406\n",
      "Epoch [4/60], Step [353/398], Loss: 0.1499\n",
      "Epoch [5/60], Step [4/398], Loss: 0.1574\n",
      "Epoch [5/60], Step [54/398], Loss: 0.1418\n",
      "Epoch [5/60], Step [104/398], Loss: 0.1687\n",
      "Epoch [5/60], Step [154/398], Loss: 0.1498\n",
      "Epoch [5/60], Step [204/398], Loss: 0.1465\n",
      "Epoch [5/60], Step [254/398], Loss: 0.1405\n",
      "Epoch [5/60], Step [304/398], Loss: 0.1391\n",
      "Epoch [5/60], Step [354/398], Loss: 0.1439\n",
      "Epoch [6/60], Step [5/398], Loss: 0.1492\n",
      "Epoch [6/60], Step [55/398], Loss: 0.1443\n",
      "Epoch [6/60], Step [105/398], Loss: 0.1835\n",
      "Epoch [6/60], Step [155/398], Loss: 0.1302\n",
      "Epoch [6/60], Step [205/398], Loss: 0.1634\n",
      "Epoch [6/60], Step [255/398], Loss: 0.1469\n",
      "Epoch [6/60], Step [305/398], Loss: 0.1475\n",
      "Epoch [6/60], Step [355/398], Loss: 0.1280\n",
      "Epoch [7/60], Step [6/398], Loss: 0.1444\n",
      "Epoch [7/60], Step [56/398], Loss: 0.1248\n",
      "Epoch [7/60], Step [106/398], Loss: 0.1409\n",
      "Epoch [7/60], Step [156/398], Loss: 0.1470\n",
      "Epoch [7/60], Step [206/398], Loss: 0.1211\n",
      "no improvement in accuracy for 10 iterations\n",
      "Loaded MachineLearningCVE/runs/1D-CNN_conv_5_fc_1/5th_run/optim_Adam_lr_0.001_reg_0.001_bs_5120 model trained with batch_size = 5120, seen 5 epochs and 154 mini batches\n",
      "Model is trained in 1271.47327542305 sec\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#hyper-params\n",
    "batch_size = 5*1024 # increasing batch size with more gpu added\n",
    "optim = 'Adam'\n",
    "\n",
    "input_dim = X_train.shape[1]\n",
    "num_class = len(np.unique(y_train))\n",
    "num_epochs = 60\n",
    "learning_rates = [1e-3]\n",
    "regularizations = [1e-3]\n",
    "\n",
    "\n",
    "accuracies = {}\n",
    "best_model = None\n",
    "best_acc = -1\n",
    "architecture = '1D-CNN_conv_5_fc_1'\n",
    "run_number = 5\n",
    "method='cnn5'\n",
    "\n",
    "for lr in learning_rates:\n",
    "    for reg in regularizations:\n",
    "\n",
    "        classifier_args = (method,optim,lr,reg,batch_size,input_dim,num_class,num_epochs)\n",
    "        config =  '{}/{}th_run/optim_{}_lr_{}_reg_{}_bs_{}'.format(architecture,run_number,optim,lr,reg,batch_size)\n",
    "        runs_dir = join(dataroot,'runs',config)\n",
    "        \n",
    "        X_train = X_train.astype(float)\n",
    "        y_train = y_train.astype(int)\n",
    "        p = np.random.permutation(len(y_train))\n",
    "        X_train = X_train[p]\n",
    "        y_train = y_train[p]\n",
    "        X_train,y_train = balance_data(X_train,y_train,seed=SEED)\n",
    "\n",
    "        tick = time.time()\n",
    "        clf = getClassifier(classifier_args,runs_dir)\n",
    "        \n",
    "        clf.fit(X_train,y_train)\n",
    "        pred = clf.predict(X_test,eval_mode=True)\n",
    "        \n",
    "        acc = metrics.balanced_accuracy_score(y_test,pred)\n",
    "        if acc >best_acc:\n",
    "            best_model = clf\n",
    "            best_acc = acc\n",
    "        accuracies[(lr,reg)]=acc\n",
    "        tock = time.time()\n",
    "        print(\"Model is trained in {} sec\".format(tock-tick))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.001, 0.001)\n",
      "0.8329863280999342\n"
     ]
    }
   ],
   "source": [
    "# accuracies for CV\n",
    "for x in accuracies:\n",
    "    print(x)\n",
    "    print(accuracies[x])\n",
    "results = accuracies  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEWCAYAAABi5jCmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3defxd073/8dc7ERSRIKmLJKihqKJtSmkNrZZwtWlpiw5oDbcDbXW6XL2ooVXaanvpkKJK1dwhNEW0aOsnJDSGBBVqijGNSIKS4f37Y60j+3ucYef7Pec7nHyej8d+ZA9r7/05J+e7195rrb2WbBNCCCFUDOrrAEIIIfQvkTGEEELoIjKGEEIIXUTGEEIIoYvIGEIIIXQRGUMIIYQuImNYDpIWSnpDP4hjZ0n3t/B4F0v6YKuOtyKTdKWkvdp07PMlnZLnG/4Gimm7ea5+8VsPfaNjMgZJD0t6RtLqhXWHSbqx5P5N/5Bsr2H7oZLHs6RNy6RdXrb/avuNrTiWpG2AbYHfF9atJ+lcSU9KWiDpPknfrHy31Z9N0uaSLpc0R9Lzku6S9GVJg+uc80ZJh+X53SQtzReihZIel3SZpLdX7TNe0nRJ8/N5/ixp4zrH31zS7yU9K2mupGslvbEqzdGSnsrHO0/SKoVtJ0u6W9JiSSdW7fc/hVgXSnopxz8iJ/kO0O0Lclkt/g28+v9ROH7p33roPB2TMWSDgS/2dRBlSFqpr2PI/gu4yPlNR0lrA7cArwN2tD0UeB8wHNikemdJmwC3Ao8Bb7Y9DPgIMBYYWjKGJ2yvkdO/A7gP+Kuk3fM5NgUuAL4CDAM2Bs4GltQ53nBgIvBGYF3gNrpmfHsCxwC7AxsCbwC+Wdh/FvB14A/VB7b9rXzRXCPH/B3gRttz8vbbgDUljS352UMvUdJp17z2sN0RE/Aw6Y99LjA8rzuM9EdbSbMFMDmnuR/4aF5/BLAIeAVYCFxV5xwGNs3z55MuTn8AFpAujpvkbX/JaV/Ix9sf2A14HPhv4CngQuAQ4G8NzrE3MDMffzbw1bx+N+Dxwj5bAjcC84AZwAcK2+rGmbc/BLyrsHwKcDcwqMF3XYzxV8AflvP/6kbgsFqfpZDmLGBanv8wML0Hv421c8zr5OVfA98qbN8deKrGfr8CTmxwXOXv7+Cq9T8HTqizz73APoXllYBngbfm5cvz7+P5/Dt6U9X/5Sl1fgNvAe7I/8eXApcU0q4FXJ3P81yeH5W3nUrKYP+df6tn1fg/HkbKmJ8FHgG+Ufl9kH/DwHfzsf8J7NXgOzsGeDDHORP4UNX2w/N3VNle+V5GA7/JMfyrEOeJwK8K+2+UY1+p8Fs7FbgZeAnYFPhU4RwPAf9VFcN4YDowP8c6jnSzc3tVui8Dv+/u77I/T52We04j/RC+Wr0hF4NMJl0UXg8cAPxY0la2JwAXAac73Qm+v+T5DiDdaa5Fuss8FcD2Lnn7tvl4l+bl/yBdpDYkZUbNnEv60Q4Ftgb+XONzDQGuAq7Ln+so4KKqopOacebvZGNSJlnxXuA3tpeWiK+S/oqSaZfHb4C35hjvALaQdKakd0taYzmPtQvpwv+vvPwm4M7C9juBdSWts5zH3Zn0nV9Ztf5eUvFcLRcDBxaW9wTm2L4jL/8R2Cwf9w7S77IhSSsDvyPdbKxNylz2KyQZBPyC9LsbQ7pAngVg+zjgr8CR+bd6ZI1T/B8pc3gDsCtwEOniWrED6Tc0AjgdOFeS6oT7IOl7G0b6Tf5K0nr5c3yEdKE/CFgT+ADwr1wkeTUpU9oI2ICU8ZX1SdLf29B8jGeAffI5PgWcKemtOYbtSZng10hPnruQbjonAhtL2rLquBcsRxwDRqdlDADHA0dJGlm1fh/gYdu/sL3Y9t9Jf9Af6cG5fmv7NtuLSX/A2zVJv5R0J/my7ZdKHH8RsJWkNW0/V7h4FL0DWAM4zfYrtv9M+iMqXnzqxTk8/7ugkHYd4MkSsXU3fVlPkO7IhzuVde9GuiBcBszJdUJNMwhJo0hPTF8urF6DdEdeUZkvW/RVcTBwhe2FVesXsOy7rfZr4AOSVsvLHyNlFgDYPs/2Atsvky6S20oa1iSOdwBDgB/YXmT7CmBq4Zj/sn2l7RdtLyDdGOxa5gPmi/IBwLE5roeB75EuihWP2P657SXAL4H1SEV4r2H7cttP2F6ab5geALbPmw8j3ZxNdTLL9iN5+/rA12y/YPvftv9WJv7sfNsz8t/9Itt/sP1gPsdNpJuqnXPaQ4HzbE/OMc62fV/+/7gU+ET+Xt5EyqSuXo44BoyOyxhs30P6zzqmatOGwA6S5lUm4OOku/jueqow/yLpgtPIs7b/vRzH349UnPSIpJsk7VgjzfrAY1V3+I+QLqLN4pyX/y1eEP9F+sMuq2F6ST8tVNT+z3IcdwNSkcA8ANtTbH/U9kjSH/EuwHH5HMXK4DGFc48k/dH/2PbFhWMvJN0tVlTmixlkQ/nC/hHShbDaUJZ9t13YnkV6onh/PsYHSJkFkgZLOk3Sg5Lmk+5UId2JN7I+MNt2sUfMR4qxSvqZpEfycf8CDK/XOKDKCFKm80hhXd3fl+0X82zNvwVJB+VGBJW/wa1Z9vlGk54oqo0mZT6LS8Rby2NVMewlaUpumDCP9DfWLAZI/9cfy09DnwQuyxlGx+m4jCE7gVRWWfzxPgbcZHt4YVrD9mfz9t7oZrb6HC8AlTtHJHXJpPKd03hSscLvSHfL1Z4ARldVqo0h1Uk0DsZ+gfRHsHlh9fXAh5ajku56uhZbVJ/jM15WWfutkscE+BBwR46x+phTSUVNW+flNQrTowCS1iJlChNtn1p1iBl0LerZFni6UNRUNr65pKLLalvStaiqWqU4aTwwM2cWkJ4expOK54aR7kghPTk18iSwQVXxzZjC/FdIFfE72F6TlKkWj9votz+H9OS6YdWxm/6+qknakFT/ciSpvmc4cE8hjseo0cAhrx9Tp8FGl78hat/ovfr5cuuzK0l1IuvmGCaViAHbU0j1kDuT/q8urJWuE3RkxpD/0C4FvlBYfTWwuaRPShqSp7cXygyfJpWhtkqZ490JvEnSdpJWJRUdAKncWNLHJQ2zvYhUEVar3P9W0lPA1/Nn2g14P+XLYCfRtVjh+6Q76F/mP2QkbSDp+0pNW6udAOwk6YxKxiZpU0m/klSvOKWm3GpkA0knkIoV/ievf5ekwyW9Pi9vQbrTnlLnOGsC1wI3265+coRULnyopK1yjN8gVexW9h+S/z8GAStJWrXG3fXBwAVVd+kVu5LqCuq5BNgD+Cz5aSEbCrxMegpbDSibkd4CLAa+kGPfl2XFM5XjvgTMU2p1dkLV/nV/q7l46DLgVElD82/iy6SK+eW1Ouki/SyApE+RM/fsHOCrkt6Wfwub5vPdRsr8TpO0ev7/eGfeZzqwi6Qxucjt2CYxrAyskmNYrPTOyR6F7ecCn5K0u6RB+fe4RWH7BaT6mUXLWZw1oHRkxpCdRPohApDLVvcglZc+QXr8/Q7pRwLpB7FVfsT9XQvOfyLp4jpP0kdrJbD9jxzn9aSy1uof2ieBh/Pj/2dIRV/Vx3iFlBHsRbq7+zFwkO37SsY5Afh45W7T9lxgJ9Jd4q2SFgB/IpXDz6re2faDwI6ku9sZkp4n3ZFNo3zRzPqSFpKKeKYCbwZ2s31d3j6PlBHcndNdA/yWVNFZy4eAt5P+wF9TzGT7mrzvDcCjpKKR4sXy56QL6YGk4qqXKJSpS9oAeA81Kh6V3r9Y6NRstSbbT5Iu5juRbmAqLsixzCa1yKmZ8dU43ivAvqQWQnNJreB+U0jyA1Lz4zn5mNdUHeKHwIclPSfpRzVOcRTpzvwh0m/018B5ZWKrinMmqX7iFlJm9GZSa6HK9stJ9R+/Jv12fgesnTOn95NaFD1Kat23f95nMuk7vAu4nSZl/vk68AVSZvcc6c5/YmH7beQKadJv/ia6Pi1dSMrMupMxDhiqfcMTViSSfk0qL21FhrhCk3QlcK7tSX0dS2g9Sa8jtWp6q+0H+jqedomMIYQQSpL0ZdJ7KO/p61jaqb+8fRtCCP2apIdJldQd369YPDGEEELoopMrn0MIIXTDClWUNGLECG+00UZ9HUYIYQC4/fbb5+QXKrvtbYNW93zX6+txmVm8fK3tcT05VyutUBnDRhttxLRp0/o6jBDCACDpkeapGpvvJfxgpQ2bpttn8T+avd3eq1aojCGEEHqVQEOavbhOej2xH4mMIYQQ2kSDxODXleiSqkyXmr0oMoYQQmgXwaCVSjwx9DORMYQQQruULUrqZyJjCCGENpEUTwwhhBAK4okhhBBCF1HHEEIIoUiCwSsPvA4mImMIIYS2ERoUTwwhhBAqBBocTwwhhBAyAYMGxxNDCCGEChFFSSGEEJaRFJXPIYQQutKgyBhCCCFURFFSCCGErjQgK58H3jNOCCEMEMpPDM2mcsfSOEn3S5ol6Zga2zeU9CdJd0m6UdKowraDJT2Qp4ObnSsyhhBCaCMNGtR0anoMaTBwNrAXsBVwoKStqpJ9F7jA9jbAScC3875rAycAOwDbAydIWqvR+fokY5B0hqT7cs72W0nD66T7oqR7JM2Q9KXC+u0kTZE0XdI0Sdv3XvQhhFCSxOAhg5pOJWwPzLL9kO1XgEuA8VVptgL+nOdvKGzfE5hse67t54DJQMPxpfvqiWEysHXO2f4BHFudQNLWwOGkL2RbYB9Jm+bNpwPftL0dcHxeDiGEfqWFRUkbAI8Vlh/P64ruBPbN8x8Chkpap+S+XfRJxmD7OtuVUU6nAKNqJNsSuNX2izntTSz70AbWzPPDgCfaGW8IIXRXyaKkEbn0ozId0Y1TfRXYVdLfgV2B2cCS7sTcH1olfRq4tMb6e4BTc473ErA3MC1v+xJwraTvkjK3neodPH/BRwCMGTOmhWGHEEIT5ZurzrE9tsH22cDowvKovO5Vtp8g3zxLWgPYz/Y8SbOB3ar2vbFRMG17YpB0fa4fqJ7GF9IcBywGLqre3/a9wHeA64BrgOksy/0+CxxtezRwNHBuvThsT7A91vbYkSNHtuzzhRBCc82LkUpmHFOBzSRtLGll4ABgYpczSSMkVa7pxwLn5flrgT0krZUrnffI6+pq2xOD7fc22i7pEGAfYHfbrnOMc8kXfUnfIpWNARwMfDHPXw6c04KQQwihpSQYtNLgHh/H9mJJR5Iu6IOB82zPkHQSMM32RNJTwbclGfgL8Pm871xJJ5MyF4CTbM9tdL4+KUqSNA74OrCr7RcbpHu97WckjSE9Ir0jb3qCVIZ2I/Ae4IH2RhxCCN3TqhfcbE8CJlWtO74wfwVwRZ19z2PZE0RTfVXHcBawCjBZEsAU25+RtD5wju29c7orcx3DIuDztufl9YcDP5S0EvBvch1CCCH0K4qBekqzvWmd9U+QKpkryzvXSfc34G3tiS6EEFonOtELIYTwKkUneiGEEKpFxhBCCGEZqSWtknpbZAwhhNA2ijqGEEIIVRRFSSGEELKofA4hhPAaUZQUQghhmXjBLYQQQrVolRRCCOFVUccQQgihiiDqGEIIIRQpmquGEEJ4laJVUgghhCIJReVzCCGEoqh8DiGE8Cohlg3DPHCUyhgk7QRsVExv+4I2xRRCCJ1BQIueGPKQyD8kjfl8ju3TqraPAX4JDM9pjrE9SdIQ4BzgraRr+AW2v93oXE0zBkkXApsA04ElebWByBhCCKGJVlQ+SxoMnA28D3gcmCppou2ZhWTfAC6z/RNJW5HGh94I+Aiwiu03S1oNmCnpYtsP1ztfmSeGscBWtt2tTxRCCCuwFtUxbA/Msv0QgKRLgPFAMWMwsGaeHwY8UVi/uqSVgNcBrwDzG52sTMZwD/AfwJMlP0AIIQRIrZIGl2qVNELStMLyBNsTCssbAI8Vlh8Hdqg6xonAdZKOAlYH3pvXX0HKRJ4EVgOOtj23UTB1MwZJV5FymqGkR4/bgJcr221/oNGBQwghUPbN5zm2x/bwTAcC59v+nqQdgQslbU162lgCrA+sBfxV0vWVp49aGj0xfLeHQYYQwgpNUqvefJ4NjC4sj8rrig4FxgHYvkXSqsAI4GPANbYXAc9IuplURVA3Y6ibldm+yfZNwN6V+eK6bnywEEJY8Qwa1HxqbiqwmaSNJa0MHABMrErzKLA7gKQtgVWBZ/P69+T1qwPvAO5rGHKJgN5XY91eJfYLIYQVngap6dSM7cXAkcC1wL2k1kczJJ0kqVKs/xXgcEl3AhcDh+RGQ2cDa0iaQcpgfmH7rkbna1TH8Fngc8AbJBUPMhS4ueknCSGEFV3qd7slh7I9idQEtbju+ML8TOCdNfZbSGqyWlqjOoZfA38Evg0cU1i/oFmNdgghhKRkq6R+pW7GYPt54HlJn6/eJmlIrsgIIYRQTwvffO5NZd5juINUG/4c6WMOB56S9DRwuO3b2xhfCCEMYBqQ3W6XiXgyqWXSCNvrkCqerybVP/y4ncGFEMKAJzWf+pkyGcM7bF9bWbB9HbCj7SnAKm2LLIQQBjrRquaqvapMUdKTkv4buCQv7w88nTt1Wtq2yEIIYcAr3SVGv1Imq/oY6S273+VpTF43GPho+0ILIYQBTqTmqs2mfqbpE4PtOcBRdTbPam04IYTQSdSZrZIkbQ58ldcO1POe9oUVQggDn6BjR3C7HPgpaQSgJU3SliLpDOD9pH7BHwQ+ZXtejXRfBA4nfb8/t/2DvH7bHNMawMPAx2037F88hBB63QB9j6FMVrbY9k9s32b79srUw/NOBra2vQ3wD+DY6gS5u9jDSV3GbgvsI2nTvPkc0rB1bwZ+C3yth/GEEEIbaEDWMZSJ6CpJn5O0nqS1K1NPTmr7utwpFMAUUuV2tS2BW22/mNPeBOybt20O/CXPTwb260k8IYTQNoMHN5/6mTJFSQfnf4t35Qbe0KIYPg1cWmP9PcCpktYBXiJ19V0Z4WgGaUSi35E6hxpdY38AJB0BHAEwZsyYFoUcQggltLATvd5UplXSxt05sKTrSUOCVjvO9u9zmuOAxcBFNc57r6TvANcBLwDTWVbH8WngR5L+l9Qn+SsN4p8ATAAYO3ZsjFsdQuhdA7COoUyrpNWALwNjbB8haTPgjbavbrSf7fc22i7pEGAfYPfcZ3itY5wLnJvTf4s0zim27wP2yOs3B/6z2ecIIYQ+MQCfGMpE/AvSHflOeXk2cEpPTippHPB14AO2X2yQ7vX53zGk+oVfV60fBHyD1EIphBD6nw7tK2kT26cDiwDyhbynn+Qs0oA/kyVNl/RTAEnrSyoORHGlpJnAVcDnC01aD5T0D9LwdE+QMq8QQuhfpI7tK+kVSa8jVTgjaRPg5Z6c1PamddY/QWE8ads710n3Q+CHPYkhhBB6xaDWtDrKJS0/JHVHdI7t06q2jwF+SRoaYTCpSf+kvG0b4GfAmqQ+7t5u+9/1zlUmYzgBuAYYLeki0tBxhyznZwohhBVP5Ymhx4fRYNLYze8j1bVOlTQxD+dZ8Q3SWNA/kbQVaRjQjSStBPwK+KTtO3NLz4YDrTXMGCSJVFyzL/AOUhHSF3P/SSGEEJppTR3C9sAs2w+lQ+oSUpP9YsZg0hMBwDBSMTukhjp32b4TwPa/mp2sYcZg25Im5TeM/7A8nyKEEAJlWyWNkDStsDwhN7Wv2AB4rLD8OLBD1TFOBK6TdBSwOlBpGbo5YEnXAiOBS3K9cV2lhvaU9HbbU0ukDSGE8KrSrY7m2B7bw5MdCJxv+3uSdgQuzF0LrQS8C3g78CLwJ0m32/5TvQOVyRh2AD4u6RHSi2YiPUxs08MPEUIIHc0Ct6bLi9l07eFhVF5XdCgwDsD2LZJWBUaQni7+UqkCyC0/3wr0KGPYs3ToIYQQClrWJcZUYDNJG5MyhANIA6YVPQrsDpwvaUtgVeBZ4Frg6/ll5VeAXYEzG52sTMZwiu1PFldIuhD4ZJ30IYQQKlqQMdheLOlI0kV+MHCe7RmSTgKm2Z4IfAX4uaSjSRXRh+ReJZ6T9H1S5mJgku2GdcZlMoY3FRdys6m3Le8HCyGEFZFb9GZzfidhUtW64wvzM0mvE9Ta91ekJqul1M3KJB0raQGwjaT5eVoAPAP8vuwJQghhhaUOG4/B9rdtDwXOsL1mnobaXsf2awbWCSGEUEOH9pV0taTVASR9QtL3JW3Y5rhCCKEDCA8e3HTqb8pkDD8BXszjLH+FNEbzBW2NKoQQOoHorKKkgsW5Zns8cJbts0k9o4YQQmjCGtR06m/KtEpaIOlY4BPALnkMhCHtDSuEEDpB/6xDaKZMVrU/qZvtQ20/RXrj7oy2RhVCCB2iI58Ycmbw/cLyo0QdQwghNCe1bDyG3lSmKCmEEEI3mNa94NabImMIIYR26odFRc1ExhBCCG1kOvCJQdI7SQNAbJjTV7rdfkN7QwshhIFO/bJyuZkyTwznAkcDtwNL2htOCCF0mA7NGJ63/ce2RxJCCB3GEks7tFXSDZLOAH5Dep8BANt3tC2qEELoFB3aKqky4HRxPFID72l9OCGE0Fk6so7B9rt7I5AQQug8GpCtkppmZZKG5a62p+Xpe5KG9UZwIYQw0LWqSwxJ4yTdL2mWpGNqbB8j6QZJf5d0l6S9a2xfKOmrzc5VJqLzgAXAR/M0H/hFqU8SQggrMtGSgXrykMpnA3sBWwEHStqqKtk3gMtsvwU4APhx1fbvA6UaEpWpY9jE9n6F5W9Kml7m4CGEsCIzYqla0ippe2CW7YcAJF1CGgphZpfTwZp5fhjwRGWDpA8C/wReKHOyMk8ML0l6V+EE7wReKnPwEEJY0ZUsShpRKK6fJumIqsNsADxWWH48rys6EfiEpMeBScBRAJLWAP4b+GbZmMs8MXwW+GWuVxAwFzik7AlCCGFFVrLyeY7tsc2TNXQgcL7t70naEbhQ0takDONM2wtVsulsmVZJ04FtJa2Zl+d3O+wQQlihtKxLjNnA6MLyqLyu6FBgHIDtWyStCowgvXLwYUmnA8OBpZL+bfuseiermzFI+oTtX0n6ctV68om/X3PHEEIIr2pRt9tTgc0kbUzKEA4APlaV5lFgd+B8SVsCqwLP2t65kkDSicDCRpkCNH5iWD3/W2t8Zzc6aAj93aJFi3h+3jwWLFiAbSQxdOhQhg0fzpAhMXJtaA2rNZXPthdLOhK4FhgMnGd7hqSTgGm2JwJfAX4u6WjSNfoQ2926VtfNGGz/LM9eb/vm4rZcAR3CgPTiCy/w9NNPU/ybsc38+fNZsGAB6667LqutvnqDI4RQXqtecLM9iVSpXFx3fGF+JtDw2mz7xDLnKlP49X8l1y0XSSfnlzCmS7pO0vp10h0s6YE8HVxY/zZJd+eXPX6ksrUqYYW2aNGi12QKRbZ5+umnWbRoUS9HFjpVR435nGu1dwJGVtUzrEl6lOmpM2z/bz7XF4Djgc9UxbA2cAKpnyYDt0uaaPs54CfA4cCtpFx0HCVf3ggrrufnzaubKVTY5vl58xgxcmQvRRU6Wad1ibEysAYp8xhamOYDH+7piataN61O7XqLPYHJtufmzGAyME7SesCatqfkMrQLgA/2NKbQ+RYsWNDSdCE04twqqWOeGGzfBNwk6Xzbj7Tj5JJOBQ4CngdqddZX76WODfJ89fpa5zgCOAJgzJgxPQ86DGhl6+K6WWcXwmt02hNDxYuSzpA0SdKfK1OZg0u6XtI9NabxALaPsz0auAg4sgefoy7bE2yPtT12ZBQNrPDKVkVFlVVolaUMajr1N2XefL4IuBTYh1QHcDDwbJmD235vyTguItUTnFC1fjawW2F5FHBjXj+qan31yx4hvMbQoUOZP7/5O5pDh9ZqpR3C8hLuhxf+ZspEvI7tc4FFtm+y/WlaMEiPpM0Ki+OB+2okuxbYQ9JaktYC9gCutf0kMF/SO3JrpIOA3/c0ptD5hg0f3vRpQBLDhg/vpYhCJzO5nqHJ1N+UeWKotNt7UtJ/knrsW7sF5z5N0huBpcAj5BZJksYCn7F9mO25kk4mvfUHcJLtuXn+c8D5wOtIrZGiRVJoasiQIay77rp1m6xKYt11142X3ELL9McLfzNlMoZTcgd6XyG9v7AmcHRPT1zVlXdx/TTgsMLyeaQxIWql27qncYQVz2qrr86o0aPjzefQKzo1Y7jT9vMUWg5J+o+2RhVCmw0ZMoQRI0fGuwqhzfpnUVEzZeoY/inpYkmrFdZNqps6hBACkOoYlnpQ06m/KRPR3cBfgb9J2iSvG3hZYAgh9IFOrXy27R9LuhO4StJ/E72rhhBCKf3xwt9MmYxBALZvlrQ7cBmwRVujCiGEjiDszswY9q7M2H5S0rtJneuFEEJowMDSTnpiqIzgBhxY54Wgv7QtqhBC6ASmX1YuN9PdEdxCCCGU0FF1DLZ/JmkwMN/2mb0YUwghdIjW1TFIGgf8kDQezjm2T6vaPgb4JTA8pznG9iRJ7wNOIw2l8ArwNdsNO0Jt+IxjewlwYHc/SAghrMha1VdSvkk/G9gL2IpUxL9VVbJvAJfZfgtwAPDjvH4O8H7bbyZ1gnphs/OVqXy+WdJZpB5WX6istH1HiX1DCGGF1qInhu2BWbYfApB0Canz0ZnFU5G6LAIYRurXDtt/L6SZAbxO0iq2X653sjIZw3b535OqAuhxD6shhNDplpZLNkLStMLyBNsTCsu1Bi3boeoYJwLXSTqKVEdca9iD/YA7GmUKUCJjsF1rZLUQQghNGJVtlTTH9tgenu5A4Hzb35O0I3ChpK1tLwWQ9CbgO6ThCxoq88RA7m77TcCqlXW2T6q/RwghBGhZUdJsYHRhudbgZIcC49I5fYukVYERwDOSRgG/BQ6y/WCzkzXNyiT9FNgfOIr0FvRHgA2bf44QQggt6itpKrCZpI0lrUyqXJ5YleZRYHcASVuSbuSflTQc+AOpldLNZU5W5hlnJ9sHAc/Z/iawI7B5mYOHEMIKzbC0xNT0MPZi4EjSqJb3klofzZB0kqQP5GRfAQ7P/dpdDBziNBrVkcCmwPGSpufp9Y3OV6Yo6aX874uS1gf+BaxXYr8QQlihVZqrtuRY9iSqhjywfXxhfibwzhr7nR9PzE8AABRvSURBVAKcsjznKpMxXJ0fRc4A7iB91nOW5yQhhLCi6shO9GyfnGevlHQ1sGoe0S2EEEJDYkknZQyS9m2wDdu/aU9IIYTQGUznPTG8v8E2A5ExhBBCEx6Aw5o16kTvU70ZSAghdKKO6l21QtLxtdbHC24hhNBEyeao/U2ZVkkvFOZXBfYhtaMNIYTQgIGlSzvwicH294rLkr5LeskihBBCEx01tGcDq5H66QghhNBER1U+V0i6m/REBGlUoJF07YI7hBBCDW7hCG69qcwTwz6F+cXA07nfjhBCCI10cOXzgqrlNSUtsL2oHQGFEEIn6ciiJFL/SKOB50jdbg8HnpL0NHC47dvbGF8IIQxYhgHZJUaZbrcnA3vbHmF7HdJg1FcDn2PZYNMhhBBqsJtP/U2ZjOEdtl9tnmr7OmBH21OAVdoWWQghdICBmDGUKUp6UtJ/A5fk5f2BpyUNpvQ41yGEsOKxYWmHFiV9jPTewu9IY4aOzusGAx9tX2ghhDDwDcQnhqYZg+05to8C3mX7rbaPsv2s7Vdsz+ruiSWdLOmuPMzcdXl0uFrpDpb0QJ4OLqw/VdJjkhZ2N4YQQmi3JUubT2VIGifpfkmzJB1TY/sYSTdI+nu+tu5d2HZs3u9+SXs2O1fTjEHSTpJmkvtHkrStpFZUOp9hexvb25Eqs1/TWZ+ktYETgB2A7YETJK2VN1+V14UQQr9UGY+h2dRMLro/m9T4ZyvgQElbVSX7Bmks6LcAB5AbB+V0BwBvAsYBP87Hq6tMUdKZwJ6ksZ6xfSewS4n9GrI9v7C4Osveri7aE5hse67t50gtpMbl/afYfrKncYQQQtuUKEYqWZS0PTDL9kO2XyHV+Y5/7dlYM88PA57I8+OBS2y/bPufwCya3FSX6ivJ9mNSl1xtSZn9mpF0KnAQ8Dzw7hpJNgAeKyw/ntctzzmOAI4AGDNmTPcCDSGEbir55vMISdMKyxNsTygs17oW7lB1jBOB6yQdRbrZfm9h3ylV+za8jpZ5YnhM0k6AJQ2R9FVKdrst6XpJ99SYxgPYPs72aOAi4Mgyx1xetifYHmt77MiRI9txihBCqCkVJZV6YphTuU7laULjI9d0IHC+7VHA3sCFkspc41+jzBPDZ4AfknKY2cB1wOfLHNz2e5unAlLGMIlUn1A0G9itsDwKuLHkMUMIoc+1qNXRbFKL0IpReV3RoSwrar9F0qrAiJL7dtEwN8kVFJ+0/XHb69p+ve1P2P5XqY/S+NibFRbHA/fVSHYtsIektXKl8x7EWBAhhIHCLWuVNBXYTNLGklYmVSZPrErzKLA7gKQtSQOrPZvTHSBpFUkbA5sBtzU6WcOMwfYS0jsL7XBaLla6i3TB/yKApLGSzsnnnwucTPpSpgIn5XVIOl3S48Bqkh6XdGKb4gwhhG5JI7g1n5oeJ/VofSTpxvheUuujGZJOkvSBnOwrwOGS7gQuBg5xMgO4DJgJXAN8Pl/b65KbPOdIOhMYAlxKYZhP23c0/zj9y9ixYz1t2rTmCUMIKzxJt9se25NjbLj5WB/7f82vOZ8d1/NztVKZOobt8r/FwXkMvKf14YQQQmfpj282N1NmzOdazUhDCCE04Q4eqCeEEEI3NSuu748iYwghhDZa0pLXgXtXZAwhhNAm/bX31GaaZgyS9q2x+nngbtvPtD6kEELoHJ1ax3AosCNwQ17eDbgd2FjSSbYvbFNsIYQw4HXkE0NOs6XtpwEkrQtcQOrA6S9AZAwhhFCHB+AjQ5mMYXQlU8ieyevmSlrUprhCCGHAs8sPxNOflMkYbpR0NXB5Xv5wXrc6MK9tkYUQQgdY2qFPDJ8H9gXelZd/CVzp1Dg3Xn4LIYQ6Kt1uDzRl3ny2pL8Br5A+520eiG9shBBCbxugzVXLjPn8UVIXrR8GPgrcKunD7Q4shBAGPrPUzaf+pkxR0nHA2yvvLEgaCVwPXNHOwEIIoRO4QyufB1W9yPYvyg0JGkIIKzQblizpf08EzZTJGK6RdC1p4AeA/UnDcIYQQmhiIFbJlql8/pqk/YB35lUTbP+2vWGFEMLAZzq3SwxsXwlc2eZYQgihs7h1bz5LGgf8EBgMnGP7tKrtZ7LsFYLVgNfbHp63nQ78J6kaYDLwxUatS+tmDJIWkDK812witWJds/QnCiGEFVQrSpIkDQbOBt4HPA5MlTTR9sxl5/HRhfRHAW/J8zuRSny2yZv/BuwK3FjvfHUzBttDu/0pQgghAC1783l7YJbthwAkXQKMB2bWSX8gcEKeN7AqsDLpxn4I8HSd/YAYjyGEENrGNkvLtUoaIWlaYXmC7QmF5Q2AxwrLj5M6Mn0NSRsCGwN/zjHcIukG4ElSxnCW7XsbBRMZQwghtFHJF9jm2B7bolMeAFxhewmApE2BLYFReftkSTvb/mu9A8T7CCGE0Ea2m04lzAZGF5ZH5XW1HMCy1wsAPgRMsb3Q9kLgj6QxduqKjCGEENrETnUMzaYSpgKbSdpY0sqki//E6kSStgDWAm4prH4U2FXSSpKGkCqeGxYlRcYQQghtVBn3udHU/BheDBwJXEu6qF9me4akkyR9oJD0AOCSqqaoVwAPAncDdwJ32r6q0fmijiGEENrENktaNFKP7UlU9Tph+/iq5RNr7LcE+K/lOVdkDCGE0EadOrRnCCGEboqMIYQQwjLu4L6SQgghLD8TTwwhhBC6KP2eQr8SGUMIIbSLaVmrpN4UGUMIIbRJFCWFEELoqoXjMfSmPnvzWdLJku6SNF3SdZLWr5PuYEkP5OngvG41SX+QdJ+kGZJOq7VvCCH0LbPUzaf+pi+7xDjD9ja2twOuBo6vTiBpbVKf4juQ+iM/QdJaefN3bW9BGozinZL26qW4QwihNC9106m/6bOMwfb8wuLq1B4tbk9gsu25tp8jDUk3zvaLtm/Ix3kFuINlXcqGEEK/YFrWu2qv6tM6BkmnAgcBz7NsrNKiWoNTbFB1jOHA+0ljodY6xxHAEQBjxozpedAhhFCWYcnigdcqqa1PDJKul3RPjWk8gO3jbI8GLiL1HLi8x1+J1O/4jypD3lWzPcH2WNtjR44c2ZOPE0IIy6n508IK98Rg+70lk15E6jXwhKr1s4HdCsuj6DqA9QTgAds/6GaIIYTQNjZ4aTwxlCZps8LieOC+GsmuBfaQtFaudN4jr0PSKcAw4EvtjjWEELqrRQP19Kq+rGM4TdIbgaXAI8BnACSNBT5j+zDbcyWdTBq9COCkvG4UcBwpM7lDEqQBrs/p9U8RQggN9Meiomb6LGOwvV+d9dOAwwrL5wHnVaV5HFBbAwwhhB6yzdKofA4hhFC01EubTmVIGifpfkmzJB1TY/uZ+YXh6ZL+IWleYduY/CLxvZJmStqo0bmiS4wQQmiXFnWJIWkwcDbwPlKz/amSJtqe+eqp7KML6Y8ivfxbcQFwqu3JktYgFeHXFU8MIYTQJqb5W88lM47tgVm2H8ov9V5CarRTz4GkpvxI2gpYyfZkANsLbb/Y6GSRMYQQQhuVfI9hhKRphemIqsM0fdm3QtKGwMbAn/OqzYF5kn4j6e+SzshPIHVFUVIIIbSLYWm59xjm2B7borMeAFxhe0leXgnYmVS09ChwKXAIcG69A8QTQwghtIkxS5csaTqVMBsYXVgeldfVcgC5GCl7HJiei6EWA78D3troZJExhBBCu7hlvatOBTaTtLGklUkX/4nViSRtAawF3FK173BJlT6B3gPMrN63KDKGEEJoo1ZkDPlO/0hSzw/3ApfZniHpJEkfKCQ9ALjEhbfqcpHSV4E/Sbqb9A7YzxudL+oYQgihbVz6PYWmR7InkfqUK647vmr5xDr7Tga2KXuuyBhCCKFNPECH9oyMIYQQ2mgg9q4aGUMIIbSLXbbVUb8SGUMIIbSJoV92q91MZAwhhNAuA3SgnsgYQgihbUq/p9CvRMYQQght5BY1V+1NkTGEEEKbpIF6Bl7lswbisHPdJelZ0jCiZY0A5rQpnFbo7/FBxNgq/T3G/h4fLH+MG9oe2TxZfZKuyedtZo7tcT05VyutUBnD8pI0rYU9HrZcf48PIsZW6e8x9vf4YGDE2F9EX0khhBC6iIwhhBBCF5ExNDahrwNoor/HBxFjq/T3GPt7fDAwYuwXoo4hhBBCF/HEEEIIoYvIGEIIIXSxwmUMkk6WdJek6ZKuk7R+nXQHS3ogTwcX1p8q6TFJC6vSHyLp2Xzc6ZIO64cxriLpUkmzJN0qaaM+jPFtku7OsfxIkvL6EyXNLnyPe/fDGNeWNDmnnyxprW7Gd4ak+3KMv5U0vE66L0q6R9IMSV8qrN9W0i05xqskrZnXbyTppcJ3+NPuxNfOGPO2Y/N3e7+kPfswxu0kTcnf1TRJ2+f1u0l6vvA9Hl/ruB3J9go1AWsW5r8A/LRGmrWBh/K/a+X5tfK2dwDrAQur9jkEOKufx/i5yrFIQwBe2ocx3pbjFPBHYK+8/kTgq/3ke6wX4+nAMXn+GOA73YxvD2ClPP+dWscBtgbuAVYj9VRwPbBp3jYV2DXPfxo4Oc9vBNzTou+wXTFuBdwJrAJsDDwIDO6jGK8r/N/uDdyY53cDrm7F9zjQphXuicH2/MLi6qSecavtCUy2Pdf2c8BkYFzef4rtJwdojOOBX+b5K4DdK3fBvRmjpPVIF+0pTn+BFwAf7E4cfRRj8Xv8ZXdjt32d01i+AFOAUTWSbQncavvFnPYmYN+8bXPgL3l+MrBfd+LooxjHk8Ymftn2P4FZwPZ9FKOBypPMMOCJ7sTRSVa4jAGWFbUAHwdqPR5uADxWWH48r2tmv/w4e4Wk0f0wxlf3yX8czwPr9EGMG+T56vUVR+bv8bzuFtO0OcZ1CxnvU8C6PYkx+zTpqaTaPcDOktaRtBrpjrby25pBusACfKSwHmBjSX+XdJOknVsQX6tj7O7fWDti/BJwRv6dfBc4trDfjpLulPRHSW9qQXwDQkdmDJKuz2WJ1dN4ANvH2R4NXAQc2aLTXgVsZHsb0p3RLxsl7qMYl0sfxfgTYBNgO+BJ4Hv9MMZX5aeJum2+m8WX0xwHLM4xVh//XlLxyHXANcB0oNIr26eBz0m6HRgKvJLXPwmMsf0W4MvAr4tl+/0kxuXS5hg/CxydfydHA+fm9XeQ+kvaFvg/4HfdiX1A6uuyrL6cgDHUKIsFDgR+Vlj+GXBgVZqFDY47GHi+v8UIXAvsmOdXInUopt6OkVT/cV+9dIX1G9U6bl/HCNwPrJfn1wPu70FchwC3AKuVTP8t4HM11m8O3FZnnxuBsf0pRtJd+bG1fpu9HSPpybnyTpeA+XX2eRgY0YrfY3+f+jyAXv/AsFlh/ijgihpp1gb+SaqMXCvPr12Vpvqiu15h/kPAlH4Y4+fpWvl8WV/FyGsrdveu8T0eTSqH7m8xnkHXyufTuxnfOGAmMLJJutfnf8cA9wHDq9YPItWBfDovjyRX5AJvAGZX/zb6QYxvomvl80N0v/K5pzHeC+yW53cHbs/z/8GyDGN74FFacCM1EKY+D6DXPzBcSSpvvItU/LNBXj8WOKeQ7tOkCrFZwKcK608nlYcuzf+emNd/m1SeeidwA7BFP4xxVeDynP424A19GOPYvP+DwFmFP8ALgbvzcSdSyCj6UYzrAH8CHiC1bunuRXcWqZx9ep4qmfb6wKRCur+SLnx3ArsX1n8R+EeeTivEt1/+LU4nFYe8vwffYVtizNuOy9/t/eRWQX0U47uA2/P6W4G35fVHsuxvegqwU3djHGhTdIkRQgihi46sfA4hhNB9kTGEEELoIjKGEEIIXUTGEEIIoYvIGEIIIXQRGUNoK1X18NqD45wv6cOtOFaT8/y/dp+j6nzDJX2uN88ZQjORMYQViqSVGm23vVMvn3M4qdfbEPqNyBhCr1ByRu7f5m5J++f1gyT9OPenP1nSpGZPBkrjJNwk6XZJ1+aeUJF0uKSpudOzK3NnaZWnjZ9KuhU4XWnMh/Mk3SjpIUlfKBx7Yf53t7z9ihzbRdKr4zHsndfdrjROw9U1YjxE0kRJfwb+JGkNSX+SdEf+/JU+fk4DNlHq7/+MvO/X8ue4S9I3e/rdh7C8Gt49hdBC+5I6xtsWGAFMlfQX4J2kPpG2Al5P6p7gvHoHkTSE1KHZeNvP5gzmVNLby7+x/fOc7hTg0JwWUlfMO9leIulEYAvg3aSO3e6X9BPbi6pO9xZS1w1PADcD75Q0jdSf0i62/ynp4gaf+a3ANrbn5qeGD9meL2kEMEXSRFKXGlvb3i7HvQewGakLBgETJe1i+y91zhFCy0XGEHrLu4CLbS8BnpZ0E/D2vP5y20uBpyTd0OQ4byQNujI538APJvUmCrB1zhCGA2uQOmaruDyfu+IPtl8GXpb0DKnr7GI325A6fHscQNJ0Uga2EHjIaQwBgIuBI+rEOtn23Dwv4FuSdiF1VbIBtbvr3iNPf8/La5AyisgYQq+JjCEMNAJm2N6xxrbzgQ/avlPSIaQRuCpeqEr7cmF+CbX/FsqkaaR4zo+TOrd7m+1Fkh4m9V1VTcC3bf9sOc8VQstEHUPoLX8F9pc0WNJIYBdSR343kwY4GiRpXbpezGu5HxgpaUdIRUtaNoDKUODJXNz08XZ8iHz+N2jZeNn7l9xvGPBMzhTeDWyY1y8gxV1xLfBpSWsASNpA0ut7HHUIyyGeGEJv+S2wI6mnSgNft/2UpCtJXR3PJPWQeQepf/yabL+SK6d/JGkY6Tf8A1IvmP9L6h3z2fzv0HrH6S7bL+XmpddIeoE0pnEZFwFXSbobmEbq9hnb/5J0s6R7gD/a/pqkLYFbclHZQuATwDOt/iwh1BO9q4Y+J2kN2wslrUN6inin7af6Oq56CvEKOBt4wPaZfR1XCK0STwyhP7ha0nBgZeDk/pwpZIdLOpgU799JrZRC6BjxxBBCCKGLqHwOIYTQRWQMIYQQuoiMIYQQQheRMYQQQugiMoYQQghd/H/NlQCRQlawkgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualize the cross-validation results\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "x_scatter = [math.log10(x[0]) for x in results]\n",
    "y_scatter = [math.log10(x[1]) for x in results]\n",
    "\n",
    "\n",
    "# plot validation accuracy\n",
    "marker_size=100\n",
    "colors = [results[x] for x in results] # default size of markers is 20\n",
    "\n",
    "plt.scatter(x_scatter, y_scatter, marker_size, c=colors, cmap=plt.cm.coolwarm)\n",
    "plt.colorbar()\n",
    "plt.xlabel('log learning rate')\n",
    "plt.ylabel('log regularization strength')\n",
    "plt.title('Net intrusion(CIC-IDS-2017) validation accuracy')\n",
    "plt.savefig('MachineLearningCVE/method_{}_arch_run_{}-2.png'.format(method,architecture,run_number))\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}