Hybrid learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Supervised/Hybrid learning on dataset augmented with outliers score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "extendend_dataset_selected = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import h2o\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n"
     ]
    }
   ],
   "source": [
    "import plotly.graph_objs as go\n",
    "import plotly\n",
    "plotly.offline.init_notebook_mode() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import collections"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load selected configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if extendend_dataset_selected:\n",
    "    from extendedgenv import *\n",
    "else:\n",
    "    from genv import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### H2O cluster initialisation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "API doc : http://docs.h2o.ai/h2o/latest-stable/h2o-py/docs/index.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking whether there is an H2O instance running at http://localhost:54321. connected.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O cluster uptime:</td>\n",
       "<td>56 mins 57 secs</td></tr>\n",
       "<tr><td>H2O cluster version:</td>\n",
       "<td>3.10.4.8</td></tr>\n",
       "<tr><td>H2O cluster version age:</td>\n",
       "<td>5 days </td></tr>\n",
       "<tr><td>H2O cluster name:</td>\n",
       "<td>H2O_from_python_pierre_2e44fe</td></tr>\n",
       "<tr><td>H2O cluster total nodes:</td>\n",
       "<td>1</td></tr>\n",
       "<tr><td>H2O cluster free memory:</td>\n",
       "<td>27.77 Gb</td></tr>\n",
       "<tr><td>H2O cluster total cores:</td>\n",
       "<td>4</td></tr>\n",
       "<tr><td>H2O cluster allowed cores:</td>\n",
       "<td>4</td></tr>\n",
       "<tr><td>H2O cluster status:</td>\n",
       "<td>locked, healthy</td></tr>\n",
       "<tr><td>H2O connection url:</td>\n",
       "<td>http://localhost:54321</td></tr>\n",
       "<tr><td>H2O connection proxy:</td>\n",
       "<td>None</td></tr>\n",
       "<tr><td>H2O internal security:</td>\n",
       "<td>False</td></tr>\n",
       "<tr><td>Python version:</td>\n",
       "<td>3.6.0 final</td></tr></table></div>"
      ],
      "text/plain": [
       "--------------------------  -----------------------------\n",
       "H2O cluster uptime:         56 mins 57 secs\n",
       "H2O cluster version:        3.10.4.8\n",
       "H2O cluster version age:    5 days\n",
       "H2O cluster name:           H2O_from_python_pierre_2e44fe\n",
       "H2O cluster total nodes:    1\n",
       "H2O cluster free memory:    27.77 Gb\n",
       "H2O cluster total cores:    4\n",
       "H2O cluster allowed cores:  4\n",
       "H2O cluster status:         locked, healthy\n",
       "H2O connection url:         http://localhost:54321\n",
       "H2O connection proxy:\n",
       "H2O internal security:      False\n",
       "Python version:             3.6.0 final\n",
       "--------------------------  -----------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "h2o.init() # max_mem_size = \"4g\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "df_train = h2o.import_file(path=DIR_OUTPUT +\"train_unbalanced_outlier.csv\", col_names=COL_NAME_OUTLIER, col_types=COL_TYPE_OUTLIER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "df_validtest = h2o.import_file(path=DIR_OUTPUT +\"evaltest_outlier.csv\", col_names=COL_NAME_OUTLIER, col_types=COL_TYPE_OUTLIER)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">  datetime</th><th style=\"text-align: right;\">      V1</th><th style=\"text-align: right;\">        V2</th><th style=\"text-align: right;\">     V3</th><th style=\"text-align: right;\">     V4</th><th style=\"text-align: right;\">       V5</th><th style=\"text-align: right;\">      V6</th><th style=\"text-align: right;\">      V7</th><th style=\"text-align: right;\">       V8</th><th style=\"text-align: right;\">      V9</th><th style=\"text-align: right;\">      V10</th><th style=\"text-align: right;\">    V11</th><th style=\"text-align: right;\">      V12</th><th style=\"text-align: right;\">     V13</th><th style=\"text-align: right;\">      V14</th><th style=\"text-align: right;\">    V15</th><th style=\"text-align: right;\">      V16</th><th style=\"text-align: right;\">     V17</th><th style=\"text-align: right;\">      V18</th><th style=\"text-align: right;\">     V19</th><th style=\"text-align: right;\">     V20</th><th style=\"text-align: right;\">       V21</th><th style=\"text-align: right;\">     V22</th><th style=\"text-align: right;\">      V23</th><th style=\"text-align: right;\">      V24</th><th style=\"text-align: right;\">     V25</th><th style=\"text-align: right;\">      V26</th><th style=\"text-align: right;\">     V27</th><th style=\"text-align: right;\">       V28</th><th style=\"text-align: right;\">  amount</th><th style=\"text-align: right;\">  fraud</th><th style=\"text-align: right;\">  outlier_autoenc</th><th style=\"text-align: right;\">  outlier_pca</th><th style=\"text-align: right;\">  outlier_zscore</th><th style=\"text-align: right;\">  ensemble_outlier</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">-1.35981</td><td style=\"text-align: right;\">-0.0727812</td><td style=\"text-align: right;\">2.53635</td><td style=\"text-align: right;\">1.37816</td><td style=\"text-align: right;\">-0.338321</td><td style=\"text-align: right;\">0.462388</td><td style=\"text-align: right;\">0.239599</td><td style=\"text-align: right;\">0.0986979</td><td style=\"text-align: right;\">0.363787</td><td style=\"text-align: right;\">0.0907942</td><td style=\"text-align: right;\">-0.5516</td><td style=\"text-align: right;\">-0.617801</td><td style=\"text-align: right;\">-0.99139</td><td style=\"text-align: right;\">-0.311169</td><td style=\"text-align: right;\">1.46818</td><td style=\"text-align: right;\">-0.470401</td><td style=\"text-align: right;\">0.207971</td><td style=\"text-align: right;\">0.0257906</td><td style=\"text-align: right;\">0.403993</td><td style=\"text-align: right;\">0.251412</td><td style=\"text-align: right;\">-0.0183068</td><td style=\"text-align: right;\">0.277838</td><td style=\"text-align: right;\">-0.110474</td><td style=\"text-align: right;\">0.0669281</td><td style=\"text-align: right;\">0.128539</td><td style=\"text-align: right;\">-0.189115</td><td style=\"text-align: right;\">0.133558</td><td style=\"text-align: right;\">-0.0210531</td><td style=\"text-align: right;\">     149.62</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">      2.10499e-06</td><td style=\"text-align: right;\">    0.0627478</td><td style=\"text-align: right;\">        0.278364</td><td style=\"text-align: right;\">         0.0455621</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">  datetime</th><th style=\"text-align: right;\">       V1</th><th style=\"text-align: right;\">     V2</th><th style=\"text-align: right;\">      V3</th><th style=\"text-align: right;\">      V4</th><th style=\"text-align: right;\">      V5</th><th style=\"text-align: right;\">      V6</th><th style=\"text-align: right;\">      V7</th><th style=\"text-align: right;\">      V8</th><th style=\"text-align: right;\">        V9</th><th style=\"text-align: right;\">     V10</th><th style=\"text-align: right;\">       V11</th><th style=\"text-align: right;\">     V12</th><th style=\"text-align: right;\">    V13</th><th style=\"text-align: right;\">     V14</th><th style=\"text-align: right;\">      V15</th><th style=\"text-align: right;\">    V16</th><th style=\"text-align: right;\">      V17</th><th style=\"text-align: right;\">    V18</th><th style=\"text-align: right;\">    V19</th><th style=\"text-align: right;\">       V20</th><th style=\"text-align: right;\">      V21</th><th style=\"text-align: right;\">      V22</th><th style=\"text-align: right;\">      V23</th><th style=\"text-align: right;\">     V24</th><th style=\"text-align: right;\">      V25</th><th style=\"text-align: right;\">      V26</th><th style=\"text-align: right;\">      V27</th><th style=\"text-align: right;\">      V28</th><th style=\"text-align: right;\">  amount</th><th style=\"text-align: right;\">  fraud</th><th style=\"text-align: right;\">  outlier_autoenc</th><th style=\"text-align: right;\">  outlier_pca</th><th style=\"text-align: right;\">  outlier_zscore</th><th style=\"text-align: right;\">  ensemble_outlier</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\">       126397</td><td style=\"text-align: right;\">-0.765586</td><td style=\"text-align: right;\">1.11071</td><td style=\"text-align: right;\">0.497975</td><td style=\"text-align: right;\">-1.24207</td><td style=\"text-align: right;\">0.809253</td><td style=\"text-align: right;\">0.299673</td><td style=\"text-align: right;\">0.352753</td><td style=\"text-align: right;\">0.332642</td><td style=\"text-align: right;\">0.00719612</td><td style=\"text-align: right;\">-1.36338</td><td style=\"text-align: right;\">0.00152343</td><td style=\"text-align: right;\">0.759458</td><td style=\"text-align: right;\">1.21781</td><td style=\"text-align: right;\">-1.42397</td><td style=\"text-align: right;\">-0.316525</td><td style=\"text-align: right;\">1.14988</td><td style=\"text-align: right;\">-0.371549</td><td style=\"text-align: right;\">1.22995</td><td style=\"text-align: right;\">0.55664</td><td style=\"text-align: right;\">-0.0120937</td><td style=\"text-align: right;\">-0.223609</td><td style=\"text-align: right;\">-0.553292</td><td style=\"text-align: right;\">-0.268529</td><td style=\"text-align: right;\">-1.48553</td><td style=\"text-align: right;\">-0.149443</td><td style=\"text-align: right;\">-0.252863</td><td style=\"text-align: right;\">-0.286616</td><td style=\"text-align: right;\">-0.148096</td><td style=\"text-align: right;\">       3.79</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">      4.87494e-06</td><td style=\"text-align: right;\">    0.0353845</td><td style=\"text-align: right;\">        0.729109</td><td style=\"text-align: right;\">                 0</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_validtest.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we defined several interchangeable supervised model to be run"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sampling factor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Deal with unbalanced classification using under-sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "CLASS_SAMPLING_FACTOR = list([0.01,1.0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random forrest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "NTREES = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "randomforrest = h2o.estimators.random_forest.H2ORandomForestEstimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "randomforrestargs = {\"ntrees\" : NTREES, \n",
    "                     \"balance_classes\" : True, \n",
    "                     \"class_sampling_factors\" : CLASS_SAMPLING_FACTOR, \n",
    "                     \"binomial_double_trees\" : True,\n",
    "                     \"stopping_metric\" : \"auc\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gradient boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gradientboosting = h2o.estimators.gbm.H2OGradientBoostingEstimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gradientboostingargs = {\"balance_classes\" : True, \n",
    "                        \"class_sampling_factors\" : CLASS_SAMPLING_FACTOR, \n",
    "                        \"stopping_metric\" : \"auc\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Neural networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lenft = len(X_fraud) # NN input size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "neuralnetwork = h2o.estimators.deeplearning.H2ODeepLearningEstimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "neuralnetworkargs = {\"hidden\" : [lenft*2, lenft*4, lenft*8, lenft*16, lenft*8, lenft*4, lenft],\n",
    "                     \"balance_classes\" : True, \n",
    "                     \"class_sampling_factors\" : CLASS_SAMPLING_FACTOR,  \n",
    "                     \"stopping_metric\" : \"auc\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We select at this point one of the models above (RF or GB or NN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = randomforrest\n",
    "modelargs = randomforrestargs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Supervised model without any outliers information"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model without the anomaly score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_simple = model(**modelargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "drf Model Build progress: |███████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "model_simple.train(x                = X_fraud, \n",
    "                   y                = y_fraud, \n",
    "                   training_frame   = df_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Details\n",
      "=============\n",
      "H2ORandomForestEstimator :  Distributed Random Forest\n",
      "Model Key:  DRF_model_python_1495800034122_110\n",
      "\n",
      "\n",
      "ModelMetricsBinomial: drf\n",
      "** Reported on train data. **\n",
      "\n",
      "MSE: 0.057002971756047426\n",
      "RMSE: 0.23875295130332405\n",
      "LogLoss: 0.3688770595461154\n",
      "Mean Per-Class Error: 0.0521833751241898\n",
      "AUC: 0.9746408844679946\n",
      "Gini: 0.9492817689359891\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.0077386197215977524: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>0</b></td>\n",
       "<td><b>1</b></td>\n",
       "<td><b>Error</b></td>\n",
       "<td><b>Rate</b></td></tr>\n",
       "<tr><td>0</td>\n",
       "<td>1830.0</td>\n",
       "<td>8.0</td>\n",
       "<td>0.0044</td>\n",
       "<td> (8.0/1838.0)</td></tr>\n",
       "<tr><td>1</td>\n",
       "<td>42.0</td>\n",
       "<td>326.0</td>\n",
       "<td>0.1141</td>\n",
       "<td> (42.0/368.0)</td></tr>\n",
       "<tr><td>Total</td>\n",
       "<td>1872.0</td>\n",
       "<td>334.0</td>\n",
       "<td>0.0227</td>\n",
       "<td> (50.0/2206.0)</td></tr></table></div>"
      ],
      "text/plain": [
       "       0     1    Error    Rate\n",
       "-----  ----  ---  -------  -------------\n",
       "0      1830  8    0.0044   (8.0/1838.0)\n",
       "1      42    326  0.1141   (42.0/368.0)\n",
       "Total  1872  334  0.0227   (50.0/2206.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum Metrics: Maximum metrics at their respective thresholds\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>metric</b></td>\n",
       "<td><b>threshold</b></td>\n",
       "<td><b>value</b></td>\n",
       "<td><b>idx</b></td></tr>\n",
       "<tr><td>max f1</td>\n",
       "<td>0.0077386</td>\n",
       "<td>0.9287749</td>\n",
       "<td>77.0</td></tr>\n",
       "<tr><td>max f2</td>\n",
       "<td>0.0037231</td>\n",
       "<td>0.9130435</td>\n",
       "<td>98.0</td></tr>\n",
       "<tr><td>max f0point5</td>\n",
       "<td>0.0360239</td>\n",
       "<td>0.9627139</td>\n",
       "<td>60.0</td></tr>\n",
       "<tr><td>max accuracy</td>\n",
       "<td>0.0077386</td>\n",
       "<td>0.9773345</td>\n",
       "<td>77.0</td></tr>\n",
       "<tr><td>max precision</td>\n",
       "<td>0.0360239</td>\n",
       "<td>0.9936909</td>\n",
       "<td>60.0</td></tr>\n",
       "<tr><td>max recall</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>252.0</td></tr>\n",
       "<tr><td>max specificity</td>\n",
       "<td>1.0</td>\n",
       "<td>0.9989119</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max absolute_mcc</td>\n",
       "<td>0.0077386</td>\n",
       "<td>0.9168562</td>\n",
       "<td>77.0</td></tr>\n",
       "<tr><td>max min_per_class_accuracy</td>\n",
       "<td>0.0019891</td>\n",
       "<td>0.9266304</td>\n",
       "<td>131.0</td></tr>\n",
       "<tr><td>max mean_per_class_accuracy</td>\n",
       "<td>0.0037231</td>\n",
       "<td>0.9478166</td>\n",
       "<td>98.0</td></tr></table></div>"
      ],
      "text/plain": [
       "metric                       threshold    value     idx\n",
       "---------------------------  -----------  --------  -----\n",
       "max f1                       0.00773862   0.928775  77\n",
       "max f2                       0.00372309   0.913043  98\n",
       "max f0point5                 0.0360239    0.962714  60\n",
       "max accuracy                 0.00773862   0.977335  77\n",
       "max precision                0.0360239    0.993691  60\n",
       "max recall                   0            1         252\n",
       "max specificity              1            0.998912  0\n",
       "max absolute_mcc             0.00773862   0.916856  77\n",
       "max min_per_class_accuracy   0.00198911   0.92663   131\n",
       "max mean_per_class_accuracy  0.00372309   0.947817  98"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gains/Lift Table: Avg response rate:  0.20 %\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>group</b></td>\n",
       "<td><b>cumulative_data_fraction</b></td>\n",
       "<td><b>lower_threshold</b></td>\n",
       "<td><b>lift</b></td>\n",
       "<td><b>cumulative_lift</b></td>\n",
       "<td><b>response_rate</b></td>\n",
       "<td><b>cumulative_response_rate</b></td>\n",
       "<td><b>capture_rate</b></td>\n",
       "<td><b>cumulative_capture_rate</b></td>\n",
       "<td><b>gain</b></td>\n",
       "<td><b>cumulative_gain</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>1</td>\n",
       "<td>0.0101836</td>\n",
       "<td>0.0053373</td>\n",
       "<td>98.1971307</td>\n",
       "<td>98.1971307</td>\n",
       "<td>0.1955367</td>\n",
       "<td>0.1955367</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>9719.7130712</td>\n",
       "<td>9719.7130712</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2</td>\n",
       "<td>0.0200263</td>\n",
       "<td>0.0032674</td>\n",
       "<td>0.0</td>\n",
       "<td>49.9343421</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0994326</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>4893.4342070</td></tr>\n",
       "<tr><td></td>\n",
       "<td>3</td>\n",
       "<td>0.0301937</td>\n",
       "<td>0.0024851</td>\n",
       "<td>0.0</td>\n",
       "<td>33.1195341</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0659498</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>3211.9534050</td></tr>\n",
       "<tr><td></td>\n",
       "<td>4</td>\n",
       "<td>0.0401338</td>\n",
       "<td>0.0020369</td>\n",
       "<td>0.0</td>\n",
       "<td>24.9166779</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0496157</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>2391.6677902</td></tr>\n",
       "<tr><td></td>\n",
       "<td>5</td>\n",
       "<td>0.0503877</td>\n",
       "<td>0.0016387</td>\n",
       "<td>0.0</td>\n",
       "<td>19.8461125</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0395189</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>1884.6112543</td></tr>\n",
       "<tr><td></td>\n",
       "<td>6</td>\n",
       "<td>0.1003588</td>\n",
       "<td>0.0009739</td>\n",
       "<td>0.0</td>\n",
       "<td>9.9642530</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0198415</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>896.4252979</td></tr>\n",
       "<tr><td></td>\n",
       "<td>7</td>\n",
       "<td>0.1528838</td>\n",
       "<td>0.0006357</td>\n",
       "<td>0.0</td>\n",
       "<td>6.5409146</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0130247</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>554.0914561</td></tr>\n",
       "<tr><td></td>\n",
       "<td>8</td>\n",
       "<td>0.2185036</td>\n",
       "<td>0.0004151</td>\n",
       "<td>0.0</td>\n",
       "<td>4.5765830</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0091132</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>357.6583046</td></tr>\n",
       "<tr><td></td>\n",
       "<td>9</td>\n",
       "<td>0.3595914</td>\n",
       "<td>0.0002033</td>\n",
       "<td>0.0</td>\n",
       "<td>2.7809345</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0055376</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>178.0934467</td></tr>\n",
       "<tr><td></td>\n",
       "<td>10</td>\n",
       "<td>0.5240007</td>\n",
       "<td>0.0001007</td>\n",
       "<td>0.0</td>\n",
       "<td>1.9083943</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0038001</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>90.8394345</td></tr>\n",
       "<tr><td></td>\n",
       "<td>11</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0019913</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>0.0</td></tr></table></div>"
      ],
      "text/plain": [
       "    group    cumulative_data_fraction    lower_threshold    lift     cumulative_lift    response_rate    cumulative_response_rate    capture_rate    cumulative_capture_rate    gain     cumulative_gain\n",
       "--  -------  --------------------------  -----------------  -------  -----------------  ---------------  --------------------------  --------------  -------------------------  -------  -----------------\n",
       "    1        0.0101836                   0.00533732         98.1971  98.1971            0.195537         0.195537                    1               1                          9719.71  9719.71\n",
       "    2        0.0200263                   0.00326737         0        49.9343            0                0.0994326                   0               1                          -100     4893.43\n",
       "    3        0.0301937                   0.00248515         0        33.1195            0                0.0659498                   0               1                          -100     3211.95\n",
       "    4        0.0401338                   0.00203694         0        24.9167            0                0.0496157                   0               1                          -100     2391.67\n",
       "    5        0.0503877                   0.00163866         0        19.8461            0                0.0395189                   0               1                          -100     1884.61\n",
       "    6        0.100359                    0.000973922        0        9.96425            0                0.0198415                   0               1                          -100     896.425\n",
       "    7        0.152884                    0.000635682        0        6.54091            0                0.0130247                   0               1                          -100     554.091\n",
       "    8        0.218504                    0.000415051        0        4.57658            0                0.0091132                   0               1                          -100     357.658\n",
       "    9        0.359591                    0.000203333        0        2.78093            0                0.00553758                  0               1                          -100     178.093\n",
       "    10       0.524001                    0.00010065         0        1.90839            0                0.00380012                  0               1                          -100     90.8394\n",
       "    11       1                           0                  0        1                  0                0.00199127                  0               1                          -100     0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Scoring History: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>timestamp</b></td>\n",
       "<td><b>duration</b></td>\n",
       "<td><b>number_of_trees</b></td>\n",
       "<td><b>training_rmse</b></td>\n",
       "<td><b>training_logloss</b></td>\n",
       "<td><b>training_auc</b></td>\n",
       "<td><b>training_lift</b></td>\n",
       "<td><b>training_classification_error</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:34</td>\n",
       "<td> 0.008 sec</td>\n",
       "<td>0.0</td>\n",
       "<td>nan</td>\n",
       "<td>nan</td>\n",
       "<td>nan</td>\n",
       "<td>nan</td>\n",
       "<td>nan</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:34</td>\n",
       "<td> 0.144 sec</td>\n",
       "<td>1.0</td>\n",
       "<td>0.2255827</td>\n",
       "<td>1.7575945</td>\n",
       "<td>0.9104091</td>\n",
       "<td>26.2520260</td>\n",
       "<td>0.0508876</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:34</td>\n",
       "<td> 0.339 sec</td>\n",
       "<td>2.0</td>\n",
       "<td>0.2083755</td>\n",
       "<td>1.4581009</td>\n",
       "<td>0.9213363</td>\n",
       "<td>15.7878899</td>\n",
       "<td>0.0434473</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:34</td>\n",
       "<td> 0.562 sec</td>\n",
       "<td>3.0</td>\n",
       "<td>0.2033062</td>\n",
       "<td>1.2912305</td>\n",
       "<td>0.9266887</td>\n",
       "<td>94.7882564</td>\n",
       "<td>0.0414236</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:34</td>\n",
       "<td> 0.818 sec</td>\n",
       "<td>4.0</td>\n",
       "<td>0.1990700</td>\n",
       "<td>1.1678774</td>\n",
       "<td>0.9283953</td>\n",
       "<td>60.6991127</td>\n",
       "<td>0.0392365</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:35</td>\n",
       "<td> 1.229 sec</td>\n",
       "<td>5.0</td>\n",
       "<td>0.1850400</td>\n",
       "<td>0.9357815</td>\n",
       "<td>0.9366753</td>\n",
       "<td>43.8727672</td>\n",
       "<td>0.0339152</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:35</td>\n",
       "<td> 1.892 sec</td>\n",
       "<td>6.0</td>\n",
       "<td>0.1905538</td>\n",
       "<td>0.9742368</td>\n",
       "<td>0.9359067</td>\n",
       "<td>97.4243528</td>\n",
       "<td>0.0355940</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:36</td>\n",
       "<td> 2.329 sec</td>\n",
       "<td>7.0</td>\n",
       "<td>0.1868065</td>\n",
       "<td>0.8914377</td>\n",
       "<td>0.9367051</td>\n",
       "<td>72.8310169</td>\n",
       "<td>0.0328638</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:37</td>\n",
       "<td> 2.948 sec</td>\n",
       "<td>8.0</td>\n",
       "<td>0.1887078</td>\n",
       "<td>0.8409664</td>\n",
       "<td>0.9405281</td>\n",
       "<td>59.8510629</td>\n",
       "<td>0.0332410</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:37</td>\n",
       "<td> 3.507 sec</td>\n",
       "<td>9.0</td>\n",
       "<td>0.1911725</td>\n",
       "<td>0.7719689</td>\n",
       "<td>0.9438780</td>\n",
       "<td>49.4136364</td>\n",
       "<td>0.0312356</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:39</td>\n",
       "<td> 5.167 sec</td>\n",
       "<td>100.0</td>\n",
       "<td>0.2387530</td>\n",
       "<td>0.3688771</td>\n",
       "<td>0.9746409</td>\n",
       "<td>98.1971307</td>\n",
       "<td>0.0226655</td></tr></table></div>"
      ],
      "text/plain": [
       "    timestamp            duration    number_of_trees    training_rmse    training_logloss    training_auc    training_lift    training_classification_error\n",
       "--  -------------------  ----------  -----------------  ---------------  ------------------  --------------  ---------------  -------------------------------\n",
       "    2017-05-26 14:57:34  0.008 sec   0                  nan              nan                 nan             nan              nan\n",
       "    2017-05-26 14:57:34  0.144 sec   1                  0.225583         1.75759             0.910409        26.252           0.0508876\n",
       "    2017-05-26 14:57:34  0.339 sec   2                  0.208376         1.4581              0.921336        15.7879          0.0434473\n",
       "    2017-05-26 14:57:34  0.562 sec   3                  0.203306         1.29123             0.926689        94.7883          0.0414236\n",
       "    2017-05-26 14:57:34  0.818 sec   4                  0.19907          1.16788             0.928395        60.6991          0.0392365\n",
       "    2017-05-26 14:57:35  1.229 sec   5                  0.18504          0.935782            0.936675        43.8728          0.0339152\n",
       "    2017-05-26 14:57:35  1.892 sec   6                  0.190554         0.974237            0.935907        97.4244          0.035594\n",
       "    2017-05-26 14:57:36  2.329 sec   7                  0.186807         0.891438            0.936705        72.831           0.0328638\n",
       "    2017-05-26 14:57:37  2.948 sec   8                  0.188708         0.840966            0.940528        59.8511          0.033241\n",
       "    2017-05-26 14:57:37  3.507 sec   9                  0.191173         0.771969            0.943878        49.4136          0.0312356\n",
       "    2017-05-26 14:57:39  5.167 sec   100                0.238753         0.368877            0.974641        98.1971          0.0226655"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable Importances: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>variable</b></td>\n",
       "<td><b>relative_importance</b></td>\n",
       "<td><b>scaled_importance</b></td>\n",
       "<td><b>percentage</b></td></tr>\n",
       "<tr><td>V17</td>\n",
       "<td>10052.8740234</td>\n",
       "<td>1.0</td>\n",
       "<td>0.1891703</td></tr>\n",
       "<tr><td>V14</td>\n",
       "<td>9777.9843750</td>\n",
       "<td>0.9726556</td>\n",
       "<td>0.1839976</td></tr>\n",
       "<tr><td>V10</td>\n",
       "<td>6137.5122070</td>\n",
       "<td>0.6105231</td>\n",
       "<td>0.1154929</td></tr>\n",
       "<tr><td>V12</td>\n",
       "<td>4681.5341797</td>\n",
       "<td>0.4656911</td>\n",
       "<td>0.0880949</td></tr>\n",
       "<tr><td>V11</td>\n",
       "<td>4517.4550781</td>\n",
       "<td>0.4493695</td>\n",
       "<td>0.0850074</td></tr>\n",
       "<tr><td>---</td>\n",
       "<td>---</td>\n",
       "<td>---</td>\n",
       "<td>---</td></tr>\n",
       "<tr><td>V27</td>\n",
       "<td>175.5013580</td>\n",
       "<td>0.0174578</td>\n",
       "<td>0.0033025</td></tr>\n",
       "<tr><td>amount</td>\n",
       "<td>171.2793732</td>\n",
       "<td>0.0170379</td>\n",
       "<td>0.0032231</td></tr>\n",
       "<tr><td>V1</td>\n",
       "<td>146.8365021</td>\n",
       "<td>0.0146064</td>\n",
       "<td>0.0027631</td></tr>\n",
       "<tr><td>V24</td>\n",
       "<td>136.2295074</td>\n",
       "<td>0.0135513</td>\n",
       "<td>0.0025635</td></tr>\n",
       "<tr><td>V28</td>\n",
       "<td>132.3459778</td>\n",
       "<td>0.0131650</td>\n",
       "<td>0.0024904</td></tr></table></div>"
      ],
      "text/plain": [
       "variable    relative_importance    scaled_importance     percentage\n",
       "----------  ---------------------  --------------------  ---------------------\n",
       "V17         10052.8740234375       1.0                   0.18917034086570086\n",
       "V14         9777.984375            0.9726556159167402    0.18399759440790797\n",
       "V10         6137.51220703125       0.6105231392258685    0.1154928703537553\n",
       "V12         4681.5341796875        0.4656911216407233    0.08809494821890619\n",
       "V11         4517.455078125         0.44936951040994866   0.0850073834589031\n",
       "---         ---                    ---                   ---\n",
       "V27         175.50135803222656     0.017457829236003423  0.003302503507349966\n",
       "amount   171.2793731689453      0.017037851341777554  0.0032230561459431982\n",
       "V1          146.8365020751953      0.014606420187187996  0.0027631014856380073\n",
       "V24         136.22950744628906     0.013551299571513627  0.002563503959116459\n",
       "V28         132.34597778320312     0.013164989183655211  0.002490425491365321"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "See the whole table with table.as_data_frame()\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(model_simple)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Supervised model with outliers score as features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model with additionals columns, one for each of the anomaly detection techniques tested"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_outliers = model(**modelargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "drf Model Build progress: |███████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "model_outliers.train(x                = X_fraud_outlier, \n",
    "                     y                = y_fraud, \n",
    "                     training_frame   = df_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Details\n",
      "=============\n",
      "H2ORandomForestEstimator :  Distributed Random Forest\n",
      "Model Key:  DRF_model_python_1495800034122_132\n",
      "\n",
      "\n",
      "ModelMetricsBinomial: drf\n",
      "** Reported on train data. **\n",
      "\n",
      "MSE: 0.04567900953548608\n",
      "RMSE: 0.2137264829998521\n",
      "LogLoss: 0.2900708980905961\n",
      "Mean Per-Class Error: 0.056388489639477735\n",
      "AUC: 0.9774800874356211\n",
      "Gini: 0.9549601748712422\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.00586973394851451: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>0</b></td>\n",
       "<td><b>1</b></td>\n",
       "<td><b>Error</b></td>\n",
       "<td><b>Rate</b></td></tr>\n",
       "<tr><td>0</td>\n",
       "<td>1809.0</td>\n",
       "<td>6.0</td>\n",
       "<td>0.0033</td>\n",
       "<td> (6.0/1815.0)</td></tr>\n",
       "<tr><td>1</td>\n",
       "<td>41.0</td>\n",
       "<td>327.0</td>\n",
       "<td>0.1114</td>\n",
       "<td> (41.0/368.0)</td></tr>\n",
       "<tr><td>Total</td>\n",
       "<td>1850.0</td>\n",
       "<td>333.0</td>\n",
       "<td>0.0215</td>\n",
       "<td> (47.0/2183.0)</td></tr></table></div>"
      ],
      "text/plain": [
       "       0     1    Error    Rate\n",
       "-----  ----  ---  -------  -------------\n",
       "0      1809  6    0.0033   (6.0/1815.0)\n",
       "1      41    327  0.1114   (41.0/368.0)\n",
       "Total  1850  333  0.0215   (47.0/2183.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum Metrics: Maximum metrics at their respective thresholds\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>metric</b></td>\n",
       "<td><b>threshold</b></td>\n",
       "<td><b>value</b></td>\n",
       "<td><b>idx</b></td></tr>\n",
       "<tr><td>max f1</td>\n",
       "<td>0.0058697</td>\n",
       "<td>0.9329529</td>\n",
       "<td>87.0</td></tr>\n",
       "<tr><td>max f2</td>\n",
       "<td>0.0032695</td>\n",
       "<td>0.9061313</td>\n",
       "<td>113.0</td></tr>\n",
       "<tr><td>max f0point5</td>\n",
       "<td>0.0274823</td>\n",
       "<td>0.9688264</td>\n",
       "<td>73.0</td></tr>\n",
       "<tr><td>max accuracy</td>\n",
       "<td>0.0058697</td>\n",
       "<td>0.9784700</td>\n",
       "<td>87.0</td></tr>\n",
       "<tr><td>max precision</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max recall</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>257.0</td></tr>\n",
       "<tr><td>max specificity</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max absolute_mcc</td>\n",
       "<td>0.0058697</td>\n",
       "<td>0.9217974</td>\n",
       "<td>87.0</td></tr>\n",
       "<tr><td>max min_per_class_accuracy</td>\n",
       "<td>0.0016374</td>\n",
       "<td>0.9347826</td>\n",
       "<td>157.0</td></tr>\n",
       "<tr><td>max mean_per_class_accuracy</td>\n",
       "<td>0.0032695</td>\n",
       "<td>0.9436115</td>\n",
       "<td>113.0</td></tr></table></div>"
      ],
      "text/plain": [
       "metric                       threshold    value     idx\n",
       "---------------------------  -----------  --------  -----\n",
       "max f1                       0.00586973   0.932953  87\n",
       "max f2                       0.00326949   0.906131  113\n",
       "max f0point5                 0.0274823    0.968826  73\n",
       "max accuracy                 0.00586973   0.97847   87\n",
       "max precision                1            1         0\n",
       "max recall                   0            1         257\n",
       "max specificity              1            1         0\n",
       "max absolute_mcc             0.00586973   0.921797  87\n",
       "max min_per_class_accuracy   0.00163742   0.934783  157\n",
       "max mean_per_class_accuracy  0.00326949   0.943612  113"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gains/Lift Table: Avg response rate:  0.20 %\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>group</b></td>\n",
       "<td><b>cumulative_data_fraction</b></td>\n",
       "<td><b>lower_threshold</b></td>\n",
       "<td><b>lift</b></td>\n",
       "<td><b>cumulative_lift</b></td>\n",
       "<td><b>response_rate</b></td>\n",
       "<td><b>cumulative_response_rate</b></td>\n",
       "<td><b>capture_rate</b></td>\n",
       "<td><b>cumulative_capture_rate</b></td>\n",
       "<td><b>gain</b></td>\n",
       "<td><b>cumulative_gain</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>1</td>\n",
       "<td>0.0106002</td>\n",
       "<td>0.0041997</td>\n",
       "<td>94.3374170</td>\n",
       "<td>94.3374170</td>\n",
       "<td>0.1878509</td>\n",
       "<td>0.1878509</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>9333.7417050</td>\n",
       "<td>9333.7417050</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2</td>\n",
       "<td>0.0209732</td>\n",
       "<td>0.0029308</td>\n",
       "<td>0.0</td>\n",
       "<td>47.6798246</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0949432</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>4667.9824561</td></tr>\n",
       "<tr><td></td>\n",
       "<td>3</td>\n",
       "<td>0.0300205</td>\n",
       "<td>0.0023612</td>\n",
       "<td>0.0</td>\n",
       "<td>33.3105624</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0663302</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>3231.0562365</td></tr>\n",
       "<tr><td></td>\n",
       "<td>4</td>\n",
       "<td>0.0415190</td>\n",
       "<td>0.0020115</td>\n",
       "<td>0.0</td>\n",
       "<td>24.0853643</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0479604</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>2308.5364264</td></tr>\n",
       "<tr><td></td>\n",
       "<td>5</td>\n",
       "<td>0.0526387</td>\n",
       "<td>0.0017336</td>\n",
       "<td>0.0</td>\n",
       "<td>18.9974301</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0378289</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>1799.7430099</td></tr>\n",
       "<tr><td></td>\n",
       "<td>6</td>\n",
       "<td>0.1094601</td>\n",
       "<td>0.0009723</td>\n",
       "<td>0.0</td>\n",
       "<td>9.1357457</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0181917</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>813.5745712</td></tr>\n",
       "<tr><td></td>\n",
       "<td>7</td>\n",
       "<td>0.1661896</td>\n",
       "<td>0.0006277</td>\n",
       "<td>0.0</td>\n",
       "<td>6.0172240</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0119819</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>501.7223977</td></tr>\n",
       "<tr><td></td>\n",
       "<td>8</td>\n",
       "<td>0.2001223</td>\n",
       "<td>0.0004142</td>\n",
       "<td>0.0</td>\n",
       "<td>4.9969446</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0099502</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>399.6944625</td></tr>\n",
       "<tr><td></td>\n",
       "<td>9</td>\n",
       "<td>0.3816252</td>\n",
       "<td>0.0002008</td>\n",
       "<td>0.0</td>\n",
       "<td>2.6203723</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0052179</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>162.0372340</td></tr>\n",
       "<tr><td></td>\n",
       "<td>10</td>\n",
       "<td>0.5631984</td>\n",
       "<td>0.0000994</td>\n",
       "<td>0.0</td>\n",
       "<td>1.7755733</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0035356</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>77.5573341</td></tr>\n",
       "<tr><td></td>\n",
       "<td>11</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0019913</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>0.0</td></tr></table></div>"
      ],
      "text/plain": [
       "    group    cumulative_data_fraction    lower_threshold    lift     cumulative_lift    response_rate    cumulative_response_rate    capture_rate    cumulative_capture_rate    gain     cumulative_gain\n",
       "--  -------  --------------------------  -----------------  -------  -----------------  ---------------  --------------------------  --------------  -------------------------  -------  -----------------\n",
       "    1        0.0106002                   0.00419971         94.3374  94.3374            0.187851         0.187851                    1               1                          9333.74  9333.74\n",
       "    2        0.0209732                   0.0029308          0        47.6798            0                0.0949432                   0               1                          -100     4667.98\n",
       "    3        0.0300205                   0.00236115         0        33.3106            0                0.0663302                   0               1                          -100     3231.06\n",
       "    4        0.041519                    0.0020115          0        24.0854            0                0.0479604                   0               1                          -100     2308.54\n",
       "    5        0.0526387                   0.00173358         0        18.9974            0                0.0378289                   0               1                          -100     1799.74\n",
       "    6        0.10946                     0.000972305        0        9.13575            0                0.0181917                   0               1                          -100     813.575\n",
       "    7        0.16619                     0.000627732        0        6.01722            0                0.0119819                   0               1                          -100     501.722\n",
       "    8        0.200122                    0.000414172        0        4.99694            0                0.00995025                  0               1                          -100     399.694\n",
       "    9        0.381625                    0.000200789        0        2.62037            0                0.00521786                  0               1                          -100     162.037\n",
       "    10       0.563198                    9.93906e-05        0        1.77557            0                0.00353564                  0               1                          -100     77.5573\n",
       "    11       1                           0                  0        1                  0                0.00199127                  0               1                          -100     0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Scoring History: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>timestamp</b></td>\n",
       "<td><b>duration</b></td>\n",
       "<td><b>number_of_trees</b></td>\n",
       "<td><b>training_rmse</b></td>\n",
       "<td><b>training_logloss</b></td>\n",
       "<td><b>training_auc</b></td>\n",
       "<td><b>training_lift</b></td>\n",
       "<td><b>training_classification_error</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:47</td>\n",
       "<td> 0.016 sec</td>\n",
       "<td>0.0</td>\n",
       "<td>nan</td>\n",
       "<td>nan</td>\n",
       "<td>nan</td>\n",
       "<td>nan</td>\n",
       "<td>nan</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:48</td>\n",
       "<td> 0.377 sec</td>\n",
       "<td>1.0</td>\n",
       "<td>0.2224971</td>\n",
       "<td>1.7098404</td>\n",
       "<td>0.9100569</td>\n",
       "<td>32.0528971</td>\n",
       "<td>0.0495050</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:48</td>\n",
       "<td> 0.722 sec</td>\n",
       "<td>2.0</td>\n",
       "<td>0.2348486</td>\n",
       "<td>1.7479393</td>\n",
       "<td>0.9104336</td>\n",
       "<td>15.3402987</td>\n",
       "<td>0.0552571</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:48</td>\n",
       "<td> 1.100 sec</td>\n",
       "<td>3.0</td>\n",
       "<td>0.2152835</td>\n",
       "<td>1.3503858</td>\n",
       "<td>0.9241747</td>\n",
       "<td>86.8416636</td>\n",
       "<td>0.0453154</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:49</td>\n",
       "<td> 1.527 sec</td>\n",
       "<td>4.0</td>\n",
       "<td>0.2092365</td>\n",
       "<td>1.1632445</td>\n",
       "<td>0.9320237</td>\n",
       "<td>52.3117640</td>\n",
       "<td>0.0429815</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:49</td>\n",
       "<td> 1.860 sec</td>\n",
       "<td>5.0</td>\n",
       "<td>0.2006902</td>\n",
       "<td>1.0067153</td>\n",
       "<td>0.9383918</td>\n",
       "<td>41.7350686</td>\n",
       "<td>0.0381098</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:50</td>\n",
       "<td> 2.436 sec</td>\n",
       "<td>6.0</td>\n",
       "<td>0.1939296</td>\n",
       "<td>0.8733814</td>\n",
       "<td>0.9446291</td>\n",
       "<td>94.6992391</td>\n",
       "<td>0.0351048</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:50</td>\n",
       "<td> 3.007 sec</td>\n",
       "<td>7.0</td>\n",
       "<td>0.1890470</td>\n",
       "<td>0.7502821</td>\n",
       "<td>0.9503181</td>\n",
       "<td>69.6222023</td>\n",
       "<td>0.0328885</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:51</td>\n",
       "<td> 4.104 sec</td>\n",
       "<td>8.0</td>\n",
       "<td>0.1874718</td>\n",
       "<td>0.6969191</td>\n",
       "<td>0.9510172</td>\n",
       "<td>53.6605691</td>\n",
       "<td>0.0324096</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:57:54</td>\n",
       "<td> 6.331 sec</td>\n",
       "<td>100.0</td>\n",
       "<td>0.2137265</td>\n",
       "<td>0.2900709</td>\n",
       "<td>0.9774801</td>\n",
       "<td>94.3374170</td>\n",
       "<td>0.0215300</td></tr></table></div>"
      ],
      "text/plain": [
       "    timestamp            duration    number_of_trees    training_rmse    training_logloss    training_auc    training_lift    training_classification_error\n",
       "--  -------------------  ----------  -----------------  ---------------  ------------------  --------------  ---------------  -------------------------------\n",
       "    2017-05-26 14:57:47  0.016 sec   0                  nan              nan                 nan             nan              nan\n",
       "    2017-05-26 14:57:48  0.377 sec   1                  0.222497         1.70984             0.910057        32.0529          0.049505\n",
       "    2017-05-26 14:57:48  0.722 sec   2                  0.234849         1.74794             0.910434        15.3403          0.0552571\n",
       "    2017-05-26 14:57:48  1.100 sec   3                  0.215284         1.35039             0.924175        86.8417          0.0453154\n",
       "    2017-05-26 14:57:49  1.527 sec   4                  0.209237         1.16324             0.932024        52.3118          0.0429815\n",
       "    2017-05-26 14:57:49  1.860 sec   5                  0.20069          1.00672             0.938392        41.7351          0.0381098\n",
       "    2017-05-26 14:57:50  2.436 sec   6                  0.19393          0.873381            0.944629        94.6992          0.0351048\n",
       "    2017-05-26 14:57:50  3.007 sec   7                  0.189047         0.750282            0.950318        69.6222          0.0328885\n",
       "    2017-05-26 14:57:51  4.104 sec   8                  0.187472         0.696919            0.951017        53.6606          0.0324096\n",
       "    2017-05-26 14:57:54  6.331 sec   100                0.213726         0.290071            0.97748         94.3374          0.02153"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable Importances: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>variable</b></td>\n",
       "<td><b>relative_importance</b></td>\n",
       "<td><b>scaled_importance</b></td>\n",
       "<td><b>percentage</b></td></tr>\n",
       "<tr><td>V14</td>\n",
       "<td>9970.6777344</td>\n",
       "<td>1.0</td>\n",
       "<td>0.1825004</td></tr>\n",
       "<tr><td>outlier_autoenc</td>\n",
       "<td>9615.8710938</td>\n",
       "<td>0.9644150</td>\n",
       "<td>0.1760062</td></tr>\n",
       "<tr><td>V10</td>\n",
       "<td>8583.5136719</td>\n",
       "<td>0.8608756</td>\n",
       "<td>0.1571102</td></tr>\n",
       "<tr><td>V12</td>\n",
       "<td>4823.6054688</td>\n",
       "<td>0.4837791</td>\n",
       "<td>0.0882899</td></tr>\n",
       "<tr><td>V17</td>\n",
       "<td>3984.6643066</td>\n",
       "<td>0.3996383</td>\n",
       "<td>0.0729342</td></tr>\n",
       "<tr><td>---</td>\n",
       "<td>---</td>\n",
       "<td>---</td>\n",
       "<td>---</td></tr>\n",
       "<tr><td>V21</td>\n",
       "<td>171.2130127</td>\n",
       "<td>0.0171717</td>\n",
       "<td>0.0031338</td></tr>\n",
       "<tr><td>V8</td>\n",
       "<td>157.4195404</td>\n",
       "<td>0.0157882</td>\n",
       "<td>0.0028814</td></tr>\n",
       "<tr><td>V27</td>\n",
       "<td>151.6410675</td>\n",
       "<td>0.0152087</td>\n",
       "<td>0.0027756</td></tr>\n",
       "<tr><td>outlier_pca</td>\n",
       "<td>114.8507614</td>\n",
       "<td>0.0115189</td>\n",
       "<td>0.0021022</td></tr>\n",
       "<tr><td>amount</td>\n",
       "<td>108.3716888</td>\n",
       "<td>0.0108690</td>\n",
       "<td>0.0019836</td></tr></table></div>"
      ],
      "text/plain": [
       "variable         relative_importance    scaled_importance     percentage\n",
       "---------------  ---------------------  --------------------  ---------------------\n",
       "V14              9970.677734375         1.0                   0.1825004446382428\n",
       "outlier_autoenc  9615.87109375          0.9644149926336737    0.1760061649714331\n",
       "V10              8583.513671875         0.8608756496343674    0.15711018883650815\n",
       "V12              4823.60546875          0.4837790967929987    0.08828990027140976\n",
       "V17              3984.664306640625      0.3996382605871474    0.07293416025160834\n",
       "---              ---                    ---                   ---\n",
       "V21              171.2130126953125      0.01717165244495236   0.0031338342063771743\n",
       "V8               157.41954040527344     0.015788248762925353  0.0028813624192930635\n",
       "V27              151.6410675048828      0.015208702110798715  0.0027755948975913473\n",
       "outlier_pca      114.85076141357422     0.011518852025235324  0.00210219561632757\n",
       "amount        108.37168884277344     0.010869039370227583  0.0019836045178571004"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "See the whole table with table.as_data_frame()\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(model_outliers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Supervised model with outliers ensemble as one feature"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model with only one additional column, the column describing the ensemble of outliers methods (aggregated)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_ensemble = model(**modelargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "drf Model Build progress: |███████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "model_ensemble.train(x                = X_fraud_ensemble_outlier, \n",
    "                     y                = y_fraud, \n",
    "                     training_frame   = df_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Details\n",
      "=============\n",
      "H2ORandomForestEstimator :  Distributed Random Forest\n",
      "Model Key:  DRF_model_python_1495800034122_152\n",
      "\n",
      "\n",
      "ModelMetricsBinomial: drf\n",
      "** Reported on train data. **\n",
      "\n",
      "MSE: 0.044973618853376904\n",
      "RMSE: 0.21206984428102196\n",
      "LogLoss: 0.3274650960273324\n",
      "Mean Per-Class Error: 0.05631841923251213\n",
      "AUC: 0.9723967898113478\n",
      "Gini: 0.9447935796226956\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.00671732746794254: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>0</b></td>\n",
       "<td><b>1</b></td>\n",
       "<td><b>Error</b></td>\n",
       "<td><b>Rate</b></td></tr>\n",
       "<tr><td>0</td>\n",
       "<td>1815.0</td>\n",
       "<td>8.0</td>\n",
       "<td>0.0044</td>\n",
       "<td> (8.0/1823.0)</td></tr>\n",
       "<tr><td>1</td>\n",
       "<td>41.0</td>\n",
       "<td>327.0</td>\n",
       "<td>0.1114</td>\n",
       "<td> (41.0/368.0)</td></tr>\n",
       "<tr><td>Total</td>\n",
       "<td>1856.0</td>\n",
       "<td>335.0</td>\n",
       "<td>0.0224</td>\n",
       "<td> (49.0/2191.0)</td></tr></table></div>"
      ],
      "text/plain": [
       "       0     1    Error    Rate\n",
       "-----  ----  ---  -------  -------------\n",
       "0      1815  8    0.0044   (8.0/1823.0)\n",
       "1      41    327  0.1114   (41.0/368.0)\n",
       "Total  1856  335  0.0224   (49.0/2191.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum Metrics: Maximum metrics at their respective thresholds\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>metric</b></td>\n",
       "<td><b>threshold</b></td>\n",
       "<td><b>value</b></td>\n",
       "<td><b>idx</b></td></tr>\n",
       "<tr><td>max f1</td>\n",
       "<td>0.0067173</td>\n",
       "<td>0.9302987</td>\n",
       "<td>69.0</td></tr>\n",
       "<tr><td>max f2</td>\n",
       "<td>0.0035813</td>\n",
       "<td>0.9066084</td>\n",
       "<td>92.0</td></tr>\n",
       "<tr><td>max f0point5</td>\n",
       "<td>0.0200181</td>\n",
       "<td>0.9638554</td>\n",
       "<td>58.0</td></tr>\n",
       "<tr><td>max accuracy</td>\n",
       "<td>0.0067173</td>\n",
       "<td>0.9776358</td>\n",
       "<td>69.0</td></tr>\n",
       "<tr><td>max precision</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max recall</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>224.0</td></tr>\n",
       "<tr><td>max specificity</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td></tr>\n",
       "<tr><td>max absolute_mcc</td>\n",
       "<td>0.0067173</td>\n",
       "<td>0.9184503</td>\n",
       "<td>69.0</td></tr>\n",
       "<tr><td>max min_per_class_accuracy</td>\n",
       "<td>0.0015917</td>\n",
       "<td>0.9293478</td>\n",
       "<td>133.0</td></tr>\n",
       "<tr><td>max mean_per_class_accuracy</td>\n",
       "<td>0.0035813</td>\n",
       "<td>0.9436816</td>\n",
       "<td>92.0</td></tr></table></div>"
      ],
      "text/plain": [
       "metric                       threshold    value     idx\n",
       "---------------------------  -----------  --------  -----\n",
       "max f1                       0.00671733   0.930299  69\n",
       "max f2                       0.00358132   0.906608  92\n",
       "max f0point5                 0.0200181    0.963855  58\n",
       "max accuracy                 0.00671733   0.977636  69\n",
       "max precision                1            1         0\n",
       "max recall                   0            1         224\n",
       "max specificity              1            1         0\n",
       "max absolute_mcc             0.00671733   0.91845   69\n",
       "max min_per_class_accuracy   0.00159166   0.929348  133\n",
       "max mean_per_class_accuracy  0.00358132   0.943682  92"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gains/Lift Table: Avg response rate:  0.20 %\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>group</b></td>\n",
       "<td><b>cumulative_data_fraction</b></td>\n",
       "<td><b>lower_threshold</b></td>\n",
       "<td><b>lift</b></td>\n",
       "<td><b>cumulative_lift</b></td>\n",
       "<td><b>response_rate</b></td>\n",
       "<td><b>cumulative_response_rate</b></td>\n",
       "<td><b>capture_rate</b></td>\n",
       "<td><b>cumulative_capture_rate</b></td>\n",
       "<td><b>gain</b></td>\n",
       "<td><b>cumulative_gain</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>1</td>\n",
       "<td>0.0102702</td>\n",
       "<td>0.0050660</td>\n",
       "<td>97.3693361</td>\n",
       "<td>97.3693361</td>\n",
       "<td>0.1938883</td>\n",
       "<td>0.1938883</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>9636.9336143</td>\n",
       "<td>9636.9336143</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2</td>\n",
       "<td>0.0201778</td>\n",
       "<td>0.0031116</td>\n",
       "<td>0.0</td>\n",
       "<td>49.5593993</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0986860</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>4855.9399303</td></tr>\n",
       "<tr><td></td>\n",
       "<td>3</td>\n",
       "<td>0.0300584</td>\n",
       "<td>0.0023131</td>\n",
       "<td>0.0</td>\n",
       "<td>33.2685869</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0662466</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>3226.8586859</td></tr>\n",
       "<tr><td></td>\n",
       "<td>4</td>\n",
       "<td>0.0400093</td>\n",
       "<td>0.0017754</td>\n",
       "<td>0.0</td>\n",
       "<td>24.9941845</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0497701</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>2399.4184474</td></tr>\n",
       "<tr><td></td>\n",
       "<td>5</td>\n",
       "<td>0.0543594</td>\n",
       "<td>0.0014747</td>\n",
       "<td>0.0</td>\n",
       "<td>18.3960780</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0366315</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>1739.6078041</td></tr>\n",
       "<tr><td></td>\n",
       "<td>6</td>\n",
       "<td>0.1038651</td>\n",
       "<td>0.0008587</td>\n",
       "<td>0.0</td>\n",
       "<td>9.6278718</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0191717</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>862.7871842</td></tr>\n",
       "<tr><td></td>\n",
       "<td>7</td>\n",
       "<td>0.1732077</td>\n",
       "<td>0.0005199</td>\n",
       "<td>0.0</td>\n",
       "<td>5.7734146</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0114964</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>477.3414558</td></tr>\n",
       "<tr><td></td>\n",
       "<td>8</td>\n",
       "<td>0.2126380</td>\n",
       "<td>0.0004117</td>\n",
       "<td>0.0</td>\n",
       "<td>4.7028272</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0093646</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>370.2827188</td></tr>\n",
       "<tr><td></td>\n",
       "<td>9</td>\n",
       "<td>0.3607223</td>\n",
       "<td>0.0002017</td>\n",
       "<td>0.0</td>\n",
       "<td>2.7722159</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0055202</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>177.2215889</td></tr>\n",
       "<tr><td></td>\n",
       "<td>10</td>\n",
       "<td>0.5339300</td>\n",
       "<td>0.0000998</td>\n",
       "<td>0.0</td>\n",
       "<td>1.8729047</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0037295</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>87.2904717</td></tr>\n",
       "<tr><td></td>\n",
       "<td>11</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0019913</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>-100.0</td>\n",
       "<td>0.0</td></tr></table></div>"
      ],
      "text/plain": [
       "    group    cumulative_data_fraction    lower_threshold    lift     cumulative_lift    response_rate    cumulative_response_rate    capture_rate    cumulative_capture_rate    gain     cumulative_gain\n",
       "--  -------  --------------------------  -----------------  -------  -----------------  ---------------  --------------------------  --------------  -------------------------  -------  -----------------\n",
       "    1        0.0102702                   0.00506598         97.3693  97.3693            0.193888         0.193888                    1               1                          9636.93  9636.93\n",
       "    2        0.0201778                   0.00311156         0        49.5594            0                0.098686                    0               1                          -100     4855.94\n",
       "    3        0.0300584                   0.00231311         0        33.2686            0                0.0662466                   0               1                          -100     3226.86\n",
       "    4        0.0400093                   0.00177538         0        24.9942            0                0.0497701                   0               1                          -100     2399.42\n",
       "    5        0.0543594                   0.00147475         0        18.3961            0                0.0366315                   0               1                          -100     1739.61\n",
       "    6        0.103865                    0.000858743        0        9.62787            0                0.0191717                   0               1                          -100     862.787\n",
       "    7        0.173208                    0.000519941        0        5.77341            0                0.0114964                   0               1                          -100     477.341\n",
       "    8        0.212638                    0.000411665        0        4.70283            0                0.00936458                  0               1                          -100     370.283\n",
       "    9        0.360722                    0.000201674        0        2.77222            0                0.00552022                  0               1                          -100     177.222\n",
       "    10       0.53393                     9.98287e-05        0        1.8729             0                0.00372945                  0               1                          -100     87.2905\n",
       "    11       1                           0                  0        1                  0                0.00199127                  0               1                          -100     0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Scoring History: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>timestamp</b></td>\n",
       "<td><b>duration</b></td>\n",
       "<td><b>number_of_trees</b></td>\n",
       "<td><b>training_rmse</b></td>\n",
       "<td><b>training_logloss</b></td>\n",
       "<td><b>training_auc</b></td>\n",
       "<td><b>training_lift</b></td>\n",
       "<td><b>training_classification_error</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:58:02</td>\n",
       "<td> 0.012 sec</td>\n",
       "<td>0.0</td>\n",
       "<td>nan</td>\n",
       "<td>nan</td>\n",
       "<td>nan</td>\n",
       "<td>nan</td>\n",
       "<td>nan</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:58:03</td>\n",
       "<td> 0.273 sec</td>\n",
       "<td>1.0</td>\n",
       "<td>0.2445998</td>\n",
       "<td>2.0664225</td>\n",
       "<td>0.8845361</td>\n",
       "<td>32.5291670</td>\n",
       "<td>0.0598291</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:58:03</td>\n",
       "<td> 0.381 sec</td>\n",
       "<td>2.0</td>\n",
       "<td>0.1968104</td>\n",
       "<td>1.1850626</td>\n",
       "<td>0.9352942</td>\n",
       "<td>17.5036178</td>\n",
       "<td>0.0388350</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:58:03</td>\n",
       "<td> 0.786 sec</td>\n",
       "<td>3.0</td>\n",
       "<td>0.1919085</td>\n",
       "<td>1.0595719</td>\n",
       "<td>0.9362557</td>\n",
       "<td>83.1882243</td>\n",
       "<td>0.0345874</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:58:03</td>\n",
       "<td> 1.050 sec</td>\n",
       "<td>4.0</td>\n",
       "<td>0.1968289</td>\n",
       "<td>1.0856047</td>\n",
       "<td>0.9329026</td>\n",
       "<td>53.0557944</td>\n",
       "<td>0.0373579</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:58:04</td>\n",
       "<td> 1.534 sec</td>\n",
       "<td>5.0</td>\n",
       "<td>0.1902667</td>\n",
       "<td>0.9853576</td>\n",
       "<td>0.9357905</td>\n",
       "<td>42.3033513</td>\n",
       "<td>0.0344304</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:58:04</td>\n",
       "<td> 1.869 sec</td>\n",
       "<td>6.0</td>\n",
       "<td>0.1939991</td>\n",
       "<td>0.9736490</td>\n",
       "<td>0.9367745</td>\n",
       "<td>93.3530636</td>\n",
       "<td>0.0346004</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:58:05</td>\n",
       "<td> 2.347 sec</td>\n",
       "<td>7.0</td>\n",
       "<td>0.1890958</td>\n",
       "<td>0.9174780</td>\n",
       "<td>0.9369380</td>\n",
       "<td>78.2191820</td>\n",
       "<td>0.0318896</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:58:05</td>\n",
       "<td> 2.797 sec</td>\n",
       "<td>8.0</td>\n",
       "<td>0.1887521</td>\n",
       "<td>0.9050501</td>\n",
       "<td>0.9376778</td>\n",
       "<td>60.6009278</td>\n",
       "<td>0.0332397</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:58:06</td>\n",
       "<td> 3.381 sec</td>\n",
       "<td>9.0</td>\n",
       "<td>0.1885705</td>\n",
       "<td>0.8397979</td>\n",
       "<td>0.9402836</td>\n",
       "<td>96.4019224</td>\n",
       "<td>0.0310185</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:58:06</td>\n",
       "<td> 3.979 sec</td>\n",
       "<td>10.0</td>\n",
       "<td>0.1900648</td>\n",
       "<td>0.7827913</td>\n",
       "<td>0.9448273</td>\n",
       "<td>84.7798036</td>\n",
       "<td>0.0331034</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-26 14:58:09</td>\n",
       "<td> 6.087 sec</td>\n",
       "<td>100.0</td>\n",
       "<td>0.2120698</td>\n",
       "<td>0.3274651</td>\n",
       "<td>0.9723968</td>\n",
       "<td>97.3693361</td>\n",
       "<td>0.0223642</td></tr></table></div>"
      ],
      "text/plain": [
       "    timestamp            duration    number_of_trees    training_rmse    training_logloss    training_auc    training_lift    training_classification_error\n",
       "--  -------------------  ----------  -----------------  ---------------  ------------------  --------------  ---------------  -------------------------------\n",
       "    2017-05-26 14:58:02  0.012 sec   0                  nan              nan                 nan             nan              nan\n",
       "    2017-05-26 14:58:03  0.273 sec   1                  0.2446           2.06642             0.884536        32.5292          0.0598291\n",
       "    2017-05-26 14:58:03  0.381 sec   2                  0.19681          1.18506             0.935294        17.5036          0.038835\n",
       "    2017-05-26 14:58:03  0.786 sec   3                  0.191908         1.05957             0.936256        83.1882          0.0345874\n",
       "    2017-05-26 14:58:03  1.050 sec   4                  0.196829         1.0856              0.932903        53.0558          0.0373579\n",
       "    2017-05-26 14:58:04  1.534 sec   5                  0.190267         0.985358            0.93579         42.3034          0.0344304\n",
       "    2017-05-26 14:58:04  1.869 sec   6                  0.193999         0.973649            0.936774        93.3531          0.0346004\n",
       "    2017-05-26 14:58:05  2.347 sec   7                  0.189096         0.917478            0.936938        78.2192          0.0318896\n",
       "    2017-05-26 14:58:05  2.797 sec   8                  0.188752         0.90505             0.937678        60.6009          0.0332397\n",
       "    2017-05-26 14:58:06  3.381 sec   9                  0.18857          0.839798            0.940284        96.4019          0.0310185\n",
       "    2017-05-26 14:58:06  3.979 sec   10                 0.190065         0.782791            0.944827        84.7798          0.0331034\n",
       "    2017-05-26 14:58:09  6.087 sec   100                0.21207          0.327465            0.972397        97.3693          0.0223642"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable Importances: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>variable</b></td>\n",
       "<td><b>relative_importance</b></td>\n",
       "<td><b>scaled_importance</b></td>\n",
       "<td><b>percentage</b></td></tr>\n",
       "<tr><td>V14</td>\n",
       "<td>14256.3271484</td>\n",
       "<td>1.0</td>\n",
       "<td>0.2646294</td></tr>\n",
       "<tr><td>ensemble_outlier</td>\n",
       "<td>8259.9560547</td>\n",
       "<td>0.5793888</td>\n",
       "<td>0.1533233</td></tr>\n",
       "<tr><td>V17</td>\n",
       "<td>5569.2075195</td>\n",
       "<td>0.3906481</td>\n",
       "<td>0.1033770</td></tr>\n",
       "<tr><td>V10</td>\n",
       "<td>5567.4355469</td>\n",
       "<td>0.3905238</td>\n",
       "<td>0.1033441</td></tr>\n",
       "<tr><td>V16</td>\n",
       "<td>3150.1193848</td>\n",
       "<td>0.2209629</td>\n",
       "<td>0.0584733</td></tr>\n",
       "<tr><td>---</td>\n",
       "<td>---</td>\n",
       "<td>---</td>\n",
       "<td>---</td></tr>\n",
       "<tr><td>V1</td>\n",
       "<td>171.1941681</td>\n",
       "<td>0.0120083</td>\n",
       "<td>0.0031777</td></tr>\n",
       "<tr><td>V27</td>\n",
       "<td>161.5181427</td>\n",
       "<td>0.0113296</td>\n",
       "<td>0.0029981</td></tr>\n",
       "<tr><td>V23</td>\n",
       "<td>150.9775085</td>\n",
       "<td>0.0105902</td>\n",
       "<td>0.0028025</td></tr>\n",
       "<tr><td>amount</td>\n",
       "<td>114.4634705</td>\n",
       "<td>0.0080290</td>\n",
       "<td>0.0021247</td></tr>\n",
       "<tr><td>V28</td>\n",
       "<td>109.8055267</td>\n",
       "<td>0.0077022</td>\n",
       "<td>0.0020382</td></tr></table></div>"
      ],
      "text/plain": [
       "variable          relative_importance    scaled_importance     percentage\n",
       "----------------  ---------------------  --------------------  ---------------------\n",
       "V14               14256.3271484375       1.0                   0.26462936536209164\n",
       "ensemble_outlier  8259.9560546875        0.5793887842699229    0.15332328627926353\n",
       "V17               5569.20751953125       0.39064812847968616   0.10337696631946819\n",
       "V10               5567.435546875         0.3905238347090817    0.10334407453783467\n",
       "V16               3150.119384765625      0.22096289962810509   0.05847327189715301\n",
       "---               ---                    ---                   ---\n",
       "V1                171.1941680908203      0.012008294023302016  0.003177747226467811\n",
       "V27               161.5181427001953      0.011329576055491808  0.002998138521386347\n",
       "V23               150.97750854492188     0.010590210716472585  0.0028024807409509625\n",
       "amount         114.46347045898438     0.008028959301171033  0.0021246984043869536\n",
       "V28               109.80552673339844     0.00770223112798258   0.0020382365352701773"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "See the whole table with table.as_data_frame()\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(model_ensemble)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Models list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "models = collections.OrderedDict([[\"model_simple\", model_simple],\n",
    "                                  [\"model_outliers\", model_outliers],\n",
    "                                  [\"model_ensemble\", model_ensemble]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here, we go to an overview of all results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "––––––––––––––– model_simple –––––––––––––––\n",
      "Model Summary: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>number_of_trees</b></td>\n",
       "<td><b>number_of_internal_trees</b></td>\n",
       "<td><b>model_size_in_bytes</b></td>\n",
       "<td><b>min_depth</b></td>\n",
       "<td><b>max_depth</b></td>\n",
       "<td><b>mean_depth</b></td>\n",
       "<td><b>min_leaves</b></td>\n",
       "<td><b>max_leaves</b></td>\n",
       "<td><b>mean_leaves</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>100.0</td>\n",
       "<td>200.0</td>\n",
       "<td>180901.0</td>\n",
       "<td>11.0</td>\n",
       "<td>20.0</td>\n",
       "<td>16.425</td>\n",
       "<td>49.0</td>\n",
       "<td>82.0</td>\n",
       "<td>66.22</td></tr></table></div>"
      ],
      "text/plain": [
       "    number_of_trees    number_of_internal_trees    model_size_in_bytes    min_depth    max_depth    mean_depth    min_leaves    max_leaves    mean_leaves\n",
       "--  -----------------  --------------------------  ---------------------  -----------  -----------  ------------  ------------  ------------  -------------\n",
       "    100                200                         180901                 11           20           16.425        49            82            66.22"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "––––––––––––––– model_outliers –––––––––––––––\n",
      "Model Summary: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>number_of_trees</b></td>\n",
       "<td><b>number_of_internal_trees</b></td>\n",
       "<td><b>model_size_in_bytes</b></td>\n",
       "<td><b>min_depth</b></td>\n",
       "<td><b>max_depth</b></td>\n",
       "<td><b>mean_depth</b></td>\n",
       "<td><b>min_leaves</b></td>\n",
       "<td><b>max_leaves</b></td>\n",
       "<td><b>mean_leaves</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>100.0</td>\n",
       "<td>200.0</td>\n",
       "<td>177878.0</td>\n",
       "<td>11.0</td>\n",
       "<td>20.0</td>\n",
       "<td>16.185</td>\n",
       "<td>45.0</td>\n",
       "<td>79.0</td>\n",
       "<td>64.91</td></tr></table></div>"
      ],
      "text/plain": [
       "    number_of_trees    number_of_internal_trees    model_size_in_bytes    min_depth    max_depth    mean_depth    min_leaves    max_leaves    mean_leaves\n",
       "--  -----------------  --------------------------  ---------------------  -----------  -----------  ------------  ------------  ------------  -------------\n",
       "    100                200                         177878                 11           20           16.185        45            79            64.91"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "––––––––––––––– model_ensemble –––––––––––––––\n",
      "Model Summary: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>number_of_trees</b></td>\n",
       "<td><b>number_of_internal_trees</b></td>\n",
       "<td><b>model_size_in_bytes</b></td>\n",
       "<td><b>min_depth</b></td>\n",
       "<td><b>max_depth</b></td>\n",
       "<td><b>mean_depth</b></td>\n",
       "<td><b>min_leaves</b></td>\n",
       "<td><b>max_leaves</b></td>\n",
       "<td><b>mean_leaves</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>100.0</td>\n",
       "<td>200.0</td>\n",
       "<td>171279.0</td>\n",
       "<td>12.0</td>\n",
       "<td>20.0</td>\n",
       "<td>16.435</td>\n",
       "<td>50.0</td>\n",
       "<td>75.0</td>\n",
       "<td>62.34</td></tr></table></div>"
      ],
      "text/plain": [
       "    number_of_trees    number_of_internal_trees    model_size_in_bytes    min_depth    max_depth    mean_depth    min_leaves    max_leaves    mean_leaves\n",
       "--  -----------------  --------------------------  ---------------------  -----------  -----------  ------------  ------------  ------------  -------------\n",
       "    100                200                         171279                 12           20           16.435        50            75            62.34"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for m in models:\n",
    "    print(\"––––––––––––––– {} –––––––––––––––\".format(m))\n",
    "    print(models[m].summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confusion matrix cheatsheet\n",
    "![alt text](https://i.stack.imgur.com/ysM0Z.png \"CM\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "––––––––––––––– model_simple –––––––––––––––\n",
      "None\n",
      "––––––––––––––– model_outliers –––––––––––––––\n",
      "None\n",
      "––––––––––––––– model_ensemble –––––––––––––––\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "for m in models:\n",
    "    print(\"––––––––––––––– {} –––––––––––––––\".format(m))\n",
    "    print(models[m].confusion_matrix(valid=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variable importance plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "––––––––––––––––––––––––––––––––– model_simple –––––––––––––––––––––––––––––––––\n",
      "     variable  relative_importance  scaled_importance  percentage\n",
      "0         V17         10052.874023           1.000000    0.189170\n",
      "1         V14          9777.984375           0.972656    0.183998\n",
      "2         V10          6137.512207           0.610523    0.115493\n",
      "3         V12          4681.534180           0.465691    0.088095\n",
      "4         V11          4517.455078           0.449370    0.085007\n",
      "5         V16          3652.042725           0.363283    0.068722\n",
      "6          V3          2865.723633           0.285065    0.053926\n",
      "7          V4          2013.787598           0.200320    0.037895\n",
      "8         V18          1692.546143           0.168364    0.031850\n",
      "9          V7          1452.721680           0.144508    0.027337\n",
      "10         V9          1107.131836           0.110131    0.020833\n",
      "11         V2           681.501953           0.067792    0.012824\n",
      "12         V6           592.541321           0.058942    0.011150\n",
      "13        V21           388.864685           0.038682    0.007317\n",
      "14        V13           385.463684           0.038344    0.007253\n",
      "15         V8           373.458252           0.037149    0.007028\n",
      "16        V19           332.142639           0.033040    0.006250\n",
      "17        V15           330.755280           0.032902    0.006224\n",
      "18        V25           278.350922           0.027689    0.005238\n",
      "19        V26           228.526047           0.022732    0.004300\n",
      "20        V23           227.726852           0.022653    0.004285\n",
      "21        V20           216.396729           0.021526    0.004072\n",
      "22        V22           209.246674           0.020815    0.003938\n",
      "23         V5           183.432968           0.018247    0.003452\n",
      "24        V27           175.501358           0.017458    0.003303\n",
      "25  amount           171.279373           0.017038    0.003223\n",
      "26         V1           146.836502           0.014606    0.002763\n",
      "27        V24           136.229507           0.013551    0.002564\n",
      "28        V28           132.345978           0.013165    0.002490\n",
      "––––––––––––––––––––––––––––––––– model_outliers –––––––––––––––––––––––––––––––––\n",
      "           variable  relative_importance  scaled_importance  percentage\n",
      "0               V14          9970.677734           1.000000    0.182500\n",
      "1   outlier_autoenc          9615.871094           0.964415    0.176006\n",
      "2               V10          8583.513672           0.860876    0.157110\n",
      "3               V12          4823.605469           0.483779    0.088290\n",
      "4               V17          3984.664307           0.399638    0.072934\n",
      "5    outlier_zscore          3214.454102           0.322391    0.058836\n",
      "6               V11          2485.380127           0.249269    0.045492\n",
      "7                V7          1979.128906           0.198495    0.036225\n",
      "8               V18          1651.078247           0.165593    0.030221\n",
      "9               V16          1644.652222           0.164949    0.030103\n",
      "10               V4          1409.576538           0.141372    0.025800\n",
      "11               V2           771.570007           0.077384    0.014123\n",
      "12              V19           407.941864           0.040914    0.007467\n",
      "13               V5           392.634796           0.039379    0.007187\n",
      "14              V13           332.599121           0.033358    0.006088\n",
      "15               V3           293.265686           0.029413    0.005368\n",
      "16               V9           259.927612           0.026069    0.004758\n",
      "17              V23           259.372040           0.026013    0.004747\n",
      "18              V15           249.873611           0.025061    0.004574\n",
      "19              V26           243.339554           0.024406    0.004454\n",
      "20              V24           218.050552           0.021869    0.003991\n",
      "21              V25           217.099915           0.021774    0.003974\n",
      "22              V20           204.186401           0.020479    0.003737\n",
      "23              V22           193.549377           0.019412    0.003543\n",
      "24               V6           177.729843           0.017825    0.003253\n",
      "25               V1           173.692795           0.017420    0.003179\n",
      "26              V28           172.785828           0.017329    0.003163\n",
      "27              V21           171.213013           0.017172    0.003134\n",
      "28               V8           157.419540           0.015788    0.002881\n",
      "29              V27           151.641068           0.015209    0.002776\n",
      "30      outlier_pca           114.850761           0.011519    0.002102\n",
      "31        amount           108.371689           0.010869    0.001984\n",
      "––––––––––––––––––––––––––––––––– model_ensemble –––––––––––––––––––––––––––––––––\n",
      "            variable  relative_importance  scaled_importance  percentage\n",
      "0                V14         14256.327148           1.000000    0.264629\n",
      "1   ensemble_outlier          8259.956055           0.579389    0.153323\n",
      "2                V17          5569.207520           0.390648    0.103377\n",
      "3                V10          5567.435547           0.390524    0.103344\n",
      "4                V16          3150.119385           0.220963    0.058473\n",
      "5                V11          2421.482666           0.169853    0.044948\n",
      "6                V12          2281.550537           0.160038    0.042351\n",
      "7                 V3          2270.889160           0.159290    0.042153\n",
      "8                 V7          1696.169922           0.118977    0.031485\n",
      "9                 V4          1379.582642           0.096770    0.025608\n",
      "10               V18          1137.457397           0.079786    0.021114\n",
      "11                V2           755.460083           0.052991    0.014023\n",
      "12                V5           679.437866           0.047659    0.012612\n",
      "13               V21           586.181519           0.041117    0.010881\n",
      "14                V9           473.372131           0.033204    0.008787\n",
      "15               V19           451.845306           0.031694    0.008387\n",
      "16                V6           437.120117           0.030661    0.008114\n",
      "17               V13           252.111084           0.017684    0.004680\n",
      "18               V20           250.042068           0.017539    0.004641\n",
      "19                V8           243.786148           0.017100    0.004525\n",
      "20               V15           222.533356           0.015609    0.004131\n",
      "21               V24           222.279633           0.015592    0.004126\n",
      "22               V26           214.083008           0.015017    0.003974\n",
      "23               V25           212.468826           0.014903    0.003944\n",
      "24               V22           173.950653           0.012202    0.003229\n",
      "25                V1           171.194168           0.012008    0.003178\n",
      "26               V27           161.518143           0.011330    0.002998\n",
      "27               V23           150.977509           0.010590    0.002802\n",
      "28         amount           114.463470           0.008029    0.002125\n",
      "29               V28           109.805527           0.007702    0.002038\n"
     ]
    }
   ],
   "source": [
    "for m in models:\n",
    "    print(\"––––––––––––––––––––––––––––––––– {} –––––––––––––––––––––––––––––––––\".format(m))\n",
    "    print(models[m].varimp(use_pandas=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "data": [
        {
         "orientation": "h",
         "type": "bar",
         "x": [
          0.013164989183655211,
          0.013551299571513627,
          0.014606420187187996,
          0.017037851341777554,
          0.017457829236003423,
          0.018246818542835476,
          0.020814612129440985,
          0.021525856984889367,
          0.022652910191261186,
          0.02273240928118057,
          0.027688690913852662,
          0.03290156414671916,
          0.0330395704139723,
          0.03714940136347435,
          0.0383436302079736,
          0.03868194151761832,
          0.05894247947595055,
          0.0677917530385968,
          0.11013087733481068,
          0.14450809552577618,
          0.1683644039139538,
          0.200319589498611,
          0.2850651093539307,
          0.3632834467133398,
          0.44936951040994866,
          0.4656911216407233,
          0.6105231392258685,
          0.9726556159167402,
          1
         ],
         "y": [
          "V28",
          "V24",
          "V1",
          "amount",
          "V27",
          "V5",
          "V22",
          "V20",
          "V23",
          "V26",
          "V25",
          "V15",
          "V19",
          "V8",
          "V13",
          "V21",
          "V6",
          "V2",
          "V9",
          "V7",
          "V18",
          "V4",
          "V3",
          "V16",
          "V11",
          "V12",
          "V10",
          "V14",
          "V17"
         ]
        }
       ],
       "layout": {
        "barmode": "group",
        "margin": {
         "b": 100,
         "l": 150,
         "pad": 4,
         "r": 0,
         "t": 100
        },
        "title": "Features importance: model_simple",
        "xaxis": {
         "title": "Scaled importance"
        },
        "yaxis": {
         "title": " "
        }
       }
      },
      "text/html": [
       "<div id=\"6b811f6c-e0b8-4534-a7d3-95b0f4177a2c\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";Plotly.newPlot(\"6b811f6c-e0b8-4534-a7d3-95b0f4177a2c\", [{\"type\": \"bar\", \"y\": [\"V28\", \"V24\", \"V1\", \"amount\", \"V27\", \"V5\", \"V22\", \"V20\", \"V23\", \"V26\", \"V25\", \"V15\", \"V19\", \"V8\", \"V13\", \"V21\", \"V6\", \"V2\", \"V9\", \"V7\", \"V18\", \"V4\", \"V3\", \"V16\", \"V11\", \"V12\", \"V10\", \"V14\", \"V17\"], \"x\": [0.013164989183655211, 0.013551299571513627, 0.014606420187187996, 0.017037851341777554, 0.017457829236003423, 0.018246818542835476, 0.020814612129440985, 0.021525856984889367, 0.022652910191261186, 0.02273240928118057, 0.027688690913852662, 0.03290156414671916, 0.0330395704139723, 0.03714940136347435, 0.0383436302079736, 0.03868194151761832, 0.05894247947595055, 0.0677917530385968, 0.11013087733481068, 0.14450809552577618, 0.1683644039139538, 0.200319589498611, 0.2850651093539307, 0.3632834467133398, 0.44936951040994866, 0.4656911216407233, 0.6105231392258685, 0.9726556159167402, 1.0], \"orientation\": \"h\"}], {\"title\": \"Features importance: model_simple\", \"xaxis\": {\"title\": \"Scaled importance\"}, \"yaxis\": {\"title\": \" \"}, \"barmode\": \"group\", \"margin\": {\"l\": 150, \"r\": 0, \"b\": 100, \"t\": 100, \"pad\": 4}}, {\"showLink\": true, \"linkText\": \"Export to plot.ly\"})});</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<div id=\"6b811f6c-e0b8-4534-a7d3-95b0f4177a2c\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";Plotly.newPlot(\"6b811f6c-e0b8-4534-a7d3-95b0f4177a2c\", [{\"type\": \"bar\", \"y\": [\"V28\", \"V24\", \"V1\", \"amount\", \"V27\", \"V5\", \"V22\", \"V20\", \"V23\", \"V26\", \"V25\", \"V15\", \"V19\", \"V8\", \"V13\", \"V21\", \"V6\", \"V2\", \"V9\", \"V7\", \"V18\", \"V4\", \"V3\", \"V16\", \"V11\", \"V12\", \"V10\", \"V14\", \"V17\"], \"x\": [0.013164989183655211, 0.013551299571513627, 0.014606420187187996, 0.017037851341777554, 0.017457829236003423, 0.018246818542835476, 0.020814612129440985, 0.021525856984889367, 0.022652910191261186, 0.02273240928118057, 0.027688690913852662, 0.03290156414671916, 0.0330395704139723, 0.03714940136347435, 0.0383436302079736, 0.03868194151761832, 0.05894247947595055, 0.0677917530385968, 0.11013087733481068, 0.14450809552577618, 0.1683644039139538, 0.200319589498611, 0.2850651093539307, 0.3632834467133398, 0.44936951040994866, 0.4656911216407233, 0.6105231392258685, 0.9726556159167402, 1.0], \"orientation\": \"h\"}], {\"title\": \"Features importance: model_simple\", \"xaxis\": {\"title\": \"Scaled importance\"}, \"yaxis\": {\"title\": \" \"}, \"barmode\": \"group\", \"margin\": {\"l\": 150, \"r\": 0, \"b\": 100, \"t\": 100, \"pad\": 4}}, {\"showLink\": true, \"linkText\": \"Export to plot.ly\"})});</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<script>function downloadimage(format, height, width, filename) {var p = document.getElementById('6b811f6c-e0b8-4534-a7d3-95b0f4177a2c');Plotly.downloadImage(p, {format: format, height: height, width: width, filename: filename});};if(document.readyState == 'complete') {{downloadimage('png', 540, 960, 'features_importance:_model_simple');}}</script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "data": [
        {
         "orientation": "h",
         "type": "bar",
         "x": [
          0.010869039370227583,
          0.011518852025235324,
          0.015208702110798715,
          0.015788248762925353,
          0.01717165244495236,
          0.017329396480343633,
          0.01742035992207228,
          0.0178252519913371,
          0.019411857708942257,
          0.02047868829048928,
          0.0217738372791155,
          0.021869180629157332,
          0.024405517891132728,
          0.025060845221055412,
          0.02601348140063824,
          0.026069202037145245,
          0.029412813636939723,
          0.03335772451526321,
          0.039378947610444456,
          0.04091415597630317,
          0.07738390788262536,
          0.14137218909666174,
          0.16494888968375435,
          0.16559338202036558,
          0.19849492270989133,
          0.2492689256603396,
          0.32239073282654784,
          0.3996382605871474,
          0.4837790967929987,
          0.8608756496343674,
          0.9644149926336737,
          1
         ],
         "y": [
          "amount",
          "outlier_pca",
          "V27",
          "V8",
          "V21",
          "V28",
          "V1",
          "V6",
          "V22",
          "V20",
          "V25",
          "V24",
          "V26",
          "V15",
          "V23",
          "V9",
          "V3",
          "V13",
          "V5",
          "V19",
          "V2",
          "V4",
          "V16",
          "V18",
          "V7",
          "V11",
          "outlier_zscore",
          "V17",
          "V12",
          "V10",
          "outlier_autoenc",
          "V14"
         ]
        }
       ],
       "layout": {
        "barmode": "group",
        "margin": {
         "b": 100,
         "l": 150,
         "pad": 4,
         "r": 0,
         "t": 100
        },
        "title": "Features importance: model_outliers",
        "xaxis": {
         "title": "Scaled importance"
        },
        "yaxis": {
         "title": " "
        }
       }
      },
      "text/html": [
       "<div id=\"7f3d6615-45cd-4c57-b85f-d737c6adfca0\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";Plotly.newPlot(\"7f3d6615-45cd-4c57-b85f-d737c6adfca0\", [{\"type\": \"bar\", \"y\": [\"amount\", \"outlier_pca\", \"V27\", \"V8\", \"V21\", \"V28\", \"V1\", \"V6\", \"V22\", \"V20\", \"V25\", \"V24\", \"V26\", \"V15\", \"V23\", \"V9\", \"V3\", \"V13\", \"V5\", \"V19\", \"V2\", \"V4\", \"V16\", \"V18\", \"V7\", \"V11\", \"outlier_zscore\", \"V17\", \"V12\", \"V10\", \"outlier_autoenc\", \"V14\"], \"x\": [0.010869039370227583, 0.011518852025235324, 0.015208702110798715, 0.015788248762925353, 0.01717165244495236, 0.017329396480343633, 0.01742035992207228, 0.0178252519913371, 0.019411857708942257, 0.02047868829048928, 0.0217738372791155, 0.021869180629157332, 0.024405517891132728, 0.025060845221055412, 0.02601348140063824, 0.026069202037145245, 0.029412813636939723, 0.03335772451526321, 0.039378947610444456, 0.04091415597630317, 0.07738390788262536, 0.14137218909666174, 0.16494888968375435, 0.16559338202036558, 0.19849492270989133, 0.2492689256603396, 0.32239073282654784, 0.3996382605871474, 0.4837790967929987, 0.8608756496343674, 0.9644149926336737, 1.0], \"orientation\": \"h\"}], {\"title\": \"Features importance: model_outliers\", \"xaxis\": {\"title\": \"Scaled importance\"}, \"yaxis\": {\"title\": \" \"}, \"barmode\": \"group\", \"margin\": {\"l\": 150, \"r\": 0, \"b\": 100, \"t\": 100, \"pad\": 4}}, {\"showLink\": true, \"linkText\": \"Export to plot.ly\"})});</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<div id=\"7f3d6615-45cd-4c57-b85f-d737c6adfca0\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";Plotly.newPlot(\"7f3d6615-45cd-4c57-b85f-d737c6adfca0\", [{\"type\": \"bar\", \"y\": [\"amount\", \"outlier_pca\", \"V27\", \"V8\", \"V21\", \"V28\", \"V1\", \"V6\", \"V22\", \"V20\", \"V25\", \"V24\", \"V26\", \"V15\", \"V23\", \"V9\", \"V3\", \"V13\", \"V5\", \"V19\", \"V2\", \"V4\", \"V16\", \"V18\", \"V7\", \"V11\", \"outlier_zscore\", \"V17\", \"V12\", \"V10\", \"outlier_autoenc\", \"V14\"], \"x\": [0.010869039370227583, 0.011518852025235324, 0.015208702110798715, 0.015788248762925353, 0.01717165244495236, 0.017329396480343633, 0.01742035992207228, 0.0178252519913371, 0.019411857708942257, 0.02047868829048928, 0.0217738372791155, 0.021869180629157332, 0.024405517891132728, 0.025060845221055412, 0.02601348140063824, 0.026069202037145245, 0.029412813636939723, 0.03335772451526321, 0.039378947610444456, 0.04091415597630317, 0.07738390788262536, 0.14137218909666174, 0.16494888968375435, 0.16559338202036558, 0.19849492270989133, 0.2492689256603396, 0.32239073282654784, 0.3996382605871474, 0.4837790967929987, 0.8608756496343674, 0.9644149926336737, 1.0], \"orientation\": \"h\"}], {\"title\": \"Features importance: model_outliers\", \"xaxis\": {\"title\": \"Scaled importance\"}, \"yaxis\": {\"title\": \" \"}, \"barmode\": \"group\", \"margin\": {\"l\": 150, \"r\": 0, \"b\": 100, \"t\": 100, \"pad\": 4}}, {\"showLink\": true, \"linkText\": \"Export to plot.ly\"})});</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<script>function downloadimage(format, height, width, filename) {var p = document.getElementById('7f3d6615-45cd-4c57-b85f-d737c6adfca0');Plotly.downloadImage(p, {format: format, height: height, width: width, filename: filename});};if(document.readyState == 'complete') {{downloadimage('png', 540, 960, 'features_importance:_model_outliers');}}</script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "data": [
        {
         "orientation": "h",
         "type": "bar",
         "x": [
          0.00770223112798258,
          0.008028959301171033,
          0.010590210716472585,
          0.011329576055491808,
          0.012008294023302016,
          0.01220164571596808,
          0.014903475774770783,
          0.015016701397453803,
          0.01559164785249203,
          0.015609445083285732,
          0.017100207194530335,
          0.017539024313765837,
          0.017684153945078798,
          0.030661481925617046,
          0.03169436992374343,
          0.033204353857686136,
          0.041117288657263544,
          0.047658689305920154,
          0.052991214016199904,
          0.07978614587177199,
          0.09676985013301728,
          0.11897664133366223,
          0.159289916435815,
          0.1600377511931209,
          0.16985319155509282,
          0.22096289962810509,
          0.3905238347090817,
          0.39064812847968616,
          0.5793887842699229,
          1
         ],
         "y": [
          "V28",
          "amount",
          "V23",
          "V27",
          "V1",
          "V22",
          "V25",
          "V26",
          "V24",
          "V15",
          "V8",
          "V20",
          "V13",
          "V6",
          "V19",
          "V9",
          "V21",
          "V5",
          "V2",
          "V18",
          "V4",
          "V7",
          "V3",
          "V12",
          "V11",
          "V16",
          "V10",
          "V17",
          "ensemble_outlier",
          "V14"
         ]
        }
       ],
       "layout": {
        "barmode": "group",
        "margin": {
         "b": 100,
         "l": 150,
         "pad": 4,
         "r": 0,
         "t": 100
        },
        "title": "Features importance: model_ensemble",
        "xaxis": {
         "title": "Scaled importance"
        },
        "yaxis": {
         "title": " "
        }
       }
      },
      "text/html": [
       "<div id=\"281ee09f-eaca-48c6-9e72-102f0e91f294\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";Plotly.newPlot(\"281ee09f-eaca-48c6-9e72-102f0e91f294\", [{\"type\": \"bar\", \"y\": [\"V28\", \"amount\", \"V23\", \"V27\", \"V1\", \"V22\", \"V25\", \"V26\", \"V24\", \"V15\", \"V8\", \"V20\", \"V13\", \"V6\", \"V19\", \"V9\", \"V21\", \"V5\", \"V2\", \"V18\", \"V4\", \"V7\", \"V3\", \"V12\", \"V11\", \"V16\", \"V10\", \"V17\", \"ensemble_outlier\", \"V14\"], \"x\": [0.00770223112798258, 0.008028959301171033, 0.010590210716472585, 0.011329576055491808, 0.012008294023302016, 0.01220164571596808, 0.014903475774770783, 0.015016701397453803, 0.01559164785249203, 0.015609445083285732, 0.017100207194530335, 0.017539024313765837, 0.017684153945078798, 0.030661481925617046, 0.03169436992374343, 0.033204353857686136, 0.041117288657263544, 0.047658689305920154, 0.052991214016199904, 0.07978614587177199, 0.09676985013301728, 0.11897664133366223, 0.159289916435815, 0.1600377511931209, 0.16985319155509282, 0.22096289962810509, 0.3905238347090817, 0.39064812847968616, 0.5793887842699229, 1.0], \"orientation\": \"h\"}], {\"title\": \"Features importance: model_ensemble\", \"xaxis\": {\"title\": \"Scaled importance\"}, \"yaxis\": {\"title\": \" \"}, \"barmode\": \"group\", \"margin\": {\"l\": 150, \"r\": 0, \"b\": 100, \"t\": 100, \"pad\": 4}}, {\"showLink\": true, \"linkText\": \"Export to plot.ly\"})});</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<div id=\"281ee09f-eaca-48c6-9e72-102f0e91f294\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";Plotly.newPlot(\"281ee09f-eaca-48c6-9e72-102f0e91f294\", [{\"type\": \"bar\", \"y\": [\"V28\", \"amount\", \"V23\", \"V27\", \"V1\", \"V22\", \"V25\", \"V26\", \"V24\", \"V15\", \"V8\", \"V20\", \"V13\", \"V6\", \"V19\", \"V9\", \"V21\", \"V5\", \"V2\", \"V18\", \"V4\", \"V7\", \"V3\", \"V12\", \"V11\", \"V16\", \"V10\", \"V17\", \"ensemble_outlier\", \"V14\"], \"x\": [0.00770223112798258, 0.008028959301171033, 0.010590210716472585, 0.011329576055491808, 0.012008294023302016, 0.01220164571596808, 0.014903475774770783, 0.015016701397453803, 0.01559164785249203, 0.015609445083285732, 0.017100207194530335, 0.017539024313765837, 0.017684153945078798, 0.030661481925617046, 0.03169436992374343, 0.033204353857686136, 0.041117288657263544, 0.047658689305920154, 0.052991214016199904, 0.07978614587177199, 0.09676985013301728, 0.11897664133366223, 0.159289916435815, 0.1600377511931209, 0.16985319155509282, 0.22096289962810509, 0.3905238347090817, 0.39064812847968616, 0.5793887842699229, 1.0], \"orientation\": \"h\"}], {\"title\": \"Features importance: model_ensemble\", \"xaxis\": {\"title\": \"Scaled importance\"}, \"yaxis\": {\"title\": \" \"}, \"barmode\": \"group\", \"margin\": {\"l\": 150, \"r\": 0, \"b\": 100, \"t\": 100, \"pad\": 4}}, {\"showLink\": true, \"linkText\": \"Export to plot.ly\"})});</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<script>function downloadimage(format, height, width, filename) {var p = document.getElementById('281ee09f-eaca-48c6-9e72-102f0e91f294');Plotly.downloadImage(p, {format: format, height: height, width: width, filename: filename});};if(document.readyState == 'complete') {{downloadimage('png', 540, 960, 'features_importance:_model_ensemble');}}</script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for m in models:\n",
    "    datavar = []\n",
    "    titleimg = 'Features importance: ' + m\n",
    "    filenamei = titleimg.lower().replace(\" \", \"_\")\n",
    "    try:\n",
    "        varimpo_df = models[m].varimp(use_pandas=True)\n",
    "    except:\n",
    "        print(\"Unable to find variable importance for \" + m)\n",
    "    var_y = varimpo_df[\"variable\"]\n",
    "    var_x = varimpo_df[\"scaled_importance\"]\n",
    "    datavar.append(go.Bar(y=var_y[::-1],\n",
    "                          x=var_x[::-1],\n",
    "                          orientation = 'h'))\n",
    "\n",
    "    layout = go.Layout(title = titleimg,\n",
    "                  xaxis = dict(title = 'Scaled importance'),\n",
    "                  yaxis = dict(title = ' '),\n",
    "                  barmode='group',\n",
    "                  margin=go.Margin(l=150,\n",
    "                                   r=0,\n",
    "                                   b=100,\n",
    "                                   t=100,\n",
    "                                   pad=4)\n",
    "    )\n",
    "\n",
    "    fig = go.Figure(data=datavar, layout=layout)\n",
    "    plotly.offline.iplot(fig, filename=filenamei, image=IMAGE_FORMAT, image_height=IMAGE_HEIGT, image_width=IMAGE_WIDTH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ROC curves plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "data": [
        {
         "line": {
          "width": 2
         },
         "mode": "lines",
         "name": "model_simple (area = 0.9746)",
         "type": "scatter",
         "x": [
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001088139281828074,
          0.001632208922742111,
          0.001632208922742111,
          0.001632208922742111,
          0.002176278563656148,
          0.002720348204570185,
          0.002720348204570185,
          0.002720348204570185,
          0.002720348204570185,
          0.002720348204570185,
          0.003264417845484222,
          0.003264417845484222,
          0.003264417845484222,
          0.003808487486398259,
          0.004352557127312296,
          0.004352557127312296,
          0.004352557127312296,
          0.004352557127312296,
          0.004896626768226333,
          0.00544069640914037,
          0.00544069640914037,
          0.005984766050054407,
          0.006528835690968444,
          0.007072905331882481,
          0.007616974972796518,
          0.008705114254624592,
          0.008705114254624592,
          0.01088139281828074,
          0.01088139281828074,
          0.011425462459194777,
          0.011969532100108813,
          0.012513601741022852,
          0.012513601741022852,
          0.013057671381936888,
          0.013601741022850925,
          0.014689880304679,
          0.015778019586507073,
          0.01632208922742111,
          0.017410228509249184,
          0.017954298150163223,
          0.018498367791077257,
          0.019042437431991296,
          0.01958650707290533,
          0.02013057671381937,
          0.021218715995647442,
          0.02176278563656148,
          0.022306855277475515,
          0.022850924918389554,
          0.023394994559303592,
          0.023939064200217627,
          0.024483133841131665,
          0.025571273122959738,
          0.026115342763873776,
          0.02720348204570185,
          0.027747551686615888,
          0.029379760609358,
          0.029379760609358,
          0.030467899891186073,
          0.031011969532100107,
          0.031556039173014146,
          0.03264417845484222,
          0.03427638737758433,
          0.03482045701849837,
          0.03536452665941241,
          0.035908596300326445,
          0.03754080522306855,
          0.03808487486398259,
          0.0397170837867247,
          0.04026115342763874,
          0.040805223068552776,
          0.041349292709466814,
          0.04570184983677911,
          0.046245919477693145,
          0.04787812840043525,
          0.04842219804134929,
          0.04896626768226333,
          0.051142546245919476,
          0.05223068552774755,
          0.05331882480957562,
          0.05386289445048966,
          0.0544069640914037,
          0.05495103373231774,
          0.05603917301414581,
          0.057127312295973884,
          0.05767138193688792,
          0.058759521218716,
          0.05984766050054407,
          0.06039173014145811,
          0.060935799782372145,
          0.06147986942328618,
          0.062023939064200215,
          0.06311207834602829,
          0.06365614798694233,
          0.06420021762785637,
          0.06692056583242655,
          0.06746463547334058,
          0.06964091403699674,
          0.07181719260065289,
          0.07236126224156691,
          0.07290533188248095,
          0.07399347116430903,
          0.07725788900979326,
          0.07834602829162132,
          0.08161044613710555,
          0.08215451577801959,
          0.08324265505984767,
          0.08378672470076169,
          0.08705114254624592,
          0.088139281828074,
          0.09031556039173014,
          0.09303590859630033,
          0.09357997823721437,
          0.09466811751904244,
          0.09630032644178455,
          0.09793253536452666,
          0.0984766050054407,
          0.09956474428726877,
          0.10010881392818281,
          0.10282916213275299,
          0.10609357997823722,
          0.10663764961915125,
          0.10663764961915125,
          0.10935799782372144,
          0.11371055495103373,
          0.1147986942328618,
          0.117519042437432,
          0.1191512513601741,
          0.12023939064200218,
          0.12078346028291621,
          0.12132752992383025,
          0.1235038084874864,
          0.12459194776931447,
          0.12785636561479868,
          0.12840043525571274,
          0.132208922742111,
          0.13601741022850924,
          0.14145810663764963,
          0.14744287268770404,
          0.15505984766050054,
          0.16050054406964093,
          0.16648531011969533,
          0.1692056583242655,
          0.16974972796517954,
          0.17519042437431992,
          0.17573449401523394,
          0.17845484221980412,
          0.1822633297062024,
          0.1838955386289445,
          0.18443960826985853,
          0.18661588683351468,
          0.18715995647442873,
          0.18879216539717084,
          0.190968443960827,
          0.19205658324265507,
          0.19368879216539717,
          0.19477693144722524,
          0.1953210010881393,
          0.19586507072905332,
          0.19640914036996734,
          0.19749727965179542,
          0.19804134929270947,
          0.19967355821545157,
          0.20184983677910773,
          0.20511425462459196,
          0.2105549510337323,
          0.21545157780195864,
          0.22306855277475518,
          0.23177366702937977,
          0.249727965179543,
          0.2622415669205658,
          0.27094668117519044,
          0.28291621327529926,
          0.29651795429815014,
          0.31229597388465724,
          0.32100108813928185,
          0.3291621327529924,
          0.3373231773667029,
          0.34221980413492925,
          0.34766050054406966,
          0.3487486398258977,
          0.35038084874863984,
          0.3514689880304679,
          0.35310119695321,
          0.35364526659412404,
          0.35418933623503807,
          0.35473340587595215,
          0.35527747551686617,
          0.3558215451577802,
          0.3563656147986942,
          0.35690968443960824,
          0.3574537540805223,
          0.35799782372143635,
          1
         ],
         "y": [
          0.5625,
          0.5652173913043478,
          0.5679347826086957,
          0.5733695652173914,
          0.5760869565217391,
          0.5788043478260869,
          0.5815217391304348,
          0.5869565217391305,
          0.592391304347826,
          0.6032608695652174,
          0.6059782608695652,
          0.6114130434782609,
          0.6168478260869565,
          0.625,
          0.6331521739130435,
          0.6440217391304348,
          0.6603260869565217,
          0.6711956521739131,
          0.6766304347826086,
          0.6820652173913043,
          0.6902173913043478,
          0.6983695652173914,
          0.7092391304347826,
          0.7119565217391305,
          0.7146739130434783,
          0.720108695652174,
          0.7228260869565217,
          0.7309782608695652,
          0.7364130434782609,
          0.7418478260869565,
          0.7445652173913043,
          0.7554347826086957,
          0.7581521739130435,
          0.7663043478260869,
          0.7690217391304348,
          0.7717391304347826,
          0.7771739130434783,
          0.779891304347826,
          0.7853260869565217,
          0.7880434782608695,
          0.7989130434782609,
          0.8016304347826086,
          0.8043478260869565,
          0.8070652173913043,
          0.8097826086956522,
          0.8125,
          0.8152173913043478,
          0.8179347826086957,
          0.8206521739130435,
          0.8233695652173914,
          0.8260869565217391,
          0.8288043478260869,
          0.8342391304347826,
          0.8369565217391305,
          0.8396739130434783,
          0.842391304347826,
          0.845108695652174,
          0.8478260869565217,
          0.8505434782608695,
          0.8532608695652174,
          0.8559782608695652,
          0.8559782608695652,
          0.8586956521739131,
          0.8614130434782609,
          0.8614130434782609,
          0.8614130434782609,
          0.8641304347826086,
          0.8668478260869565,
          0.8695652173913043,
          0.8722826086956522,
          0.8722826086956522,
          0.875,
          0.8777173913043478,
          0.8777173913043478,
          0.8777173913043478,
          0.8804347826086957,
          0.8831521739130435,
          0.8858695652173914,
          0.8858695652173914,
          0.8858695652173914,
          0.8885869565217391,
          0.8885869565217391,
          0.8885869565217391,
          0.8885869565217391,
          0.8885869565217391,
          0.8913043478260869,
          0.8940217391304348,
          0.8967391304347826,
          0.8994565217391305,
          0.9021739130434783,
          0.9021739130434783,
          0.9021739130434783,
          0.904891304347826,
          0.904891304347826,
          0.904891304347826,
          0.907608695652174,
          0.9103260869565217,
          0.9103260869565217,
          0.9130434782608695,
          0.9130434782608695,
          0.9130434782608695,
          0.9130434782608695,
          0.9130434782608695,
          0.9130434782608695,
          0.9157608695652174,
          0.9157608695652174,
          0.9157608695652174,
          0.9184782608695652,
          0.9184782608695652,
          0.9184782608695652,
          0.9184782608695652,
          0.9184782608695652,
          0.9184782608695652,
          0.9184782608695652,
          0.9184782608695652,
          0.9184782608695652,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9239130434782609,
          0.9239130434782609,
          0.9239130434782609,
          0.9239130434782609,
          0.9239130434782609,
          0.9239130434782609,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9320652173913043,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9375,
          0.9375,
          0.9375,
          0.9375,
          0.9375,
          0.9375,
          0.9402173913043478,
          0.9402173913043478,
          0.9402173913043478,
          0.9402173913043478,
          0.9402173913043478,
          0.9402173913043478,
          0.9402173913043478,
          0.9402173913043478,
          0.9402173913043478,
          0.9402173913043478,
          0.9402173913043478,
          0.9402173913043478,
          0.9402173913043478,
          0.9402173913043478,
          0.9429347826086957,
          0.9429347826086957,
          0.9429347826086957,
          0.9483695652173914,
          0.9483695652173914,
          0.9538043478260869,
          0.9538043478260869,
          0.9538043478260869,
          0.9592391304347826,
          0.9592391304347826,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9646739130434783,
          0.9646739130434783,
          0.9646739130434783,
          0.967391304347826,
          0.967391304347826,
          0.967391304347826,
          0.967391304347826,
          0.967391304347826,
          0.967391304347826,
          0.970108695652174,
          0.9728260869565217,
          0.9728260869565217,
          0.9728260869565217,
          0.9728260869565217,
          0.9728260869565217,
          0.9728260869565217,
          0.9755434782608695,
          0.9755434782608695,
          0.9782608695652174,
          0.9782608695652174,
          0.9782608695652174,
          0.9782608695652174,
          0.9782608695652174,
          0.9782608695652174,
          0.9782608695652174,
          0.9782608695652174,
          0.9782608695652174,
          0.9782608695652174,
          0.9782608695652174,
          0.9782608695652174,
          0.9782608695652174,
          1
         ]
        },
        {
         "line": {
          "width": 2
         },
         "mode": "lines",
         "name": "model_outliers (area = 0.9775)",
         "type": "scatter",
         "x": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0.0005509641873278236,
          0.0005509641873278236,
          0.0005509641873278236,
          0.0011019283746556473,
          0.0011019283746556473,
          0.0011019283746556473,
          0.001652892561983471,
          0.001652892561983471,
          0.0022038567493112946,
          0.0022038567493112946,
          0.0027548209366391185,
          0.0027548209366391185,
          0.003305785123966942,
          0.003305785123966942,
          0.004407713498622589,
          0.0049586776859504135,
          0.005509641873278237,
          0.006060606060606061,
          0.007162534435261708,
          0.007713498622589532,
          0.008264462809917356,
          0.008264462809917356,
          0.009366391184573003,
          0.009917355371900827,
          0.01046831955922865,
          0.011570247933884297,
          0.012121212121212121,
          0.012672176308539946,
          0.012672176308539946,
          0.013223140495867768,
          0.014325068870523415,
          0.01487603305785124,
          0.015426997245179064,
          0.015426997245179064,
          0.015977961432506887,
          0.01652892561983471,
          0.017079889807162536,
          0.01818181818181818,
          0.01818181818181818,
          0.020385674931129475,
          0.021487603305785124,
          0.02203856749311295,
          0.023140495867768594,
          0.02534435261707989,
          0.025895316804407712,
          0.026446280991735537,
          0.02699724517906336,
          0.027548209366391185,
          0.029201101928374655,
          0.02975206611570248,
          0.030303030303030304,
          0.03085399449035813,
          0.03140495867768595,
          0.03140495867768595,
          0.031955922865013774,
          0.03360881542699724,
          0.03581267217630854,
          0.03581267217630854,
          0.03636363636363636,
          0.03801652892561983,
          0.03856749311294766,
          0.03856749311294766,
          0.03911845730027548,
          0.04022038567493113,
          0.04077134986225895,
          0.0418732782369146,
          0.04242424242424243,
          0.04297520661157025,
          0.04352617079889807,
          0.045179063360881545,
          0.045730027548209366,
          0.04628099173553719,
          0.046831955922865015,
          0.048484848484848485,
          0.049035812672176306,
          0.050137741046831955,
          0.0512396694214876,
          0.05234159779614325,
          0.05289256198347107,
          0.05454545454545454,
          0.05619834710743802,
          0.05674931129476584,
          0.05785123966942149,
          0.05950413223140496,
          0.06005509641873278,
          0.06115702479338843,
          0.06170798898071626,
          0.0628099173553719,
          0.06391184573002755,
          0.06501377410468319,
          0.06611570247933884,
          0.06666666666666667,
          0.06721763085399449,
          0.06831955922865014,
          0.07107438016528926,
          0.0721763085399449,
          0.07382920110192838,
          0.07493112947658402,
          0.07603305785123966,
          0.07713498622589532,
          0.07878787878787878,
          0.07933884297520662,
          0.08099173553719008,
          0.08429752066115702,
          0.08539944903581267,
          0.0859504132231405,
          0.08650137741046832,
          0.0881542699724518,
          0.09256198347107437,
          0.09476584022038567,
          0.09586776859504133,
          0.09696969696969697,
          0.09807162534435261,
          0.10027548209366391,
          0.10137741046831956,
          0.10358126721763085,
          0.10413223140495868,
          0.10578512396694215,
          0.10964187327823692,
          0.11349862258953168,
          0.1140495867768595,
          0.1162534435261708,
          0.11680440771349862,
          0.1206611570247934,
          0.12341597796143251,
          0.12617079889807162,
          0.12837465564738293,
          0.12892561983471074,
          0.12947658402203857,
          0.13112947658402205,
          0.1327823691460055,
          0.13388429752066117,
          0.13608815426997245,
          0.13663911845730028,
          0.1399449035812672,
          0.14380165289256197,
          0.14986225895316804,
          0.15592286501377411,
          0.15647382920110192,
          0.16363636363636364,
          0.17134986225895316,
          0.17851239669421487,
          0.18622589531680442,
          0.19063360881542699,
          0.19173553719008266,
          0.1933884297520661,
          0.19393939393939394,
          0.19504132231404958,
          0.19614325068870522,
          0.19669421487603306,
          0.1977961432506887,
          0.19834710743801653,
          0.19889807162534434,
          0.20055096418732782,
          0.20330578512396694,
          0.20606060606060606,
          0.20881542699724517,
          0.21101928374655649,
          0.21322314049586777,
          0.2236914600550964,
          0.23415977961432508,
          0.2440771349862259,
          0.2528925619834711,
          0.2705234159779614,
          0.28429752066115704,
          0.29641873278236913,
          0.31294765840220384,
          0.3261707988980716,
          0.3393939393939394,
          0.3504132231404959,
          0.36143250688705236,
          0.37245179063360884,
          0.3752066115702479,
          0.3768595041322314,
          0.3790633608815427,
          0.37961432506887055,
          0.38126721763085397,
          0.38236914600550964,
          0.38292011019283745,
          0.3834710743801653,
          0.3840220385674931,
          0.3845730027548209,
          0.3851239669421488,
          1
         ],
         "y": [
          0.4891304347826087,
          0.49184782608695654,
          0.4945652173913043,
          0.5,
          0.5190217391304348,
          0.5489130434782609,
          0.5597826086956522,
          0.5788043478260869,
          0.5978260869565217,
          0.6222826086956522,
          0.6413043478260869,
          0.6440217391304348,
          0.6603260869565217,
          0.6657608695652174,
          0.6711956521739131,
          0.6739130434782609,
          0.6766304347826086,
          0.6793478260869565,
          0.6820652173913043,
          0.6847826086956522,
          0.6875,
          0.6902173913043478,
          0.6929347826086957,
          0.6956521739130435,
          0.7065217391304348,
          0.7092391304347826,
          0.7146739130434783,
          0.720108695652174,
          0.7228260869565217,
          0.7282608695652174,
          0.7309782608695652,
          0.7336956521739131,
          0.7364130434782609,
          0.7418478260869565,
          0.7445652173913043,
          0.7472826086956522,
          0.75,
          0.7527173913043478,
          0.7554347826086957,
          0.7581521739130435,
          0.7608695652173914,
          0.7635869565217391,
          0.7663043478260869,
          0.7717391304347826,
          0.7744565217391305,
          0.7771739130434783,
          0.779891304347826,
          0.782608695652174,
          0.7880434782608695,
          0.7907608695652174,
          0.7934782608695652,
          0.7961956521739131,
          0.7989130434782609,
          0.8016304347826086,
          0.8043478260869565,
          0.8070652173913043,
          0.8097826086956522,
          0.8125,
          0.8152173913043478,
          0.8179347826086957,
          0.8233695652173914,
          0.8260869565217391,
          0.8288043478260869,
          0.8315217391304348,
          0.8369565217391305,
          0.8396739130434783,
          0.842391304347826,
          0.845108695652174,
          0.8478260869565217,
          0.8505434782608695,
          0.8532608695652174,
          0.8559782608695652,
          0.8586956521739131,
          0.8614130434782609,
          0.8614130434782609,
          0.8641304347826086,
          0.8668478260869565,
          0.8668478260869565,
          0.8695652173913043,
          0.8722826086956522,
          0.8722826086956522,
          0.875,
          0.875,
          0.8777173913043478,
          0.8804347826086957,
          0.8831521739130435,
          0.8858695652173914,
          0.8885869565217391,
          0.8885869565217391,
          0.8885869565217391,
          0.8885869565217391,
          0.8885869565217391,
          0.8885869565217391,
          0.8885869565217391,
          0.8885869565217391,
          0.8913043478260869,
          0.8913043478260869,
          0.8913043478260869,
          0.8913043478260869,
          0.8940217391304348,
          0.8940217391304348,
          0.8940217391304348,
          0.8967391304347826,
          0.8967391304347826,
          0.8994565217391305,
          0.8994565217391305,
          0.8994565217391305,
          0.9021739130434783,
          0.9021739130434783,
          0.9021739130434783,
          0.9021739130434783,
          0.9021739130434783,
          0.904891304347826,
          0.907608695652174,
          0.907608695652174,
          0.907608695652174,
          0.907608695652174,
          0.9103260869565217,
          0.9103260869565217,
          0.9130434782608695,
          0.9130434782608695,
          0.9130434782608695,
          0.9130434782608695,
          0.9130434782608695,
          0.9130434782608695,
          0.9130434782608695,
          0.9130434782608695,
          0.9157608695652174,
          0.9157608695652174,
          0.9157608695652174,
          0.9157608695652174,
          0.9184782608695652,
          0.9184782608695652,
          0.9184782608695652,
          0.9184782608695652,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9211956521739131,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9293478260869565,
          0.9293478260869565,
          0.9293478260869565,
          0.9320652173913043,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9375,
          0.9375,
          0.9375,
          0.9375,
          0.9375,
          0.9375,
          0.9375,
          0.9375,
          0.9375,
          0.9375,
          0.9375,
          0.9375,
          0.9375,
          0.9375,
          0.9375,
          0.9402173913043478,
          0.9402173913043478,
          0.9429347826086957,
          0.9456521739130435,
          0.9456521739130435,
          0.9483695652173914,
          0.9483695652173914,
          0.9483695652173914,
          0.9483695652173914,
          0.9483695652173914,
          0.9483695652173914,
          0.9483695652173914,
          0.9483695652173914,
          0.9483695652173914,
          0.9483695652173914,
          0.9483695652173914,
          0.9483695652173914,
          0.9483695652173914,
          0.9483695652173914,
          0.9483695652173914,
          0.9483695652173914,
          0.9483695652173914,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9538043478260869,
          0.9538043478260869,
          0.9538043478260869,
          0.9538043478260869,
          0.9538043478260869,
          0.9565217391304348,
          0.9565217391304348,
          0.9592391304347826,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9619565217391305,
          0.9646739130434783,
          0.970108695652174,
          0.970108695652174,
          0.9728260869565217,
          0.9782608695652174,
          0.9782608695652174,
          0.9782608695652174,
          0.9782608695652174,
          0.9782608695652174,
          0.9836956521739131,
          0.9836956521739131,
          0.9836956521739131,
          0.9836956521739131,
          0.9836956521739131,
          0.9836956521739131,
          0.9836956521739131,
          0.9836956521739131,
          0.9836956521739131,
          0.9836956521739131,
          0.9836956521739131,
          0.9836956521739131,
          0.9836956521739131,
          1
         ]
        },
        {
         "line": {
          "width": 2
         },
         "mode": "lines",
         "name": "model_ensemble (area = 0.9724)",
         "type": "scatter",
         "x": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0.0005485463521667581,
          0.0005485463521667581,
          0.0005485463521667581,
          0.0005485463521667581,
          0.0005485463521667581,
          0.0010970927043335162,
          0.0016456390565002743,
          0.0016456390565002743,
          0.0016456390565002743,
          0.0016456390565002743,
          0.0016456390565002743,
          0.0016456390565002743,
          0.0016456390565002743,
          0.0016456390565002743,
          0.0021941854086670325,
          0.0027427317608337905,
          0.0027427317608337905,
          0.0032912781130005485,
          0.0032912781130005485,
          0.0032912781130005485,
          0.0038398244651673065,
          0.0038398244651673065,
          0.0038398244651673065,
          0.004388370817334065,
          0.004388370817334065,
          0.004936917169500823,
          0.005485463521667581,
          0.006034009873834339,
          0.006034009873834339,
          0.006582556226001097,
          0.007131102578167855,
          0.007679648930334613,
          0.008228195282501372,
          0.00877674163466813,
          0.009325287986834888,
          0.009325287986834888,
          0.009873834339001646,
          0.009873834339001646,
          0.010422380691168404,
          0.010970927043335162,
          0.01151947339550192,
          0.012068019747668678,
          0.012616566099835436,
          0.012616566099835436,
          0.013165112452002194,
          0.013713658804168952,
          0.01426220515633571,
          0.014810751508502468,
          0.015907844212835986,
          0.016456390565002744,
          0.017004936917169502,
          0.018102029621503018,
          0.018650575973669776,
          0.019199122325836534,
          0.019747668678003292,
          0.02029621503017005,
          0.021393307734503566,
          0.021941854086670324,
          0.021941854086670324,
          0.022490400438837082,
          0.02303894679100384,
          0.023587493143170598,
          0.024136039495337356,
          0.025233132199670872,
          0.026330224904004388,
          0.026878771256171146,
          0.02852441031267142,
          0.030170049369171694,
          0.030718595721338452,
          0.031267142073505214,
          0.03181568842567197,
          0.033461327482172246,
          0.034009873834339004,
          0.034009873834339004,
          0.03455842018650576,
          0.03565551289083928,
          0.036204059243006036,
          0.03730115194733955,
          0.038946791003839826,
          0.039495337356006584,
          0.04004388370817334,
          0.041689522764673616,
          0.042238069116840374,
          0.04278661546900713,
          0.04333516182117389,
          0.04388370817334065,
          0.044980800877674164,
          0.04552934722984092,
          0.047174986286341196,
          0.04827207899067471,
          0.04882062534284147,
          0.050466264399341744,
          0.05156335710367526,
          0.05211190345584202,
          0.052660449808008776,
          0.055951727921009324,
          0.058145913329676356,
          0.058694459681843114,
          0.05924300603400987,
          0.060888645090510146,
          0.06472846955567746,
          0.06527701590784421,
          0.06966538672517827,
          0.0718595721338453,
          0.07405375754251234,
          0.07569939659901262,
          0.07569939659901262,
          0.07899067471201317,
          0.07953922106417992,
          0.08008776741634668,
          0.0811848601206802,
          0.08392759188151398,
          0.08447613823368075,
          0.08557323093801426,
          0.08612177729018101,
          0.08996160175534833,
          0.09051014810751508,
          0.09434997257268239,
          0.09764125068568294,
          0.09928688974218322,
          0.10312671420735052,
          0.10532089961601755,
          0.10970927043335162,
          0.11080636313768513,
          0.11354909489851893,
          0.11684037301151948,
          0.11738891936368623,
          0.11903455842018651,
          0.12122874382885354,
          0.12287438288535381,
          0.1256171146461876,
          0.12945693911135492,
          0.13384530992868898,
          0.13494240263302248,
          0.13823368074602305,
          0.1437191442676906,
          0.14810751508502468,
          0.1568842567196928,
          0.1634668129456939,
          0.16675809105869446,
          0.17334064728469556,
          0.17718047174986287,
          0.17992320351069666,
          0.18266593527153044,
          0.18376302797586397,
          0.18650575973669775,
          0.1870543060888645,
          0.18815139879319803,
          0.18869994514536478,
          0.18924849149753153,
          0.18979703784969829,
          0.19199122325836535,
          0.1936368623148656,
          0.19583104772353263,
          0.1963795940756994,
          0.19967087218869994,
          0.20625342841470104,
          0.2068019747668678,
          0.21228743828853539,
          0.21941854086670323,
          0.22654964344487108,
          0.23752057048820624,
          0.24958859023587493,
          0.2616566099835436,
          0.2781130005485464,
          0.290181020296215,
          0.3033461327482172,
          0.3165112452002194,
          0.3269336258913878,
          0.33461327482172243,
          0.34503565551289084,
          0.3505211190345584,
          0.35545803620405925,
          0.35765222161272625,
          0.3598464070213933,
          0.3620405924300603,
          0.3625891387822271,
          0.3631376851343939,
          0.3636862314865606,
          1
         ],
         "y": [
          0.6820652173913043,
          0.6847826086956522,
          0.6875,
          0.6902173913043478,
          0.7038043478260869,
          0.7119565217391305,
          0.7146739130434783,
          0.717391304347826,
          0.7228260869565217,
          0.7255434782608695,
          0.7309782608695652,
          0.7336956521739131,
          0.7364130434782609,
          0.7391304347826086,
          0.7445652173913043,
          0.7472826086956522,
          0.75,
          0.7527173913043478,
          0.7554347826086957,
          0.7581521739130435,
          0.7690217391304348,
          0.7717391304347826,
          0.7744565217391305,
          0.7771739130434783,
          0.779891304347826,
          0.782608695652174,
          0.7853260869565217,
          0.7880434782608695,
          0.7907608695652174,
          0.7934782608695652,
          0.7961956521739131,
          0.7989130434782609,
          0.8043478260869565,
          0.8070652173913043,
          0.8097826086956522,
          0.8125,
          0.8152173913043478,
          0.8179347826086957,
          0.8206521739130435,
          0.8233695652173914,
          0.8260869565217391,
          0.8288043478260869,
          0.8315217391304348,
          0.8342391304347826,
          0.8369565217391305,
          0.8369565217391305,
          0.8396739130434783,
          0.845108695652174,
          0.8478260869565217,
          0.8505434782608695,
          0.8505434782608695,
          0.8505434782608695,
          0.8532608695652174,
          0.8559782608695652,
          0.8586956521739131,
          0.8614130434782609,
          0.8641304347826086,
          0.8668478260869565,
          0.8695652173913043,
          0.8695652173913043,
          0.8695652173913043,
          0.8722826086956522,
          0.8722826086956522,
          0.875,
          0.8777173913043478,
          0.8777173913043478,
          0.8804347826086957,
          0.8831521739130435,
          0.8858695652173914,
          0.8885869565217391,
          0.8885869565217391,
          0.8885869565217391,
          0.8885869565217391,
          0.8913043478260869,
          0.8913043478260869,
          0.8913043478260869,
          0.8913043478260869,
          0.8913043478260869,
          0.8913043478260869,
          0.8913043478260869,
          0.8940217391304348,
          0.8940217391304348,
          0.8967391304347826,
          0.8967391304347826,
          0.8967391304347826,
          0.8967391304347826,
          0.8967391304347826,
          0.8967391304347826,
          0.8994565217391305,
          0.8994565217391305,
          0.8994565217391305,
          0.8994565217391305,
          0.9021739130434783,
          0.9021739130434783,
          0.9021739130434783,
          0.9021739130434783,
          0.9021739130434783,
          0.9021739130434783,
          0.9021739130434783,
          0.9021739130434783,
          0.9021739130434783,
          0.9021739130434783,
          0.9021739130434783,
          0.904891304347826,
          0.904891304347826,
          0.904891304347826,
          0.907608695652174,
          0.907608695652174,
          0.907608695652174,
          0.907608695652174,
          0.907608695652174,
          0.9103260869565217,
          0.9130434782608695,
          0.9130434782608695,
          0.9130434782608695,
          0.9130434782608695,
          0.9130434782608695,
          0.9130434782608695,
          0.9157608695652174,
          0.9157608695652174,
          0.9157608695652174,
          0.9184782608695652,
          0.9184782608695652,
          0.9184782608695652,
          0.9184782608695652,
          0.9184782608695652,
          0.9184782608695652,
          0.9239130434782609,
          0.9239130434782609,
          0.9239130434782609,
          0.9266304347826086,
          0.9266304347826086,
          0.9266304347826086,
          0.9293478260869565,
          0.9293478260869565,
          0.9293478260869565,
          0.9293478260869565,
          0.9293478260869565,
          0.9293478260869565,
          0.9293478260869565,
          0.9293478260869565,
          0.9293478260869565,
          0.9293478260869565,
          0.9293478260869565,
          0.9293478260869565,
          0.9293478260869565,
          0.9293478260869565,
          0.9293478260869565,
          0.9293478260869565,
          0.9293478260869565,
          0.9293478260869565,
          0.9320652173913043,
          0.9320652173913043,
          0.9320652173913043,
          0.9320652173913043,
          0.9320652173913043,
          0.9320652173913043,
          0.9320652173913043,
          0.9320652173913043,
          0.9320652173913043,
          0.9320652173913043,
          0.9320652173913043,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9347826086956522,
          0.9375,
          0.9375,
          0.9375,
          0.9375,
          0.9375,
          0.9375,
          0.9402173913043478,
          0.9402173913043478,
          0.9402173913043478,
          0.9402173913043478,
          0.9402173913043478,
          0.9483695652173914,
          0.9483695652173914,
          0.9483695652173914,
          0.9483695652173914,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9510869565217391,
          0.9565217391304348,
          0.9565217391304348,
          0.9592391304347826,
          0.9619565217391305,
          0.9646739130434783,
          0.967391304347826,
          0.970108695652174,
          0.9728260869565217,
          0.9728260869565217,
          0.9728260869565217,
          0.9728260869565217,
          0.9728260869565217,
          0.9728260869565217,
          0.9728260869565217,
          0.9728260869565217,
          0.9755434782608695,
          0.9755434782608695,
          0.9755434782608695,
          1
         ]
        },
        {
         "line": {
          "color": "navy",
          "dash": "dash",
          "width": 2
         },
         "mode": "lines",
         "showlegend": false,
         "type": "scatter",
         "x": [
          0,
          1
         ],
         "y": [
          0,
          1
         ]
        }
       ],
       "layout": {
        "title": "Receiver operating characteristic (ROC)",
        "xaxis": {
         "title": "False Positive Rate"
        },
        "yaxis": {
         "title": "True Positive Rate"
        }
       }
      },
      "text/html": [
       "<div id=\"880f329d-ae75-41b7-859d-9c3810a1ec2d\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";Plotly.newPlot(\"880f329d-ae75-41b7-859d-9c3810a1ec2d\", [{\"type\": \"scatter\", \"x\": [0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001632208922742111, 0.001632208922742111, 0.001632208922742111, 0.002176278563656148, 0.002720348204570185, 0.002720348204570185, 0.002720348204570185, 0.002720348204570185, 0.002720348204570185, 0.003264417845484222, 0.003264417845484222, 0.003264417845484222, 0.003808487486398259, 0.004352557127312296, 0.004352557127312296, 0.004352557127312296, 0.004352557127312296, 0.004896626768226333, 0.00544069640914037, 0.00544069640914037, 0.005984766050054407, 0.006528835690968444, 0.007072905331882481, 0.007616974972796518, 0.008705114254624592, 0.008705114254624592, 0.01088139281828074, 0.01088139281828074, 0.011425462459194777, 0.011969532100108813, 0.012513601741022852, 0.012513601741022852, 0.013057671381936888, 0.013601741022850925, 0.014689880304679, 0.015778019586507073, 0.01632208922742111, 0.017410228509249184, 0.017954298150163223, 0.018498367791077257, 0.019042437431991296, 0.01958650707290533, 0.02013057671381937, 0.021218715995647442, 0.02176278563656148, 0.022306855277475515, 0.022850924918389554, 0.023394994559303592, 0.023939064200217627, 0.024483133841131665, 0.025571273122959738, 0.026115342763873776, 0.02720348204570185, 0.027747551686615888, 0.029379760609358, 0.029379760609358, 0.030467899891186073, 0.031011969532100107, 0.031556039173014146, 0.03264417845484222, 0.03427638737758433, 0.03482045701849837, 0.03536452665941241, 0.035908596300326445, 0.03754080522306855, 0.03808487486398259, 0.0397170837867247, 0.04026115342763874, 0.040805223068552776, 0.041349292709466814, 0.04570184983677911, 0.046245919477693145, 0.04787812840043525, 0.04842219804134929, 0.04896626768226333, 0.051142546245919476, 0.05223068552774755, 0.05331882480957562, 0.05386289445048966, 0.0544069640914037, 0.05495103373231774, 0.05603917301414581, 0.057127312295973884, 0.05767138193688792, 0.058759521218716, 0.05984766050054407, 0.06039173014145811, 0.060935799782372145, 0.06147986942328618, 0.062023939064200215, 0.06311207834602829, 0.06365614798694233, 0.06420021762785637, 0.06692056583242655, 0.06746463547334058, 0.06964091403699674, 0.07181719260065289, 0.07236126224156691, 0.07290533188248095, 0.07399347116430903, 0.07725788900979326, 0.07834602829162132, 0.08161044613710555, 0.08215451577801959, 0.08324265505984767, 0.08378672470076169, 0.08705114254624592, 0.088139281828074, 0.09031556039173014, 0.09303590859630033, 0.09357997823721437, 0.09466811751904244, 0.09630032644178455, 0.09793253536452666, 0.0984766050054407, 0.09956474428726877, 0.10010881392818281, 0.10282916213275299, 0.10609357997823722, 0.10663764961915125, 0.10663764961915125, 0.10935799782372144, 0.11371055495103373, 0.1147986942328618, 0.117519042437432, 0.1191512513601741, 0.12023939064200218, 0.12078346028291621, 0.12132752992383025, 0.1235038084874864, 0.12459194776931447, 0.12785636561479868, 0.12840043525571274, 0.132208922742111, 0.13601741022850924, 0.14145810663764963, 0.14744287268770404, 0.15505984766050054, 0.16050054406964093, 0.16648531011969533, 0.1692056583242655, 0.16974972796517954, 0.17519042437431992, 0.17573449401523394, 0.17845484221980412, 0.1822633297062024, 0.1838955386289445, 0.18443960826985853, 0.18661588683351468, 0.18715995647442873, 0.18879216539717084, 0.190968443960827, 0.19205658324265507, 0.19368879216539717, 0.19477693144722524, 0.1953210010881393, 0.19586507072905332, 0.19640914036996734, 0.19749727965179542, 0.19804134929270947, 0.19967355821545157, 0.20184983677910773, 0.20511425462459196, 0.2105549510337323, 0.21545157780195864, 0.22306855277475518, 0.23177366702937977, 0.249727965179543, 0.2622415669205658, 0.27094668117519044, 0.28291621327529926, 0.29651795429815014, 0.31229597388465724, 0.32100108813928185, 0.3291621327529924, 0.3373231773667029, 0.34221980413492925, 0.34766050054406966, 0.3487486398258977, 0.35038084874863984, 0.3514689880304679, 0.35310119695321, 0.35364526659412404, 0.35418933623503807, 0.35473340587595215, 0.35527747551686617, 0.3558215451577802, 0.3563656147986942, 0.35690968443960824, 0.3574537540805223, 0.35799782372143635, 1.0], \"y\": [0.5625, 0.5652173913043478, 0.5679347826086957, 0.5733695652173914, 0.5760869565217391, 0.5788043478260869, 0.5815217391304348, 0.5869565217391305, 0.592391304347826, 0.6032608695652174, 0.6059782608695652, 0.6114130434782609, 0.6168478260869565, 0.625, 0.6331521739130435, 0.6440217391304348, 0.6603260869565217, 0.6711956521739131, 0.6766304347826086, 0.6820652173913043, 0.6902173913043478, 0.6983695652173914, 0.7092391304347826, 0.7119565217391305, 0.7146739130434783, 0.720108695652174, 0.7228260869565217, 0.7309782608695652, 0.7364130434782609, 0.7418478260869565, 0.7445652173913043, 0.7554347826086957, 0.7581521739130435, 0.7663043478260869, 0.7690217391304348, 0.7717391304347826, 0.7771739130434783, 0.779891304347826, 0.7853260869565217, 0.7880434782608695, 0.7989130434782609, 0.8016304347826086, 0.8043478260869565, 0.8070652173913043, 0.8097826086956522, 0.8125, 0.8152173913043478, 0.8179347826086957, 0.8206521739130435, 0.8233695652173914, 0.8260869565217391, 0.8288043478260869, 0.8342391304347826, 0.8369565217391305, 0.8396739130434783, 0.842391304347826, 0.845108695652174, 0.8478260869565217, 0.8505434782608695, 0.8532608695652174, 0.8559782608695652, 0.8559782608695652, 0.8586956521739131, 0.8614130434782609, 0.8614130434782609, 0.8614130434782609, 0.8641304347826086, 0.8668478260869565, 0.8695652173913043, 0.8722826086956522, 0.8722826086956522, 0.875, 0.8777173913043478, 0.8777173913043478, 0.8777173913043478, 0.8804347826086957, 0.8831521739130435, 0.8858695652173914, 0.8858695652173914, 0.8858695652173914, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8913043478260869, 0.8940217391304348, 0.8967391304347826, 0.8994565217391305, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.904891304347826, 0.904891304347826, 0.904891304347826, 0.907608695652174, 0.9103260869565217, 0.9103260869565217, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9157608695652174, 0.9157608695652174, 0.9157608695652174, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9239130434782609, 0.9239130434782609, 0.9239130434782609, 0.9239130434782609, 0.9239130434782609, 0.9239130434782609, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9320652173913043, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9429347826086957, 0.9429347826086957, 0.9429347826086957, 0.9483695652173914, 0.9483695652173914, 0.9538043478260869, 0.9538043478260869, 0.9538043478260869, 0.9592391304347826, 0.9592391304347826, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9646739130434783, 0.9646739130434783, 0.9646739130434783, 0.967391304347826, 0.967391304347826, 0.967391304347826, 0.967391304347826, 0.967391304347826, 0.967391304347826, 0.970108695652174, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9755434782608695, 0.9755434782608695, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 1.0], \"mode\": \"lines\", \"line\": {\"width\": 2}, \"name\": \"model_simple (area = 0.9746)\"}, {\"type\": \"scatter\", \"x\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0005509641873278236, 0.0005509641873278236, 0.0005509641873278236, 0.0011019283746556473, 0.0011019283746556473, 0.0011019283746556473, 0.001652892561983471, 0.001652892561983471, 0.0022038567493112946, 0.0022038567493112946, 0.0027548209366391185, 0.0027548209366391185, 0.003305785123966942, 0.003305785123966942, 0.004407713498622589, 0.0049586776859504135, 0.005509641873278237, 0.006060606060606061, 0.007162534435261708, 0.007713498622589532, 0.008264462809917356, 0.008264462809917356, 0.009366391184573003, 0.009917355371900827, 0.01046831955922865, 0.011570247933884297, 0.012121212121212121, 0.012672176308539946, 0.012672176308539946, 0.013223140495867768, 0.014325068870523415, 0.01487603305785124, 0.015426997245179064, 0.015426997245179064, 0.015977961432506887, 0.01652892561983471, 0.017079889807162536, 0.01818181818181818, 0.01818181818181818, 0.020385674931129475, 0.021487603305785124, 0.02203856749311295, 0.023140495867768594, 0.02534435261707989, 0.025895316804407712, 0.026446280991735537, 0.02699724517906336, 0.027548209366391185, 0.029201101928374655, 0.02975206611570248, 0.030303030303030304, 0.03085399449035813, 0.03140495867768595, 0.03140495867768595, 0.031955922865013774, 0.03360881542699724, 0.03581267217630854, 0.03581267217630854, 0.03636363636363636, 0.03801652892561983, 0.03856749311294766, 0.03856749311294766, 0.03911845730027548, 0.04022038567493113, 0.04077134986225895, 0.0418732782369146, 0.04242424242424243, 0.04297520661157025, 0.04352617079889807, 0.045179063360881545, 0.045730027548209366, 0.04628099173553719, 0.046831955922865015, 0.048484848484848485, 0.049035812672176306, 0.050137741046831955, 0.0512396694214876, 0.05234159779614325, 0.05289256198347107, 0.05454545454545454, 0.05619834710743802, 0.05674931129476584, 0.05785123966942149, 0.05950413223140496, 0.06005509641873278, 0.06115702479338843, 0.06170798898071626, 0.0628099173553719, 0.06391184573002755, 0.06501377410468319, 0.06611570247933884, 0.06666666666666667, 0.06721763085399449, 0.06831955922865014, 0.07107438016528926, 0.0721763085399449, 0.07382920110192838, 0.07493112947658402, 0.07603305785123966, 0.07713498622589532, 0.07878787878787878, 0.07933884297520662, 0.08099173553719008, 0.08429752066115702, 0.08539944903581267, 0.0859504132231405, 0.08650137741046832, 0.0881542699724518, 0.09256198347107437, 0.09476584022038567, 0.09586776859504133, 0.09696969696969697, 0.09807162534435261, 0.10027548209366391, 0.10137741046831956, 0.10358126721763085, 0.10413223140495868, 0.10578512396694215, 0.10964187327823692, 0.11349862258953168, 0.1140495867768595, 0.1162534435261708, 0.11680440771349862, 0.1206611570247934, 0.12341597796143251, 0.12617079889807162, 0.12837465564738293, 0.12892561983471074, 0.12947658402203857, 0.13112947658402205, 0.1327823691460055, 0.13388429752066117, 0.13608815426997245, 0.13663911845730028, 0.1399449035812672, 0.14380165289256197, 0.14986225895316804, 0.15592286501377411, 0.15647382920110192, 0.16363636363636364, 0.17134986225895316, 0.17851239669421487, 0.18622589531680442, 0.19063360881542699, 0.19173553719008266, 0.1933884297520661, 0.19393939393939394, 0.19504132231404958, 0.19614325068870522, 0.19669421487603306, 0.1977961432506887, 0.19834710743801653, 0.19889807162534434, 0.20055096418732782, 0.20330578512396694, 0.20606060606060606, 0.20881542699724517, 0.21101928374655649, 0.21322314049586777, 0.2236914600550964, 0.23415977961432508, 0.2440771349862259, 0.2528925619834711, 0.2705234159779614, 0.28429752066115704, 0.29641873278236913, 0.31294765840220384, 0.3261707988980716, 0.3393939393939394, 0.3504132231404959, 0.36143250688705236, 0.37245179063360884, 0.3752066115702479, 0.3768595041322314, 0.3790633608815427, 0.37961432506887055, 0.38126721763085397, 0.38236914600550964, 0.38292011019283745, 0.3834710743801653, 0.3840220385674931, 0.3845730027548209, 0.3851239669421488, 1.0], \"y\": [0.4891304347826087, 0.49184782608695654, 0.4945652173913043, 0.5, 0.5190217391304348, 0.5489130434782609, 0.5597826086956522, 0.5788043478260869, 0.5978260869565217, 0.6222826086956522, 0.6413043478260869, 0.6440217391304348, 0.6603260869565217, 0.6657608695652174, 0.6711956521739131, 0.6739130434782609, 0.6766304347826086, 0.6793478260869565, 0.6820652173913043, 0.6847826086956522, 0.6875, 0.6902173913043478, 0.6929347826086957, 0.6956521739130435, 0.7065217391304348, 0.7092391304347826, 0.7146739130434783, 0.720108695652174, 0.7228260869565217, 0.7282608695652174, 0.7309782608695652, 0.7336956521739131, 0.7364130434782609, 0.7418478260869565, 0.7445652173913043, 0.7472826086956522, 0.75, 0.7527173913043478, 0.7554347826086957, 0.7581521739130435, 0.7608695652173914, 0.7635869565217391, 0.7663043478260869, 0.7717391304347826, 0.7744565217391305, 0.7771739130434783, 0.779891304347826, 0.782608695652174, 0.7880434782608695, 0.7907608695652174, 0.7934782608695652, 0.7961956521739131, 0.7989130434782609, 0.8016304347826086, 0.8043478260869565, 0.8070652173913043, 0.8097826086956522, 0.8125, 0.8152173913043478, 0.8179347826086957, 0.8233695652173914, 0.8260869565217391, 0.8288043478260869, 0.8315217391304348, 0.8369565217391305, 0.8396739130434783, 0.842391304347826, 0.845108695652174, 0.8478260869565217, 0.8505434782608695, 0.8532608695652174, 0.8559782608695652, 0.8586956521739131, 0.8614130434782609, 0.8614130434782609, 0.8641304347826086, 0.8668478260869565, 0.8668478260869565, 0.8695652173913043, 0.8722826086956522, 0.8722826086956522, 0.875, 0.875, 0.8777173913043478, 0.8804347826086957, 0.8831521739130435, 0.8858695652173914, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8913043478260869, 0.8913043478260869, 0.8913043478260869, 0.8913043478260869, 0.8940217391304348, 0.8940217391304348, 0.8940217391304348, 0.8967391304347826, 0.8967391304347826, 0.8994565217391305, 0.8994565217391305, 0.8994565217391305, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.904891304347826, 0.907608695652174, 0.907608695652174, 0.907608695652174, 0.907608695652174, 0.9103260869565217, 0.9103260869565217, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9157608695652174, 0.9157608695652174, 0.9157608695652174, 0.9157608695652174, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9320652173913043, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9402173913043478, 0.9402173913043478, 0.9429347826086957, 0.9456521739130435, 0.9456521739130435, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9538043478260869, 0.9538043478260869, 0.9538043478260869, 0.9538043478260869, 0.9538043478260869, 0.9565217391304348, 0.9565217391304348, 0.9592391304347826, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9646739130434783, 0.970108695652174, 0.970108695652174, 0.9728260869565217, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 1.0], \"mode\": \"lines\", \"line\": {\"width\": 2}, \"name\": \"model_outliers (area = 0.9775)\"}, {\"type\": \"scatter\", \"x\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0005485463521667581, 0.0005485463521667581, 0.0005485463521667581, 0.0005485463521667581, 0.0005485463521667581, 0.0010970927043335162, 0.0016456390565002743, 0.0016456390565002743, 0.0016456390565002743, 0.0016456390565002743, 0.0016456390565002743, 0.0016456390565002743, 0.0016456390565002743, 0.0016456390565002743, 0.0021941854086670325, 0.0027427317608337905, 0.0027427317608337905, 0.0032912781130005485, 0.0032912781130005485, 0.0032912781130005485, 0.0038398244651673065, 0.0038398244651673065, 0.0038398244651673065, 0.004388370817334065, 0.004388370817334065, 0.004936917169500823, 0.005485463521667581, 0.006034009873834339, 0.006034009873834339, 0.006582556226001097, 0.007131102578167855, 0.007679648930334613, 0.008228195282501372, 0.00877674163466813, 0.009325287986834888, 0.009325287986834888, 0.009873834339001646, 0.009873834339001646, 0.010422380691168404, 0.010970927043335162, 0.01151947339550192, 0.012068019747668678, 0.012616566099835436, 0.012616566099835436, 0.013165112452002194, 0.013713658804168952, 0.01426220515633571, 0.014810751508502468, 0.015907844212835986, 0.016456390565002744, 0.017004936917169502, 0.018102029621503018, 0.018650575973669776, 0.019199122325836534, 0.019747668678003292, 0.02029621503017005, 0.021393307734503566, 0.021941854086670324, 0.021941854086670324, 0.022490400438837082, 0.02303894679100384, 0.023587493143170598, 0.024136039495337356, 0.025233132199670872, 0.026330224904004388, 0.026878771256171146, 0.02852441031267142, 0.030170049369171694, 0.030718595721338452, 0.031267142073505214, 0.03181568842567197, 0.033461327482172246, 0.034009873834339004, 0.034009873834339004, 0.03455842018650576, 0.03565551289083928, 0.036204059243006036, 0.03730115194733955, 0.038946791003839826, 0.039495337356006584, 0.04004388370817334, 0.041689522764673616, 0.042238069116840374, 0.04278661546900713, 0.04333516182117389, 0.04388370817334065, 0.044980800877674164, 0.04552934722984092, 0.047174986286341196, 0.04827207899067471, 0.04882062534284147, 0.050466264399341744, 0.05156335710367526, 0.05211190345584202, 0.052660449808008776, 0.055951727921009324, 0.058145913329676356, 0.058694459681843114, 0.05924300603400987, 0.060888645090510146, 0.06472846955567746, 0.06527701590784421, 0.06966538672517827, 0.0718595721338453, 0.07405375754251234, 0.07569939659901262, 0.07569939659901262, 0.07899067471201317, 0.07953922106417992, 0.08008776741634668, 0.0811848601206802, 0.08392759188151398, 0.08447613823368075, 0.08557323093801426, 0.08612177729018101, 0.08996160175534833, 0.09051014810751508, 0.09434997257268239, 0.09764125068568294, 0.09928688974218322, 0.10312671420735052, 0.10532089961601755, 0.10970927043335162, 0.11080636313768513, 0.11354909489851893, 0.11684037301151948, 0.11738891936368623, 0.11903455842018651, 0.12122874382885354, 0.12287438288535381, 0.1256171146461876, 0.12945693911135492, 0.13384530992868898, 0.13494240263302248, 0.13823368074602305, 0.1437191442676906, 0.14810751508502468, 0.1568842567196928, 0.1634668129456939, 0.16675809105869446, 0.17334064728469556, 0.17718047174986287, 0.17992320351069666, 0.18266593527153044, 0.18376302797586397, 0.18650575973669775, 0.1870543060888645, 0.18815139879319803, 0.18869994514536478, 0.18924849149753153, 0.18979703784969829, 0.19199122325836535, 0.1936368623148656, 0.19583104772353263, 0.1963795940756994, 0.19967087218869994, 0.20625342841470104, 0.2068019747668678, 0.21228743828853539, 0.21941854086670323, 0.22654964344487108, 0.23752057048820624, 0.24958859023587493, 0.2616566099835436, 0.2781130005485464, 0.290181020296215, 0.3033461327482172, 0.3165112452002194, 0.3269336258913878, 0.33461327482172243, 0.34503565551289084, 0.3505211190345584, 0.35545803620405925, 0.35765222161272625, 0.3598464070213933, 0.3620405924300603, 0.3625891387822271, 0.3631376851343939, 0.3636862314865606, 1.0], \"y\": [0.6820652173913043, 0.6847826086956522, 0.6875, 0.6902173913043478, 0.7038043478260869, 0.7119565217391305, 0.7146739130434783, 0.717391304347826, 0.7228260869565217, 0.7255434782608695, 0.7309782608695652, 0.7336956521739131, 0.7364130434782609, 0.7391304347826086, 0.7445652173913043, 0.7472826086956522, 0.75, 0.7527173913043478, 0.7554347826086957, 0.7581521739130435, 0.7690217391304348, 0.7717391304347826, 0.7744565217391305, 0.7771739130434783, 0.779891304347826, 0.782608695652174, 0.7853260869565217, 0.7880434782608695, 0.7907608695652174, 0.7934782608695652, 0.7961956521739131, 0.7989130434782609, 0.8043478260869565, 0.8070652173913043, 0.8097826086956522, 0.8125, 0.8152173913043478, 0.8179347826086957, 0.8206521739130435, 0.8233695652173914, 0.8260869565217391, 0.8288043478260869, 0.8315217391304348, 0.8342391304347826, 0.8369565217391305, 0.8369565217391305, 0.8396739130434783, 0.845108695652174, 0.8478260869565217, 0.8505434782608695, 0.8505434782608695, 0.8505434782608695, 0.8532608695652174, 0.8559782608695652, 0.8586956521739131, 0.8614130434782609, 0.8641304347826086, 0.8668478260869565, 0.8695652173913043, 0.8695652173913043, 0.8695652173913043, 0.8722826086956522, 0.8722826086956522, 0.875, 0.8777173913043478, 0.8777173913043478, 0.8804347826086957, 0.8831521739130435, 0.8858695652173914, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8913043478260869, 0.8913043478260869, 0.8913043478260869, 0.8913043478260869, 0.8913043478260869, 0.8913043478260869, 0.8913043478260869, 0.8940217391304348, 0.8940217391304348, 0.8967391304347826, 0.8967391304347826, 0.8967391304347826, 0.8967391304347826, 0.8967391304347826, 0.8967391304347826, 0.8994565217391305, 0.8994565217391305, 0.8994565217391305, 0.8994565217391305, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.904891304347826, 0.904891304347826, 0.904891304347826, 0.907608695652174, 0.907608695652174, 0.907608695652174, 0.907608695652174, 0.907608695652174, 0.9103260869565217, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9157608695652174, 0.9157608695652174, 0.9157608695652174, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9239130434782609, 0.9239130434782609, 0.9239130434782609, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9320652173913043, 0.9320652173913043, 0.9320652173913043, 0.9320652173913043, 0.9320652173913043, 0.9320652173913043, 0.9320652173913043, 0.9320652173913043, 0.9320652173913043, 0.9320652173913043, 0.9320652173913043, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9565217391304348, 0.9565217391304348, 0.9592391304347826, 0.9619565217391305, 0.9646739130434783, 0.967391304347826, 0.970108695652174, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9755434782608695, 0.9755434782608695, 0.9755434782608695, 1.0], \"mode\": \"lines\", \"line\": {\"width\": 2}, \"name\": \"model_ensemble (area = 0.9724)\"}, {\"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1], \"mode\": \"lines\", \"line\": {\"color\": \"navy\", \"width\": 2, \"dash\": \"dash\"}, \"showlegend\": false}], {\"title\": \"Receiver operating characteristic (ROC)\", \"xaxis\": {\"title\": \"False Positive Rate\"}, \"yaxis\": {\"title\": \"True Positive Rate\"}}, {\"showLink\": true, \"linkText\": \"Export to plot.ly\"})});</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<div id=\"880f329d-ae75-41b7-859d-9c3810a1ec2d\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";Plotly.newPlot(\"880f329d-ae75-41b7-859d-9c3810a1ec2d\", [{\"type\": \"scatter\", \"x\": [0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001088139281828074, 0.001632208922742111, 0.001632208922742111, 0.001632208922742111, 0.002176278563656148, 0.002720348204570185, 0.002720348204570185, 0.002720348204570185, 0.002720348204570185, 0.002720348204570185, 0.003264417845484222, 0.003264417845484222, 0.003264417845484222, 0.003808487486398259, 0.004352557127312296, 0.004352557127312296, 0.004352557127312296, 0.004352557127312296, 0.004896626768226333, 0.00544069640914037, 0.00544069640914037, 0.005984766050054407, 0.006528835690968444, 0.007072905331882481, 0.007616974972796518, 0.008705114254624592, 0.008705114254624592, 0.01088139281828074, 0.01088139281828074, 0.011425462459194777, 0.011969532100108813, 0.012513601741022852, 0.012513601741022852, 0.013057671381936888, 0.013601741022850925, 0.014689880304679, 0.015778019586507073, 0.01632208922742111, 0.017410228509249184, 0.017954298150163223, 0.018498367791077257, 0.019042437431991296, 0.01958650707290533, 0.02013057671381937, 0.021218715995647442, 0.02176278563656148, 0.022306855277475515, 0.022850924918389554, 0.023394994559303592, 0.023939064200217627, 0.024483133841131665, 0.025571273122959738, 0.026115342763873776, 0.02720348204570185, 0.027747551686615888, 0.029379760609358, 0.029379760609358, 0.030467899891186073, 0.031011969532100107, 0.031556039173014146, 0.03264417845484222, 0.03427638737758433, 0.03482045701849837, 0.03536452665941241, 0.035908596300326445, 0.03754080522306855, 0.03808487486398259, 0.0397170837867247, 0.04026115342763874, 0.040805223068552776, 0.041349292709466814, 0.04570184983677911, 0.046245919477693145, 0.04787812840043525, 0.04842219804134929, 0.04896626768226333, 0.051142546245919476, 0.05223068552774755, 0.05331882480957562, 0.05386289445048966, 0.0544069640914037, 0.05495103373231774, 0.05603917301414581, 0.057127312295973884, 0.05767138193688792, 0.058759521218716, 0.05984766050054407, 0.06039173014145811, 0.060935799782372145, 0.06147986942328618, 0.062023939064200215, 0.06311207834602829, 0.06365614798694233, 0.06420021762785637, 0.06692056583242655, 0.06746463547334058, 0.06964091403699674, 0.07181719260065289, 0.07236126224156691, 0.07290533188248095, 0.07399347116430903, 0.07725788900979326, 0.07834602829162132, 0.08161044613710555, 0.08215451577801959, 0.08324265505984767, 0.08378672470076169, 0.08705114254624592, 0.088139281828074, 0.09031556039173014, 0.09303590859630033, 0.09357997823721437, 0.09466811751904244, 0.09630032644178455, 0.09793253536452666, 0.0984766050054407, 0.09956474428726877, 0.10010881392818281, 0.10282916213275299, 0.10609357997823722, 0.10663764961915125, 0.10663764961915125, 0.10935799782372144, 0.11371055495103373, 0.1147986942328618, 0.117519042437432, 0.1191512513601741, 0.12023939064200218, 0.12078346028291621, 0.12132752992383025, 0.1235038084874864, 0.12459194776931447, 0.12785636561479868, 0.12840043525571274, 0.132208922742111, 0.13601741022850924, 0.14145810663764963, 0.14744287268770404, 0.15505984766050054, 0.16050054406964093, 0.16648531011969533, 0.1692056583242655, 0.16974972796517954, 0.17519042437431992, 0.17573449401523394, 0.17845484221980412, 0.1822633297062024, 0.1838955386289445, 0.18443960826985853, 0.18661588683351468, 0.18715995647442873, 0.18879216539717084, 0.190968443960827, 0.19205658324265507, 0.19368879216539717, 0.19477693144722524, 0.1953210010881393, 0.19586507072905332, 0.19640914036996734, 0.19749727965179542, 0.19804134929270947, 0.19967355821545157, 0.20184983677910773, 0.20511425462459196, 0.2105549510337323, 0.21545157780195864, 0.22306855277475518, 0.23177366702937977, 0.249727965179543, 0.2622415669205658, 0.27094668117519044, 0.28291621327529926, 0.29651795429815014, 0.31229597388465724, 0.32100108813928185, 0.3291621327529924, 0.3373231773667029, 0.34221980413492925, 0.34766050054406966, 0.3487486398258977, 0.35038084874863984, 0.3514689880304679, 0.35310119695321, 0.35364526659412404, 0.35418933623503807, 0.35473340587595215, 0.35527747551686617, 0.3558215451577802, 0.3563656147986942, 0.35690968443960824, 0.3574537540805223, 0.35799782372143635, 1.0], \"y\": [0.5625, 0.5652173913043478, 0.5679347826086957, 0.5733695652173914, 0.5760869565217391, 0.5788043478260869, 0.5815217391304348, 0.5869565217391305, 0.592391304347826, 0.6032608695652174, 0.6059782608695652, 0.6114130434782609, 0.6168478260869565, 0.625, 0.6331521739130435, 0.6440217391304348, 0.6603260869565217, 0.6711956521739131, 0.6766304347826086, 0.6820652173913043, 0.6902173913043478, 0.6983695652173914, 0.7092391304347826, 0.7119565217391305, 0.7146739130434783, 0.720108695652174, 0.7228260869565217, 0.7309782608695652, 0.7364130434782609, 0.7418478260869565, 0.7445652173913043, 0.7554347826086957, 0.7581521739130435, 0.7663043478260869, 0.7690217391304348, 0.7717391304347826, 0.7771739130434783, 0.779891304347826, 0.7853260869565217, 0.7880434782608695, 0.7989130434782609, 0.8016304347826086, 0.8043478260869565, 0.8070652173913043, 0.8097826086956522, 0.8125, 0.8152173913043478, 0.8179347826086957, 0.8206521739130435, 0.8233695652173914, 0.8260869565217391, 0.8288043478260869, 0.8342391304347826, 0.8369565217391305, 0.8396739130434783, 0.842391304347826, 0.845108695652174, 0.8478260869565217, 0.8505434782608695, 0.8532608695652174, 0.8559782608695652, 0.8559782608695652, 0.8586956521739131, 0.8614130434782609, 0.8614130434782609, 0.8614130434782609, 0.8641304347826086, 0.8668478260869565, 0.8695652173913043, 0.8722826086956522, 0.8722826086956522, 0.875, 0.8777173913043478, 0.8777173913043478, 0.8777173913043478, 0.8804347826086957, 0.8831521739130435, 0.8858695652173914, 0.8858695652173914, 0.8858695652173914, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8913043478260869, 0.8940217391304348, 0.8967391304347826, 0.8994565217391305, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.904891304347826, 0.904891304347826, 0.904891304347826, 0.907608695652174, 0.9103260869565217, 0.9103260869565217, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9157608695652174, 0.9157608695652174, 0.9157608695652174, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9239130434782609, 0.9239130434782609, 0.9239130434782609, 0.9239130434782609, 0.9239130434782609, 0.9239130434782609, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9320652173913043, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9429347826086957, 0.9429347826086957, 0.9429347826086957, 0.9483695652173914, 0.9483695652173914, 0.9538043478260869, 0.9538043478260869, 0.9538043478260869, 0.9592391304347826, 0.9592391304347826, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9646739130434783, 0.9646739130434783, 0.9646739130434783, 0.967391304347826, 0.967391304347826, 0.967391304347826, 0.967391304347826, 0.967391304347826, 0.967391304347826, 0.970108695652174, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9755434782608695, 0.9755434782608695, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 1.0], \"mode\": \"lines\", \"line\": {\"width\": 2}, \"name\": \"model_simple (area = 0.9746)\"}, {\"type\": \"scatter\", \"x\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0005509641873278236, 0.0005509641873278236, 0.0005509641873278236, 0.0011019283746556473, 0.0011019283746556473, 0.0011019283746556473, 0.001652892561983471, 0.001652892561983471, 0.0022038567493112946, 0.0022038567493112946, 0.0027548209366391185, 0.0027548209366391185, 0.003305785123966942, 0.003305785123966942, 0.004407713498622589, 0.0049586776859504135, 0.005509641873278237, 0.006060606060606061, 0.007162534435261708, 0.007713498622589532, 0.008264462809917356, 0.008264462809917356, 0.009366391184573003, 0.009917355371900827, 0.01046831955922865, 0.011570247933884297, 0.012121212121212121, 0.012672176308539946, 0.012672176308539946, 0.013223140495867768, 0.014325068870523415, 0.01487603305785124, 0.015426997245179064, 0.015426997245179064, 0.015977961432506887, 0.01652892561983471, 0.017079889807162536, 0.01818181818181818, 0.01818181818181818, 0.020385674931129475, 0.021487603305785124, 0.02203856749311295, 0.023140495867768594, 0.02534435261707989, 0.025895316804407712, 0.026446280991735537, 0.02699724517906336, 0.027548209366391185, 0.029201101928374655, 0.02975206611570248, 0.030303030303030304, 0.03085399449035813, 0.03140495867768595, 0.03140495867768595, 0.031955922865013774, 0.03360881542699724, 0.03581267217630854, 0.03581267217630854, 0.03636363636363636, 0.03801652892561983, 0.03856749311294766, 0.03856749311294766, 0.03911845730027548, 0.04022038567493113, 0.04077134986225895, 0.0418732782369146, 0.04242424242424243, 0.04297520661157025, 0.04352617079889807, 0.045179063360881545, 0.045730027548209366, 0.04628099173553719, 0.046831955922865015, 0.048484848484848485, 0.049035812672176306, 0.050137741046831955, 0.0512396694214876, 0.05234159779614325, 0.05289256198347107, 0.05454545454545454, 0.05619834710743802, 0.05674931129476584, 0.05785123966942149, 0.05950413223140496, 0.06005509641873278, 0.06115702479338843, 0.06170798898071626, 0.0628099173553719, 0.06391184573002755, 0.06501377410468319, 0.06611570247933884, 0.06666666666666667, 0.06721763085399449, 0.06831955922865014, 0.07107438016528926, 0.0721763085399449, 0.07382920110192838, 0.07493112947658402, 0.07603305785123966, 0.07713498622589532, 0.07878787878787878, 0.07933884297520662, 0.08099173553719008, 0.08429752066115702, 0.08539944903581267, 0.0859504132231405, 0.08650137741046832, 0.0881542699724518, 0.09256198347107437, 0.09476584022038567, 0.09586776859504133, 0.09696969696969697, 0.09807162534435261, 0.10027548209366391, 0.10137741046831956, 0.10358126721763085, 0.10413223140495868, 0.10578512396694215, 0.10964187327823692, 0.11349862258953168, 0.1140495867768595, 0.1162534435261708, 0.11680440771349862, 0.1206611570247934, 0.12341597796143251, 0.12617079889807162, 0.12837465564738293, 0.12892561983471074, 0.12947658402203857, 0.13112947658402205, 0.1327823691460055, 0.13388429752066117, 0.13608815426997245, 0.13663911845730028, 0.1399449035812672, 0.14380165289256197, 0.14986225895316804, 0.15592286501377411, 0.15647382920110192, 0.16363636363636364, 0.17134986225895316, 0.17851239669421487, 0.18622589531680442, 0.19063360881542699, 0.19173553719008266, 0.1933884297520661, 0.19393939393939394, 0.19504132231404958, 0.19614325068870522, 0.19669421487603306, 0.1977961432506887, 0.19834710743801653, 0.19889807162534434, 0.20055096418732782, 0.20330578512396694, 0.20606060606060606, 0.20881542699724517, 0.21101928374655649, 0.21322314049586777, 0.2236914600550964, 0.23415977961432508, 0.2440771349862259, 0.2528925619834711, 0.2705234159779614, 0.28429752066115704, 0.29641873278236913, 0.31294765840220384, 0.3261707988980716, 0.3393939393939394, 0.3504132231404959, 0.36143250688705236, 0.37245179063360884, 0.3752066115702479, 0.3768595041322314, 0.3790633608815427, 0.37961432506887055, 0.38126721763085397, 0.38236914600550964, 0.38292011019283745, 0.3834710743801653, 0.3840220385674931, 0.3845730027548209, 0.3851239669421488, 1.0], \"y\": [0.4891304347826087, 0.49184782608695654, 0.4945652173913043, 0.5, 0.5190217391304348, 0.5489130434782609, 0.5597826086956522, 0.5788043478260869, 0.5978260869565217, 0.6222826086956522, 0.6413043478260869, 0.6440217391304348, 0.6603260869565217, 0.6657608695652174, 0.6711956521739131, 0.6739130434782609, 0.6766304347826086, 0.6793478260869565, 0.6820652173913043, 0.6847826086956522, 0.6875, 0.6902173913043478, 0.6929347826086957, 0.6956521739130435, 0.7065217391304348, 0.7092391304347826, 0.7146739130434783, 0.720108695652174, 0.7228260869565217, 0.7282608695652174, 0.7309782608695652, 0.7336956521739131, 0.7364130434782609, 0.7418478260869565, 0.7445652173913043, 0.7472826086956522, 0.75, 0.7527173913043478, 0.7554347826086957, 0.7581521739130435, 0.7608695652173914, 0.7635869565217391, 0.7663043478260869, 0.7717391304347826, 0.7744565217391305, 0.7771739130434783, 0.779891304347826, 0.782608695652174, 0.7880434782608695, 0.7907608695652174, 0.7934782608695652, 0.7961956521739131, 0.7989130434782609, 0.8016304347826086, 0.8043478260869565, 0.8070652173913043, 0.8097826086956522, 0.8125, 0.8152173913043478, 0.8179347826086957, 0.8233695652173914, 0.8260869565217391, 0.8288043478260869, 0.8315217391304348, 0.8369565217391305, 0.8396739130434783, 0.842391304347826, 0.845108695652174, 0.8478260869565217, 0.8505434782608695, 0.8532608695652174, 0.8559782608695652, 0.8586956521739131, 0.8614130434782609, 0.8614130434782609, 0.8641304347826086, 0.8668478260869565, 0.8668478260869565, 0.8695652173913043, 0.8722826086956522, 0.8722826086956522, 0.875, 0.875, 0.8777173913043478, 0.8804347826086957, 0.8831521739130435, 0.8858695652173914, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8913043478260869, 0.8913043478260869, 0.8913043478260869, 0.8913043478260869, 0.8940217391304348, 0.8940217391304348, 0.8940217391304348, 0.8967391304347826, 0.8967391304347826, 0.8994565217391305, 0.8994565217391305, 0.8994565217391305, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.904891304347826, 0.907608695652174, 0.907608695652174, 0.907608695652174, 0.907608695652174, 0.9103260869565217, 0.9103260869565217, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9157608695652174, 0.9157608695652174, 0.9157608695652174, 0.9157608695652174, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9211956521739131, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9320652173913043, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9402173913043478, 0.9402173913043478, 0.9429347826086957, 0.9456521739130435, 0.9456521739130435, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9538043478260869, 0.9538043478260869, 0.9538043478260869, 0.9538043478260869, 0.9538043478260869, 0.9565217391304348, 0.9565217391304348, 0.9592391304347826, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9619565217391305, 0.9646739130434783, 0.970108695652174, 0.970108695652174, 0.9728260869565217, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9782608695652174, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 0.9836956521739131, 1.0], \"mode\": \"lines\", \"line\": {\"width\": 2}, \"name\": \"model_outliers (area = 0.9775)\"}, {\"type\": \"scatter\", \"x\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0005485463521667581, 0.0005485463521667581, 0.0005485463521667581, 0.0005485463521667581, 0.0005485463521667581, 0.0010970927043335162, 0.0016456390565002743, 0.0016456390565002743, 0.0016456390565002743, 0.0016456390565002743, 0.0016456390565002743, 0.0016456390565002743, 0.0016456390565002743, 0.0016456390565002743, 0.0021941854086670325, 0.0027427317608337905, 0.0027427317608337905, 0.0032912781130005485, 0.0032912781130005485, 0.0032912781130005485, 0.0038398244651673065, 0.0038398244651673065, 0.0038398244651673065, 0.004388370817334065, 0.004388370817334065, 0.004936917169500823, 0.005485463521667581, 0.006034009873834339, 0.006034009873834339, 0.006582556226001097, 0.007131102578167855, 0.007679648930334613, 0.008228195282501372, 0.00877674163466813, 0.009325287986834888, 0.009325287986834888, 0.009873834339001646, 0.009873834339001646, 0.010422380691168404, 0.010970927043335162, 0.01151947339550192, 0.012068019747668678, 0.012616566099835436, 0.012616566099835436, 0.013165112452002194, 0.013713658804168952, 0.01426220515633571, 0.014810751508502468, 0.015907844212835986, 0.016456390565002744, 0.017004936917169502, 0.018102029621503018, 0.018650575973669776, 0.019199122325836534, 0.019747668678003292, 0.02029621503017005, 0.021393307734503566, 0.021941854086670324, 0.021941854086670324, 0.022490400438837082, 0.02303894679100384, 0.023587493143170598, 0.024136039495337356, 0.025233132199670872, 0.026330224904004388, 0.026878771256171146, 0.02852441031267142, 0.030170049369171694, 0.030718595721338452, 0.031267142073505214, 0.03181568842567197, 0.033461327482172246, 0.034009873834339004, 0.034009873834339004, 0.03455842018650576, 0.03565551289083928, 0.036204059243006036, 0.03730115194733955, 0.038946791003839826, 0.039495337356006584, 0.04004388370817334, 0.041689522764673616, 0.042238069116840374, 0.04278661546900713, 0.04333516182117389, 0.04388370817334065, 0.044980800877674164, 0.04552934722984092, 0.047174986286341196, 0.04827207899067471, 0.04882062534284147, 0.050466264399341744, 0.05156335710367526, 0.05211190345584202, 0.052660449808008776, 0.055951727921009324, 0.058145913329676356, 0.058694459681843114, 0.05924300603400987, 0.060888645090510146, 0.06472846955567746, 0.06527701590784421, 0.06966538672517827, 0.0718595721338453, 0.07405375754251234, 0.07569939659901262, 0.07569939659901262, 0.07899067471201317, 0.07953922106417992, 0.08008776741634668, 0.0811848601206802, 0.08392759188151398, 0.08447613823368075, 0.08557323093801426, 0.08612177729018101, 0.08996160175534833, 0.09051014810751508, 0.09434997257268239, 0.09764125068568294, 0.09928688974218322, 0.10312671420735052, 0.10532089961601755, 0.10970927043335162, 0.11080636313768513, 0.11354909489851893, 0.11684037301151948, 0.11738891936368623, 0.11903455842018651, 0.12122874382885354, 0.12287438288535381, 0.1256171146461876, 0.12945693911135492, 0.13384530992868898, 0.13494240263302248, 0.13823368074602305, 0.1437191442676906, 0.14810751508502468, 0.1568842567196928, 0.1634668129456939, 0.16675809105869446, 0.17334064728469556, 0.17718047174986287, 0.17992320351069666, 0.18266593527153044, 0.18376302797586397, 0.18650575973669775, 0.1870543060888645, 0.18815139879319803, 0.18869994514536478, 0.18924849149753153, 0.18979703784969829, 0.19199122325836535, 0.1936368623148656, 0.19583104772353263, 0.1963795940756994, 0.19967087218869994, 0.20625342841470104, 0.2068019747668678, 0.21228743828853539, 0.21941854086670323, 0.22654964344487108, 0.23752057048820624, 0.24958859023587493, 0.2616566099835436, 0.2781130005485464, 0.290181020296215, 0.3033461327482172, 0.3165112452002194, 0.3269336258913878, 0.33461327482172243, 0.34503565551289084, 0.3505211190345584, 0.35545803620405925, 0.35765222161272625, 0.3598464070213933, 0.3620405924300603, 0.3625891387822271, 0.3631376851343939, 0.3636862314865606, 1.0], \"y\": [0.6820652173913043, 0.6847826086956522, 0.6875, 0.6902173913043478, 0.7038043478260869, 0.7119565217391305, 0.7146739130434783, 0.717391304347826, 0.7228260869565217, 0.7255434782608695, 0.7309782608695652, 0.7336956521739131, 0.7364130434782609, 0.7391304347826086, 0.7445652173913043, 0.7472826086956522, 0.75, 0.7527173913043478, 0.7554347826086957, 0.7581521739130435, 0.7690217391304348, 0.7717391304347826, 0.7744565217391305, 0.7771739130434783, 0.779891304347826, 0.782608695652174, 0.7853260869565217, 0.7880434782608695, 0.7907608695652174, 0.7934782608695652, 0.7961956521739131, 0.7989130434782609, 0.8043478260869565, 0.8070652173913043, 0.8097826086956522, 0.8125, 0.8152173913043478, 0.8179347826086957, 0.8206521739130435, 0.8233695652173914, 0.8260869565217391, 0.8288043478260869, 0.8315217391304348, 0.8342391304347826, 0.8369565217391305, 0.8369565217391305, 0.8396739130434783, 0.845108695652174, 0.8478260869565217, 0.8505434782608695, 0.8505434782608695, 0.8505434782608695, 0.8532608695652174, 0.8559782608695652, 0.8586956521739131, 0.8614130434782609, 0.8641304347826086, 0.8668478260869565, 0.8695652173913043, 0.8695652173913043, 0.8695652173913043, 0.8722826086956522, 0.8722826086956522, 0.875, 0.8777173913043478, 0.8777173913043478, 0.8804347826086957, 0.8831521739130435, 0.8858695652173914, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8885869565217391, 0.8913043478260869, 0.8913043478260869, 0.8913043478260869, 0.8913043478260869, 0.8913043478260869, 0.8913043478260869, 0.8913043478260869, 0.8940217391304348, 0.8940217391304348, 0.8967391304347826, 0.8967391304347826, 0.8967391304347826, 0.8967391304347826, 0.8967391304347826, 0.8967391304347826, 0.8994565217391305, 0.8994565217391305, 0.8994565217391305, 0.8994565217391305, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.9021739130434783, 0.904891304347826, 0.904891304347826, 0.904891304347826, 0.907608695652174, 0.907608695652174, 0.907608695652174, 0.907608695652174, 0.907608695652174, 0.9103260869565217, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9130434782608695, 0.9157608695652174, 0.9157608695652174, 0.9157608695652174, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9184782608695652, 0.9239130434782609, 0.9239130434782609, 0.9239130434782609, 0.9266304347826086, 0.9266304347826086, 0.9266304347826086, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9293478260869565, 0.9320652173913043, 0.9320652173913043, 0.9320652173913043, 0.9320652173913043, 0.9320652173913043, 0.9320652173913043, 0.9320652173913043, 0.9320652173913043, 0.9320652173913043, 0.9320652173913043, 0.9320652173913043, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9347826086956522, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9375, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9402173913043478, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9483695652173914, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9510869565217391, 0.9565217391304348, 0.9565217391304348, 0.9592391304347826, 0.9619565217391305, 0.9646739130434783, 0.967391304347826, 0.970108695652174, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9728260869565217, 0.9755434782608695, 0.9755434782608695, 0.9755434782608695, 1.0], \"mode\": \"lines\", \"line\": {\"width\": 2}, \"name\": \"model_ensemble (area = 0.9724)\"}, {\"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1], \"mode\": \"lines\", \"line\": {\"color\": \"navy\", \"width\": 2, \"dash\": \"dash\"}, \"showlegend\": false}], {\"title\": \"Receiver operating characteristic (ROC)\", \"xaxis\": {\"title\": \"False Positive Rate\"}, \"yaxis\": {\"title\": \"True Positive Rate\"}}, {\"showLink\": true, \"linkText\": \"Export to plot.ly\"})});</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<script>function downloadimage(format, height, width, filename) {var p = document.getElementById('880f329d-ae75-41b7-859d-9c3810a1ec2d');Plotly.downloadImage(p, {format: format, height: height, width: width, filename: filename});};if(document.readyState == 'complete') {{downloadimage('png', 540, 960, 'roc_curves');}}</script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "rocdata = []\n",
    "\n",
    "for m in models:\n",
    "    xroc, yroc = models[m].roc()\n",
    "    rocauc = models[m].auc()\n",
    "    rocdata.append(go.Scatter(x=xroc, y=yroc, \n",
    "                   mode='lines', \n",
    "                   line=dict( width=2),\n",
    "                   name='%s (area = %0.4f)' % (m,rocauc)))\n",
    "\n",
    "\n",
    "rocdata.append(go.Scatter(x=[0, 1], y=[0, 1], \n",
    "                          mode='lines', \n",
    "                          line=dict(color='navy', width=2, dash='dash'),\n",
    "                          showlegend=False))\n",
    "\n",
    "layout = go.Layout(title='Receiver operating characteristic (ROC)',\n",
    "                   xaxis=dict(title='False Positive Rate'),\n",
    "                   yaxis=dict(title='True Positive Rate'))\n",
    "\n",
    "fig = go.Figure(data=rocdata, layout=layout)\n",
    "plotly.offline.iplot(fig, filename=\"roc_curves\", image=IMAGE_FORMAT, image_height=IMAGE_HEIGT, image_width=IMAGE_WIDTH) # offline method guarantee privacy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Area under the (ROC) curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model 'model_simple' AUC : 0.9746408844679946\n",
      "Model 'model_outliers' AUC : 0.9774800874356211\n",
      "Model 'model_ensemble' AUC : 0.9723967898113478\n"
     ]
    }
   ],
   "source": [
    "for m in models:\n",
    "    print(\"Model '{}' AUC : {}\".format(m, models[m].auc()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model 'model_simple' AUC : 0.9746408844679946\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEXCAYAAAC+mHPKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8HOWV7//PsWR53yVvkne8YBtjS2KbsBgngNmx5R+B\nLBMySZgESLg/hiQwmUkmzCRkfsNvMskNN3eYhBuYTCAZicUQwm6WLIAtecOAF4yNW4sXyZZlyVr7\n3D+qRBphSW1brZa6v+/XSy91VT3VfUpl9+l6nurnmLsjIiLSnQHJDkBERPoHJQwREYmLEoaIiMRF\nCUNEROKihCEiInFRwhARkbgoYUifZGYZZnbEzKb2ZNu+ysyWmtmWJL7+s2b26WS9vvQPShjSI8I3\n7PafqJkdjVk+7jcid29z9+Hu/n5Ptj1eZjbGzH5hZlVmdtjMtprZ13v6ddz9JXdf0NPPC2Bmvzez\nGzqs+4SZ7Yp5/Yvd/b+6eZ5MM3Mzm56IOKXvy0x2AJIa3H14++PwjeiL7v58Z+3NLNPdW3sjtpP0\nYyADmAccBuYCp/bkC/Sjv8VJSZfjTGW6wpBeYWb/ZGa/NrOHzKwO+IyZnWNmr5nZITOrNLMfm9nA\nsP2HPs2a2S/D7b8zszoz+5OZzTjetuH2S81sm5nVmtn/NLM/dPwEHuMM4Ffufsjdo+7+trs/EvNc\np5nZ82ZWE16FfCNcPziModLMys3sX80sK9z2CTPbZWZ/a2ZVwH90/MRvZhEzu83MNodxPmRmg2K2\n3xm+XrmZfelkP/nHXoWY2RwzeyV83QNm9quw2Svh7y3hlWNR2P7LZrbDzKrN7DEzmxSubz8vN5nZ\nDuAdM/t3M/vnDq/9lJl99URjl96jhCG9aQXwK2AU8GugFbgVyAY+BiwH/rqL/T8F/D0wFngf+Mfj\nbWtm44HfAF8PX/c94Mwunuc14G4zu8HMZsduMLNRwPPAE8AkYA7wUrj520AhsAhYEh7fnTG75wHD\nganATZ289rXARcBMoAD4bPi6VwBfBS4MX3NZF/GfiO8BvwXGhHHeG64/P/y9IOwCLDGzi4G7gFVA\nLlABdOzauoog8Z4GPABcb2YWHssEYCnwUA8fgySAEob0pt+7+xPhJ/Wj7r7W3V9391Z33wncB1zQ\nxf7F7r7O3VsI3pQWn0DbK4AN7v54uO2HwIEunucmguT2NeBtM9sevklC8Eb4vrv/yN2b3P2wu78R\nbvs08A/uvt/d9xG8qX425nlbw+3N7n60k9f+N3evcvdq4MmYY7gW+Hl4tVMPfLeL+Nv9r/BK7pCZ\nHQIe66JtCzAdmOTuje7+hy7afhr4mbtvcPdG4A7gAjPLi2nzfXc/GJ7zPwKNBEkC4HrgeXfv6hxI\nH6GEIb1pT+yCmc0zs9+2DygTvKlmd7F/VczjBoJP6MfbdnJsHB7Mvhnp7EncvcHd/8nd84FxwCNA\nSXh1MQV4t5NdJwO7Y5Z3E3wCb7fX3Zu7iD/uY+jwuDM3ufvo9h/gmi7a/g0wEFgXdol9rou2HzpO\ndz8MHOTDx9oxvgeBz4SPPwP8ZxzxSx+ghCG9qePUyP8OvAmc4u4jCbpxLMExVBJ0swAQdo3kdt78\nz9y9Frib4I17OsEb4axOmlcA02KWpwLlsU8Xd8Qf9aFjIEhcPcbdK939i+4+CbgZuC8cAzpWzB86\nTjMbQdCV1dWx/iewwsyWEPz9nujJ+CVxlDAkmUYAtUC9mZ1K1+MXPeVJIN/MrjSzTIIxlJzOGpvZ\nd8ys0MyyzGwwQddUDbAdWA1MNbNbzGyQmY00s/bxkIeAb5tZtpnlEIyn/LKHjuE3wBfMbK6ZDQ2f\nu8eY2bVm1p5EDxG84be5extQTTCm0u6hMJZF4aD83cCr7t7VVdtuYCPBeMZ/h11Z0g8oYUgy/Q3w\nOaCO4Grj14l+QXffC3wS+FeCN79ZwHqgqYvdHgjbVhD0vV8edlXVEgxKFwF7gW38eQzmuwRvim8C\nm4DXCd5Me+IYngB+SnDX0nagfYyhq2M4HmcBa82snqAL7uaY77h8B/hVOBay0t2fJuhKfJTgymcq\nwbhGdx4gGARXd1Q/YiqgJOnMzDIIEsEqd3812fGcCDM7DSgDBrl7NNnxxMPMlgE/B2a63oT6DV1h\nSNoxs+VmNjrsQvl7gruC3uhmtz7FzFaE3WRjgR8Aj/ejZJFF0BX4H0oW/YsShqSjc4GdwH7gEmCF\nu/dUd05vuZngduAdBLep3pzccOITXg0dJPh+zI+THI4cJ3VJiYhIXHSFISIicUmpyQezs7N9+vTp\nyQ5DRCThWqPOoYZmDja00NjSxrCsTGbmDDuh5yotLT3g7p3eXt4upRLG9OnTWbduXbLDEBFJiKbW\nNl58ex/FpRFe2rYfos7SvFGsKsjjytMnM3po1gk9r5nt7r5ViiUMEZFU4+5sjNRSUhph9cYKao+2\nMGHkIL503kxWFeRyyvgRvRaLEoaISB9UVdvIo+vLKSmLsGPfEQZlDuCSBRMpKsjj3FOyyRiQ6Fl0\nPkoJQ0Skj2hsaeOZLVWUlJXz++37iToUThvD3StP4/JFkxg5eGBS41PCEBFJInendPdBiksj/HZT\nJXVNreSOHsItF57Cyvw8pmef2EB2IihhiIgkQeRgA4+UlfNIWYRd1Q0Mzcrg0oWTKCrI5ewZ4xiQ\nhC6n7ihhiIj0kvqmVn73ZhUlpRH+tLMagHNmjuOWZbO5dOFEhg3q22/JfTs6EZF+Lhp1XnuvmpLS\ncn73ZiUNzW1MGzeU2y6aw4oluUwZOzTZIcZNCUNEJAF2HajnkbIIJWXllB86yohBmVx1+mRWFeRR\nMG0MYVnzfkUJQ0SkhxxubOG3myopKY2wbvdBzODcU7L5xvK5XLJgIoMHZiQ7xJOihCEichLaos4f\ndhyguDTCM1uqaGqNMitnGN9cPo8VS3KZOGpwskPsMUoYIiInYMe+OopLy3lsfTlVhxsZNWQg1xZO\noaggj9PzRvXLLqfuJDxhmNly4EdABvAzd/9Bh+3TgPsJ6irXAJ9x94iZXQj8MKbpPOA6d38s0TGL\niBzLoYZmnthYQXFZORv3HCJjgLF0Tg7fvnI+Hz91PIMy+3eXU3cSmjDC8pf3EtQ9jhDUCV7t7m/F\nNLsHeNDdHwjLNt4NfNbd1wCLw+cZS1Ao5tlExisi0lFLW5RXtu2nuDTCC2/vo7ktyryJI/i7y0/l\n6sW55IwYlOwQe02irzDOBHa4+04AM3sYuBqITRjzgdvCx2uAY11BrAJ+5+4NCYxVROQDb1ceprg0\nwuMbyjlwpJlxw7L49NlTWVWQx4LJo5IdXlIkOmHkAntiliPAWR3abARWEnRbrQBGmNk4d6+OaXMd\n8K/HegEzuxG4EWDq1Kk9FLaIpKMDR5p4fEMFJaUR3qo8zMAM4+PzJlBUkMfSuTkMzEjvmnN9YdD7\nduAnZnYD8ApQDrS1bzSzScBpwDPH2tnd7wPuAygsLFS9WRE5Ls2tUV58Zy/FpeW8tHUfrVFnUd4o\nvnvVAq46fTJjhp1YjYlUlOiEUQ5MiVnOC9d9wN0rCK4wMLPhQJG7H4ppci3wqLu3JDhWEUkT7s7m\n8qDGxOMbKzjU0ML4EYP4wrkzKCrIY86E3qsx0Z8kOmGsBWab2QyCRHEd8KnYBmaWDdS4exS4k+CO\nqVjXh+tFRE7KvsNBjYni0gjb9x0hK3MAF8+fwKqwxkRmmnc5dSehCcPdW83sFoLupAzgfnffYmZ3\nAevcfTWwFLjbzJygS+rm9v3NbDrBFcrLiYxTRFJXY0sbz761l5LSCK+GNSYKpo3h+yuCGhOjhiS3\nxkR/Yu6p0+1fWFjoquktIu5O2fsHKS4t58lNFdQ1tjJ51GBW5uexMj+XmTnDkx1in2Jmpe5e2F27\nvjDoLSLSI8oPHeXRcMK/9w7UM2RgBpcuDMqanjOzb9aY6E+UMESkX2tobuXpN6soDmtMuMNZM8by\nlaWzuOy0SQzv4zUm+hP9JUWk34lGnTd21VBcGuF3myupb25j6tih/I+Pz2Flfv+qMdGfKGGISL+x\nu7qekrCsaeTgUYYPyuSKRZMpKsjjjOn9s8ZEf6KEISJ9Wl1jC09trqSktJw3dtV8UGPi9ouDGhND\nslJ7wr++RAlDRPqctqjzx3cPUFIa4ektVTS2RJmZM4yvXzKXFUtymTx6SLJDTEtKGCLSZ7y7/wgl\npREeXV9OZW0jIwdnUpSfx6qCPBZPGa0upyRTwhCRpKptaGH1pmDCvw1hjYkL5uTwd5cHNSb6e1nT\nVKKEISK9rrUtyivb91NSWs5zb+2luS3K3Akj+NZlp3L1ksmMH5E6ZU1TiRKGiPSad6oOh11OFRw4\n0sTYYVl86qz2GhMj1eXUxylhiEhCVR9pYvXGCopLI2ypOEzmAGPZvPEUFeRx4dzxZGVqwr/+QglD\nRHpcc2uUNVv3UVwaYc07QY2Jhbkj+Ycr53PV4lzGqsZEv6SEISI9wt3ZUvHnsqYHG1rIHj6Ivzp3\nBkX5ecydqBoT/Z0ShoiclH2HG3lsQzklpeVs3VtHVuYALpo/gVX5eZw3WzUmUokShogct8aWNp5/\nO6gx8fK2oMbEkqmj+adrFnLlosmMGqoaE6lICUNE4uLurN9ziOLSCE9urOBwYyuTRg3mK0tnsTI/\nj1mqMZHylDBEpEsVh47y6PpySkoj7DxQz+CBA7h04SSK8vM4Z9Y4MlRjIm0oYYjIRxxtbuPpLcGE\nf3949wDucOaMsXz5gllcetpERgxWl1M6UsIQESDocnrjvRpKyiI8tbmKI02tTBk7hK8tm01Rfh5T\nx6nGRLpTwhBJc3tqGigpi1BSFmFPzVGGZWVw2WmTWFWQxxnTx6qsqXxACUMkDR1pauWpzZUUl0Z4\n472gxsRfzBrHbRfN4ZIFExmapbcG+Sj9qxBJE9Go86ed1RSXRnj6zSqOtrQxIzuoMXHNklxyVWNC\nuqGEIZLidu4/QklZhEfLyqmobWTE4ExW5OdSlJ9H/lTVmJD4KWGIpKDaoy08GdaYKHv/EAMMzp+T\nw52XncpF8yeoxoScECUMkRTR2hbl1R1BWdNn39pLc2uUOROGc+el87hmSS4TRqrGhJychCcMM1sO\n/AjIAH7m7j/osH0acD+QA9QAn3H3SLhtKvAzYArgwGXuvivRMYv0J1ur6oIup/Xl7K9rYszQgXzq\nzKkU5eexMFc1JqTnJDRhmFkGcC9wERAB1prZand/K6bZPcCD7v6AmS0D7gY+G257EPieuz9nZsOB\naCLjFekvauqbWb2hnJKycjaX15I5wLhw3niK8vNYNk81JiQxEn2FcSaww913ApjZw8DVQGzCmA/c\nFj5eAzwWtp0PZLr7cwDufiTBsYr0aS1tUda8s4+SsggvvrOPljZnweSRfPuK+Vy9eDLjhg9KdoiS\n4hKdMHKBPTHLEeCsDm02AisJuq1WACPMbBwwBzhkZo8AM4DngTvcvS12ZzO7EbgRYOrUqYk4BpGk\naa8xUVIW4fENFdTUN5M9fBCfO2c6RQV5nDppZLJDlDTSFwa9bwd+YmY3AK8A5UAbQWznAUuA94Ff\nAzcAP4/d2d3vA+4DKCws9N4KWiSR9tc18fiGcopLI7xTVUdWxgA+MX88qwryOH92jmpMSFIkOmGU\nEwxYt8sL133A3SsIrjAIxymK3P2QmUWADTHdWY8BZ9MhYYikisaWNl54O+hyennbftqizuIpo/nH\naxZy5aJJjB6qsqaSXIlOGGuB2WY2gyBRXAd8KraBmWUDNe4eBe4kuGOqfd/RZpbj7vuBZcC6BMcr\n0qvcnQ17DlFSFuGJjZXUHm1h4sjB3Hj+TIry8zhlvGpMSN+R0ITh7q1mdgvwDMFttfe7+xYzuwtY\n5+6rgaXA3WbmBF1SN4f7tpnZ7cALFtwXWAr8RyLjFektVbWNPLI+QklphHf31zMocwDLF06kKD+P\nj52SrRoT0ieZe+p0+xcWFvq6dboIkb7paHMbz75VRXFphN/vCGpMnDF9DEX5eVy2aBIjVWNCksTM\nSt29sLt2fWHQWyRluTvrdh+keF2E326u5EhTK7mjh/DVZbMpys9l2rhhyQ5RJG5KGCIJsKemgUfK\nynlkfYTd1Q0MDWtMFOXncdYM1ZiQ/kkJQ6SH1Ic1JkrKIry2swYIakx8bdlsli+cyLBB+u8m/Zv+\nBYuchGjUeW1nNcVlQY2JhuY2po8byt9cNIcV+bnkjVFZU0kdShgiJ+C9A/U8UhbhkbJyyg8dZcSg\nTK5ePJlVBXnkTx2jCf8kJSlhiMSp9mgLv90UdDmV7j7IAINzZ+fwzUvncbFqTEgaUMIQ6UJb1Hl1\n+35Kysp5dksVTa1RZo8fzh2XzmOFakxImlHCEDmG7XvrKC6L8Nj6cvYebmL00IF88owpFOXnsShv\nlLqcJC0pYYiEDtY388SmCopLI2yK1JIxwLhwbg7fvSqPC+eNZ1CmupwkvSlhSFpraYvy0tb9lJRG\neOGdvbS0OadOGsnfhzUmslVjQuQDShiSlt6qOExxaYTHN5RTXd/MuGFZ/OU50ynKz2P+ZNWYEDkW\nJQxJGweONPHY+qCs6duVh8nKGMDHTw3Kml4wN4eBqjEh0iUlDElpTa1tvBjWmFizNagxcXreKO66\negFXLprMmGGqMSESLyUMSTnuzqZILSVlEVZvrOBQQwsTRg7ii+fNYFV+HrMnjEh2iCL9khKGpIy9\nhxt5dH1Q1nTHviMMyhzAxQsmsqogj3NVY0LkpClhSL/W2NLGM1uqKCkr5/fb9xN1KJw2hrtXnsbl\nqjEh0qOUMKTfcXdKdx+kpCzCkxsrqQtrTNx84SmszM9jRrZqTIgkghKG9BuRgw08WlZOSVmEXdUN\nDBmYwaWnBV1OZ88YpxoTIgmmhCF9Wn1TK0+/GZQ1/dPOagDOnjmWmy88hUtPm8Rw1ZgQ6TX63yZ9\nTjTqvP5eDcWlEX73ZiUNzW1MGzeU2y6aw4oluUwZqxoTIsmghCF9xq6wxkRJTI2Jq06fTFFBHoXT\nVGNCJNmUMCSpDje28FRYY2LtroOYwbmnZPON5XO5eP5EhmRpwj+RvkIJQ3pdW9T5w44DlIRlTZta\no8zKGcY3ls9lxZJcJo0akuwQReQYlDCk1+zYV0dxaTmPrS+n6nAjo4YM5NrCKRQV5HG6akyI9HlK\nGJJQhxqaeWJjBcVl5Wzcc4iMAcbSOTl8+8r5fPxU1ZgQ6U/iShhmdjawxd3rwuWRwKnu/noc+y4H\nfgRkAD9z9x902D4NuB/IAWqAz7h7JNzWBmwOm77v7lfFdVSSVK1tUV7etp+SsgjPv7WP5rYo8yaO\n4O8uP5WrF+eSM0I1JkT6o3ivMH4K5McsHznGuo8wswzgXuAiIAKsNbPV7v5WTLN7gAfd/QEzWwbc\nDXw23HbU3RfHGaMk2duVhykpjfDYhgoOHGli7LAsPn32VIry81gweaS6nET6uXgThrm7ty+4e9TM\n4tn3TGCHu+8EMLOHgauB2IQxH7gtfLwGeCzOmKQPqD7SxOMbgrKmb1UeZmCGsWzeeFYVTGGpakyI\npJR4E8ZOM/sawVUFwE3Azjj2ywX2xCxHgLM6tNkIrCTotloBjDCzce5eDQw2s3VAK/ADd/9IMjGz\nG4EbAaZOnRrn4cjJaG6N8uI7eykuLeelrftojTqn5Y7iu1ct4MrTJzNWNSZEUlK8CePLwI+BvwMc\neIHwTboH3A78xMxuAF4ByoG2cNs0dy83s5nAi2a22d3fjd3Z3e8D7gMoLCx0JCHcnTfLD1NcuofV\nGys42NBCzohBfOHcGRQV5DFHNSZEUl5cCcPd9wHXncDzlwNTYpbzwnWxz11BcIWBmQ0Hitz9ULit\nPPy908xeApYAH0oYklj7whoTJWURtu09QlbmAC6eP4GigjzOOyWbTHU5iaSNeO+S+j8EVxYf4u5/\n1c2ua4HZZjaDIFFcB3yqw3NnAzXuHgXuJLhjCjMbAzS4e1PY5mPA/xdPvHJyGlvaeO6tvZSURXhl\nW1BjIn/qaL63YiFXLJrMqCGqMSGSjuLtknoy5vFggrGGiu52cvdWM7sFeIbgttr73X2Lmd0FrHP3\n1cBS4G4zc4IuqZvD3U8F/t3MosAAgjGMtz7yItIj3J2y9w9RXBrhyU0V1DW2MnnUYG5aegor83OZ\nmTM82SGKSJJZzM1P8e9kNgD4vbv/Rc+HdOIKCwt93bp1yQ6jXyk/dJRHwwn/3jtQH9SYWDiRooI8\nzpmpGhMi6cDMSt29sLt2J/pN79nA+BPcV5KsoTmoMVFSFuGP71bjDmfNGMtXls7iMtWYEJFOxDuG\nUUcwhmHh7yrgmwmMS3pYNOq8sauGktIIT22upL65jaljh3Lrx2dTlJ+nGhMi0q1475LSPZP91PvV\nDZSURSgpixA5eJThgzK5fNEkVhVM4YzpqjEhIvGLu+8hvGtpNsGgNwDu/koigpKTU9fYwu82B2VN\n39hVgxl8bFY2t188l0sWqMaEiJyYeLukvgjcSvA9ig3A2cCfgGWJC02OR1vU+eO7BygpjfD0lioa\nW6LMzBnG1y8JakxMHq0aEyJycuK9wrgVOAN4zd0vNLN5wPcTF5bE6939RygpjfDo+nIqaxsZOTiT\novw8igryWDJltLqcRKTHxJswGt290cwws0Hu/o6ZzU1oZNKp2oYWntgUTPi3Yc8hBhhcMCeHb11+\nKp84dQKDB6rLSUR6XrwJI2Jmowlmkn3OzA4CuxMXlnTU2hbl1e0HKC6N8Nzbe2lujTJ3wgj+9rJ5\nXLM4l/EjB3f/JCIiJyHeu6RWhA//wczWAKOAp9u3m9kYdz+YgPjS3jtVf64xsb+uiTFDB/KpM6ey\nqkA1JkSkdx33N7Tc/eVjrH6BboopSfxq6pt5fEMw4d+b5YfJHBDUmCgqyOPCuePJytSEfyLS+3rq\nK736mHuSmlujrNm6j5LSCC++E9SYWJg7ku9cOZ+rTp/MuOEqayoiydVTCUN1KE6Au7Ol4jDFpRFW\nb6ygpr6Z7OGD+PzHplNUkMe8iSOTHaKIyAc0aVAS7Ktr5PH1wV1OW/fWkZUxgIvmT2BVQR7nzVaN\nCRHpm9Ql1Uta2qI8u2UvxaV7eGX7AdqizpKpo/mnaxZy5aLJjBqqGhMi0rfF+03vscdYXefuLeHj\nj/dcSKnpnme38u8v72TSqMH89fkzKSrIY5ZqTIhIPxLvFUYZQanVgwRXE6OBKjPbC3zJ3UsTFF/K\neObNKs6bnc0vPn8mGaoxISL9ULyd5c8Bl7l7truPAy4lqMJ3E/C/EhVcqthT08Cu6gaWzRuvZCEi\n/Va8CeNsd3+mfcHdnwXOcffXAN3v2Y1Xtx8A4LzZOUmORETkxMXbJVVpZt8EHg6XPwnsNbMMIJqQ\nyFLIq9v3M2nUYGblDEt2KCIiJyzeK4xPEUxt/lj4MzVclwFcm5jQUkMw7Xg1583O1jQeItKvxTuX\n1AHgq51s3tFz4aSezeW11B5t4Vx1R4lIPxfvbbVzgNuB6bH7uLsKKHXj1W37MYNzT8lOdigiIicl\n3jGM/wb+N/AzoC1x4aSeV3ccYMHkkYwdlpXsUERETkq8CaPV3X+a0EhS0JGmVsp2H+SL581Mdigi\nIict3kHvJ8zsJjObZGZj23/i2dHMlpvZVjPbYWZ3HGP7NDN7wcw2mdlLZpbXYftIM4uY2U/ijLXP\neH1nNa1R5/zZ6o4Skf4v3iuMz4W/vx6zzoEuPzqHt93eC1wERIC1Zrba3d+KaXYP8KC7P2Bmy4C7\ngc/GbP9H4JU44+xTXt1+gMEDB1AwfUyyQxEROWnx3iU14wSf/0xgh7vvBDCzh4GrgdiEMR+4LXy8\nhuC2XcL2BcAEgup+hScYQ9K8/l4NhdPGMihTNbZFpP/rsksq/MSPma081k8cz58L7IlZjoTrYm0E\n2p9rBTDCzMaZ2QDg/ye4O6vfqT3awjtVhzljelw9dyIifV53VxgXAC8CVx5jmwOP9EAMtwM/MbMb\nCLqeygnuxLoJeMrdI1194c3MbgRuBJg6dWoPhNMzynYfxB3OmKHuKBFJDV0mDHf/Tvj78yf4/OUE\ns9y2ywvXxb5GBeEVhpkNB4rc/ZCZnQOcZ2Y3AcOBLDM74u53dNj/PuA+gMLCwj5T+e+NXTUMzDCW\nTFHCEJHUEO8X9wYBRXz0i3t3dbPrWmC2mc0gSBTXEUwpEvvc2UCNu0eBO4H7w+f+dEybG4DCjsmi\nL1v7Xg0Lc0cxJEvjFyKSGuK9rfZxgsHqVqA+5qdL7t4K3AI8A7wN/Mbdt5jZXWZ2VdhsKbDVzLYR\nDHB/77iOoA9qbGljU6RW4xciklLiva02z92Xn8gLuPtTwFMd1n075nExUNzNc/wC+MWJvH4ybIrU\n0twWVcIQkZQS7xXGH83stIRGkkLW7qoBoHCaxi9EJHXEe4VxLnCDmb0HNBGUaXV3X5SwyPqxN96r\nYc6E4YzR/FEikkLiTRiXJjSKFNIWdcp2H+TKxZOTHYqISI/qMmGY2Uh3PwzU9VI8/d47VYepa2rl\nTI1fiEiK6e4K41fAFUApwRf1Yr9B1+1cUulo7XvB+MUZM5QwRCS1dPfFvSvC3yc6l1TKunfNDoYM\nzODzH5v+QenVxpY2fru5ktzRQ8gdPSTJEYqI9Kx4xzAwszHAbGBw+zp375ezyPaEn770LkeaWnmz\nopa7V57G0eY2vvTgOtbuOsj3V+iGMhFJPfF+0/uLwK0EU3tsAM4G/gSkZYnWusYWjjS1Mm/iCB4p\nK2dPTQPV9c1Eao7yP69fwpWna8BbRFJPvFcYtwJnAK+5+4VmNg/4fuLC6tuqahsB+MrSWZgZt//3\nRoYMzOCXXzyLMzV2ISIpKt6E0ejujWaGmQ1y93fMbG5CI+vDKsOEMWnUEM6cMZaFk0cyeGAGkzVu\nISIpLN6EETGz0QTFjZ4zs4PA7sSF1bdVfZAwguGcmTnDkxmOiEiviLfi3orw4T+Y2RpgFEEVvLTU\nfoUxfuSgJEciItJ7uk0YYV3uLe4+D8DdX054VH1c1eGjZA/PUulVEUkr3U4+6O5tBNOP951ydklW\nWdvIxFFAz2z2AAAOwElEQVSDu28oIpJC4h3DGANsMbM3iKmD4e5Xdb5L6qqqbSRvzNBkhyEi0qvi\nTRiDCaYIaWfAP/d8OP1DZW2jal2ISNqJN2Fkdhy7MLO0vIe0obmV2qMt6pISkbTT3Wy1XwFuAmaa\n2aaYTSOAPyQysL6q4y21IiLpIp7Zan8H3A3cEbO+zt1rEhZVH1YV86U9EZF00t1stbVALXB974TT\n91XoCkNE0lS8Nb0lVFV7FEBjGCKSdpQwjlNlbSNjhg5k8EB9aU9E0osSxnGqqm1kosYvRCQNKWEc\np8raRo1fiEhaUsI4TlWHNS2IiKSnhCcMM1tuZlvNbIeZ3XGM7dPM7AUz22RmL5lZXsz6MjPbYGZb\nzOzLiY61O40tbdTUNzNppBKGiKSfhCaMcKbbe4FLgfnA9WY2v0Oze4AH3X0RcBfBdz4AKoFz3H0x\ncBZwh5kltfbp3sPBLbW6whCRdJToK4wzgR3uvtPdm4GHgas7tJkPvBg+XtO+3d2b3b0pXD+oF2Lt\nVqW+tCciaSzRb8K5wJ6Y5Ui4LtZGYGX4eAUwwszGAZjZlHBKkj3AP7t7RYLj7VL7t7x1hSEi6Sjp\nn9qB24ELzGw9cAFQDrQBuPuesKvqFOBzZjah485mdqOZrTOzdfv3709ooJX6lreIpLFEJ4xyYErM\ncl647gPuXuHuK919CfCtcN2hjm2AN4HzOr6Au9/n7oXuXpiTk9PT8X9IVe1RRg7OZNigeCf5FRFJ\nHYlOGGuB2WY2w8yygOuA1bENzCzbzNrjuBO4P1yf1z6FupmNAc4FtiY43i4F38HQ+IWIpKeEJgx3\nbwVuAZ4B3gZ+4+5bzOwuM2uv1reUoATsNmAC8L1w/anA62a2EXgZuMfdNycy3u6oNKuIpLOE9624\n+1PAUx3WfTvmcTFQfIz9ngMWJTq+eLk7FYeOsmDyyGSHIiKSFH1h0LtfeGbLXqrrmylUaVYRSVNK\nGHFobYvyL8+8w6ycYVyzOKnfHRQRSRoljDg8UlbOu/vr+folc8nM0J9MRNKT3v260djSxg+f38bp\nU0ZzyYKJyQ5HRCRplDC68cvXdlNZ28g3l8/FzJIdjohI0ihhdKGlLcq9a3Zw3uxs/mJWdrLDERFJ\nKiWMLuyurudgQwsrlnSc/kpEJP0oYXRha9URAOZMGJHkSEREkk8Jowvb9tYxwOCU8cOTHYqISNIp\nYXRh2946po0bxuCBGckORUQk6ZQwurBtbx2zdXUhIgIoYXSqqbWNXdUNzJ2o8QsREVDC6NTO/fW0\nRZ3ZGvAWEQGUMDq1bW8dAHOVMEREACWMTm3bW0fmAGNG9rBkhyIi0icoYXRia9URpmcPIytTfyIR\nEVDC6NT2fXXqjhIRiaGEcQxHm9t4v6aB2RN0S62ISDsljGPYse8I7hrwFhGJpYRxDO13SOmWWhGR\nP1PCOIZte+vIyhjA9HFDkx2KiEifoYRxDNv21jEzZ5jKsYqIxNA74jFs23tEU5qLiHSghNHBkaZW\nyg8dZY7ukBIR+RAljA6qao8CMGWsxi9ERGIlPGGY2XIz22pmO8zsjmNsn2ZmL5jZJjN7yczywvWL\nzexPZrYl3PbJRMcKUH2kGYCxw7J64+VERPqNhCYMM8sA7gUuBeYD15vZ/A7N7gEedPdFwF3A3eH6\nBuAv3X0BsBz4NzMbnch4AWrqlTBERI4l0VcYZwI73H2nuzcDDwNXd2gzH3gxfLymfbu7b3P37eHj\nCmAfkJPgeKkOE0b28EGJfikRkX4l0QkjF9gTsxwJ18XaCKwMH68ARpjZuNgGZnYmkAW82/EFzOxG\nM1tnZuv2799/0gG3d0mNGaorDBGRWH1h0Pt24AIzWw9cAJQDbe0bzWwS8J/A59092nFnd7/P3Qvd\nvTAn5+QvQGrqmxgxOFOz1IqIdJCZ4OcvB6bELOeF6z4QdjetBDCz4UCRux8Kl0cCvwW+5e6vJThW\nIOiSUneUiMhHJfpj9FpgtpnNMLMs4DpgdWwDM8s2s/Y47gTuD9dnAY8SDIgXJzjOD9TUN2vAW0Tk\nGBKaMNy9FbgFeAZ4G/iNu28xs7vM7Kqw2VJgq5ltAyYA3wvXXwucD9xgZhvCn8WJjBeCMQwlDBGR\nj0p0lxTu/hTwVId13455XAx85ArC3X8J/DLR8XVUXd9M/rSE370rItLvaGQ3RjTqHGzQFYaIyLEo\nYcSoPdpCW9QZO0yD3iIiHSlhxGj/0t44XWGIiHyEEkaM9mlBxg1XwhAR6UgJI0ZNfROgeaRERI5F\nCSPGgSPtXVIawxAR6UgJI4ZmqhUR6ZwSRoya+mbNIyUi0gm9M8aorm/WHVIiIp1QwohRfaRJ3VEi\nIp1QwohRU9/MOM1UKyJyTEoYMdQlJSLSOSWMUDTqHNTU5iIinVLCCB1ubKE16uqSEhHphBJGSPNI\niYh0TQkjpC/tiYh0TQkjVH0kmEdKEw+KiBybEkboz11SGsMQETkWJYxQTTjx4JhhA5MciYhI36SE\nEaqub2bEoEwGZWYkOxQRkT5JCSNUXd+s8QsRkS4oYYRq6jWPlIhIV5QwQtVHmhmrAW8RkU4pYYRq\n6pvJVpeUiEinlDAAd6dG80iJiHRJCQM4fLSV1qgrYYiIdEEJA6iu17e8RUS6Y+6e7Bh6jJntB3Yf\n527ZwIEEhNOX6ZjTRzoedzoeM5zccU9z95zuGqVUwjgRZrbO3QuTHUdv0jGnj3Q87nQ8Zuid41aX\nlIiIxEUJQ0RE4qKEAfclO4Ak0DGnj3Q87nQ8ZuiF4077MQwREYmPrjBERCQuShgiIhKXtE0YZrbc\nzLaa2Q4zuyPZ8SSCmU0xszVm9paZbTGzW8P1Y83sOTPbHv4ek+xYE8HMMsxsvZk9GS7PMLPXw3P+\nazNLqW9qmtloMys2s3fM7G0zOycdzrWZ/b/hv+83zewhMxuciufazO43s31m9mbMumOeXwv8ODz+\nTWaW3xMxpGXCMLMM4F7gUmA+cL2ZzU9uVAnRCvyNu88HzgZuDo/zDuAFd58NvBAup6Jbgbdjlv8Z\n+KG7nwIcBL6QlKgS50fA0+4+Dzid4NhT+lybWS7wNaDQ3RcCGcB1pOa5/gWwvMO6zs7vpcDs8OdG\n4Kc9EUBaJgzgTGCHu+9092bgYeDqJMfU49y90t3Lwsd1BG8guQTH+kDY7AHgmuREmDhmlgdcDvws\nXDZgGVAcNkmp4zazUcD5wM8B3L3Z3Q+RBucayASGmFkmMBSoJAXPtbu/AtR0WN3Z+b0aeNADrwGj\nzWzSycaQrgkjF9gTsxwJ16UsM5sOLAFeBya4e2W4qQqYkKSwEunfgG8A0XB5HHDI3VvD5VQ75zOA\n/cD/CbvhfmZmw0jxc+3u5cA9wPsEiaIWKCW1z3Wszs5vQt7j0jVhpBUzGw6UAP/D3Q/HbvPgvuqU\nurfazK4A9rl7abJj6UWZQD7wU3dfAtTTofspRc/1GIJP0zOAycAwPtptkxZ64/yma8IoB6bELOeF\n61KOmQ0kSBb/5e6PhKv3tl+ehr/3JSu+BPkYcJWZ7SLoblxG0L8/Ouy2gNQ75xEg4u6vh8vFBAkk\n1c/1J4D33H2/u7cAjxCc/1Q+17E6O78JeY9L14SxFpgd3kmRRTBItjrJMfW4sN/+58Db7v6vMZtW\nA58LH38OeLy3Y0skd7/T3fPcfTrBuX3R3T8NrAFWhc1S6rjdvQrYY2Zzw1UfB94ixc81QVfU2WY2\nNPz33n7cKXuuO+js/K4G/jK8W+psoDam6+qEpe03vc3sMoJ+7gzgfnf/XpJD6nFmdi7wKrCZP/fl\n/y3BOMZvgKkE08Ff6+4dB9NSgpktBW539yvMbCbBFcdYYD3wGXdvSmZ8PcnMFhMM8mcBO4HPE3wo\nTOlzbWbfBT5JcFfgeuCLBP31KXWuzewhYCnBNOZ7ge8Aj3GM8xsmz58QdM81AJ9393UnHUO6JgwR\nETk+6dolJSIix0kJQ0RE4qKEISIicVHCEBGRuChhiIhIXJQwREQkLkoYIl0ws5fMrLAXX+9fwqm6\n/6WT7dek6MzK0g9kdt9ERE6EmWXGTIAXrxuBse7e1sn2a4AnCb7N3BOvJxI3XWFISjCz6WHRoP8I\nP6E/a2ZDYq8QzCw7nF8KM7vBzB4Li87sMrNbzOy2cKbX18xsbMzTf9bMNoQFes4M9x8WFrR5I9zn\n6pjnXW1mLxLUJzhWrBZeSbxpZpvN7JPh+tXAcKC0fV2H/f4CuAr4lzCeWeHx/ZuZrQNuNbMcMysx\ns7Xhz8e6iXdBuG5DWGhndk+cD0lNusKQVDIbuN7dv2RmvwGKumm/kGDK98HADuCb7r7EzH4I/CXB\n1DEAQ919sZmdD9wf7vctgjmq/srMRgNvmNnzYft8YFEXU3CsBBYTFDnKBtaa2SvufpWZHXH3xcfa\nyd3/GCaVJ929GCCYAYIsd29Pir8iKBz0ezObCjwDnNpFvF8GfuTu/xXOq5bRzd9M0pgShqSS99x9\nQ/i4FJjeTfs1YWGpOjOrBZ4I128GFsW0ewiCAjZmNjJ8w72YYEbc28M2gwnm8wF4rpv5ms4FHgq7\nnfaa2cvAGZz4BJi/jnn8CWB+mEgARobT23cW75+Ab1lQcOoRd99+gjFIGlDCkFQSO7lcGzCEYEK6\n9q7XwV20j8YsR/nw/42OE645YECRu2+N3WBmZxHUouhNsa83ADjb3RtjG4ST0X0kXuBtM3udoDrh\nU2b21+7+YmLDlf5KYxiS6nYBBeHjVV2060r7GMO5BNNE1xJ09Xw1fCPGzJYcx/O9CnzSzDLMLIeg\ntOobce5bB4zoYvuzwFfbF8IZbOks3nAG353u/mOCqbEXIdIJJQxJdfcAXzGz9QTjBSeiMdz/fwNf\nCNf9IzAQ2GRmW8LleD0KbAI2Ai8C3wjrWcTjYeDr4cD1rGNs/xpQGA5gv0UwRtFVvNcCb5rZBoKx\nmQeP4zgkzWh6cxERiYuuMEREJC4a9BZJEDM7DfjPDqub3P2sOPb9FvD/dFj936lYGVL6D3VJiYhI\nXNQlJSIicVHCEBGRuChhiIhIXJQwREQkLv8XFiFnqbm3lEYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1088d5940>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model 'model_outliers' AUC : 0.9774800874356211\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEXCAYAAAC+mHPKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt8nWWZ7//Pt0nT8zkHSs8tLU0VBCac5FDaFAcdFQG3\ngKLg1sEDKFsHR9nOdmaY7Q/nN/5m1J9u9zDKHpgZQQdP6DAqLUVABSlysqQnUqBNS5IekqbpIadr\n//E8qYuQJqslK4e1vu/XK6+s57TW9fCUdeW+7+e5L0UEZmZm/Rk11AGYmdnI4IRhZmZZccIwM7Os\nOGGYmVlWnDDMzCwrThhmZpYVJwwbliQVSdovae5A7jtcSbpI0voh/PxfSHrfUH2+jQxOGDYg0i/s\n7p8uSQczlo/5iygiOiNiYkS8PJD7HitJ0yT9s6RXJO2TtFHSZwb6cyLioYh4w0C/L4CkRyVd12Pd\nKkkvZnz+WyLi3/p5n2JJIWl+LuK04a94qAOw/BARE7tfp19EH46I1UfbX1JxRHQMRmyv09eAImAp\nsA84GagcyA8YQf8tXpdCOc985haGDQpJ/1PSdyXdLakFuEbSuZIek9Qkaaekr0kane7/qr9mJf1r\nuv0/JbVI+o2kBce6b7r9rZI2SWqW9P9L+lXPv8AznAl8JyKaIqIrImoi4gcZ73WKpNWS9qStkD9P\n149NY9gpqU7S30sqSbetkvSipP8u6RXgn3r+xS9pu6RPS3oujfNuSWMytt+Sfl6dpD99vX/5Z7ZC\nJC2R9HD6ubskfSfd7eH09/q05XhFuv9HJW2RtFvSjyTNTNd3X5ePS9oCbJD0j5L+tsdn3y/pE8cb\nuw0eJwwbTJcB3wGmAN8FOoCbgFLgPOAS4CN9HP9e4H8A04GXgb851n0llQPfAz6Tfu5W4Kw+3ucx\n4DZJ10lanLlB0hRgNfATYCawBHgo3fwFoAo4FTg9Pb9bMg6fDUwE5gIfP8pnvwe4GFgI/BHw/vRz\n3w58AliRfubKPuI/Hl8E/gOYlsb5jXT9henvN6RdgN+X9BbgVuDdwCxgB9Cza+udJIn3FOBO4GpJ\nSs+lArgIuHuAz8FywAnDBtOjEfGT9C/1gxHxREQ8HhEdEVEL3A4s7+P4eyNiXUS0k3wpnXYc+74d\neDoifpxu+wdgVx/v83GS5PZJoEbS5vRLEpIvwpcj4qsRcTgi9kXEb9Nt7wP+KiIaI6KB5Ev1/Rnv\n25Fub4uIg0f57K9ExCsRsRv4acY5vAf4dtraaQX+uo/4u/2vtCXXJKkJ+FEf+7YD84GZEXEoIn7V\nx77vA74VEU9HxCHgc8BySbMz9vl/ImJves1/DRwiSRIAVwOrI6Kva2DDhBOGDaZtmQuSlkr6j+4B\nZZIv1dI+jn8l4/UBkr/Qj3XfEzPjiGT2ze1He5OIOBAR/zMizgBmAD8Avp+2LuYALxzl0BOBlzKW\nXyL5C7xbfUS09RF/1ufQ4/XRfDwipnb/AO/qY98/A0YD69IusWv72PdV5xkR+4C9vPpce8Z3F3BN\n+voa4F+yiN+GAScMG0w9p0b+R+D3wEkRMZmkG0c5jmEnSTcLAGnXyKyj7/4HEdEM3EbyxT2f5Itw\n0VF23wHMy1ieC9Rlvl3WEb/Wq86BJHENmIjYGREfjoiZwA3A7ekYUG8xv+o8JU0i6crq61z/BbhM\n0ukk//1+MpDxW+44YdhQmgQ0A62SKul7/GKg/BQ4Q9I7JBWTjKGUHW1nSX8pqUpSiaSxJF1Te4DN\nwH3AXEk3ShojabKk7vGQu4EvSCqVVEYynvKvA3QO3wM+JOlkSePT9x4wkt4jqTuJNpF84XdGRCew\nm2RMpdvdaSynpoPytwGPRERfrbaXgGdIxjP+Pe3KshHACcOG0p8B1wItJK2N7+b6AyOiHrgS+HuS\nL79FwFPA4T4OuzPddwdJ3/ufpF1VzSSD0lcA9cAm/jAG89ckX4q/B54FHif5Mh2Ic/gJ8E2Su5Y2\nA91jDH2dw7E4G3hCUitJF9wNGc+4/CXwnXQs5PKI+BlJV+IPSVo+c0nGNfpzJ8kguLujRhC5gJIV\nMklFJIng3RHxyFDHczwknQL8DhgTEV1DHU82JK0Evg0sDH8JjRhuYVjBkXSJpKlpF8r/ILkr6Lf9\nHDasSLos7SabDnwJ+PEIShYlJF2B/+RkMbI4YVghOh+oBRqBPwYui4iB6s4ZLDeQ3A68heQ21RuG\nNpzspK2hvSTPx3xtiMOxY+QuKTMzy4pbGGZmlpW8mnywtLQ05s+fP9RhmJnlVATsP9zOvkMd7DvY\nTkdXMHFMMQtKJxzX+z355JO7IuKot5d3y6uEMX/+fNatWzfUYZiZDbiGlkM8WNPA6poGHt3SyOH2\nLqaXFPGuJWVUV1aw4uQyZkwc0/8b9ULSS/3vlWcJw8wsX0QENTtbWF1Tz5qaep7Z3gzArKnjeE/V\nHFZVVnD2wumMKS4atJicMMzMhonDHZ08VruH1c8nSWJHc/IQ/JvmTOXmtyyhurKCpSdMIp3sd9A5\nYZiZDaHd+w/z4IYG1tQ08MjmRlrbOhk3uojzF5dy06rFrFhaTvmksUMdJuCEYWY2qCKCzQ37066m\nBn738l4ioGLyGC49fRarKst586JSxo4evK6mbDlhmJnlWFtHF0+8uIfVNfWsrqln256kBMobZ03m\nkysXs6qygjfOmjxkXU3ZcsIwM8uBpgNtPLSxkQdq6nl4YyMthzsoKR7FeYtm8NHli6heWsEJU4ZH\nV1O2nDDMzAZIbeP+tBXRwJMv7aWzKyidOIa3nTKT6spyzl9cyviSkfu1O3IjNzMbYh2dXax7aS9r\n0vGI2l2tACw9YRIfW76I6spy3jR7KqNGDe+upmw5YZiZHYN9h9r55cZG1tTUs3ZjI80H2xldJM5Z\nOINr3zyf6spyZk8bP9Rh5oQThplZP17efSC5q2lDPY/X7qGjK5g2fjTVleVcXFnBBUvKmDgm/79O\n8/8MzcyOUWdX8PS2vayuaWD18/VsbtgPwEnlE/nQBQu4uLKC0+dOoyhPupqy5YRhZgbsP9zBI5sa\nWV3TwNqNDexpbaN4lDhz/nSuOmsuqyrLmTfj+Cb3yxdOGGZWsOqaDrImvavpsRd209bZxeSxxaxY\nWk51ZQXLl5QxZdzooQ5z2HDCMLOC0dUVPFvXfCRJ1OzcB8CC0gl84Nx5VFdWUDV/GqOLXCqoN04Y\nZpbXDrZ18uiWXcmtrxsaaGw5zChB1bzp3PLWpaxaVsGisolDHeaI4IRhZnmnft8h1tQ0sLqmnl9t\n2cXhji4mjilm+ZIyVi0r56Il5UybUDLUYY44ThhmNuJFBOt37Dsyod9zdUntiNnTxnH1WXNZVVnB\nWQumU1LsrqbXwwnDzEakQ+2d/OaF3ayuqefBDQ3sbD6EBKfPmcpn/vhkVlVWsKRi4rCf0G8kccIw\nsxGjseUwazckXU2PbtnFgbZOxpcUccHiUj518RJWLi2n9DjLlFr/cp4wJF0CfBUoAr4VEV/qsX0e\ncAdQBuwBromI7ZJWAP+QsetS4KqI+FGuYzaz4SEi2FjfcmQ84ultTUTAzCljufyMWVRXVnDuwhnD\nsnZEPsppwpBUBHwDuBjYDjwh6b6IeD5jty8Dd0XEnZJWArcB74+ItcBp6ftMB7YAv8hlvGY29No6\nunh86+4jSWL73qR2xKmzp/CpVUuorixn2czhXzsiH+W6hXEWsCUiagEk3QNcCmQmjGXAp9PXa4He\nWhDvBv4zIg7kMFYzGyJ7W9tYuzFJEA9v2sX+wx2MKR7FBYtLuWHFSaxcWk7F5JFVOyIf5TphzAK2\nZSxvB87usc8zwOUk3VaXAZMkzYiI3Rn7XAX8fS4DNbPBExG80Nia3tVUz5Mv7aUroHzSGN7xpplU\nL63gvJNKGVfirqbhZDgMet8MfF3SdcDDQB3Q2b1R0kzgFODnvR0s6XrgeoC5c+fmOlYzO07tnUmZ\n0jU1DaypqefF3UmHwbKZk7lxxUlUV1ZwyqwpeVM7Ih/lOmHUAXMylmen646IiB0kLQwkTQSuiIim\njF3eA/wwItp7+4CIuB24HaCqqioGLnQze72aD7Tz0KYG1tQ08NDGBvYd6qCkaBTnLprBh85fwMrK\nCmZNHTfUYVqWcp0wngAWS1pAkiiuAt6buYOkUmBPRHQBt5DcMZXp6nS9mY0AL+5qTcuU1vPEi0mZ\n0hkTSnjLG05gVWUFFywuZUIB1I7IRzm9ahHRIelGku6kIuCOiFgv6VZgXUTcB1wE3CYpSLqkbug+\nXtJ8khbKL3MZp5kdv86u4Hcv72X180mSeKExKVO6pGIiH7lwIdWVFZw2Z2rB1Y7IR4rIn16cqqqq\nWLdu3VCHYZb3Wg6188jmXax+vp61GxvYe6Cd4lFJmdLqynJWVVYwZ3p+linNR5KejIiq/vZzu9DM\nsrJtz4EjM74+Vrub9s5g6vjRrDi5nOrKci5cUsbksa4dkc+cMMysV11dwdPbm5IkUdPAhldaAFhY\nNoEPnreAVZUVnDF3KsWuHVEwnDDM7IgDbR08sjmpHfHghgZ27W+jaJSomjeNv/iTSqorK1hQWthl\nSguZE4ZZgdvZfPDINBy/fmE3bR1dTBpbzEUnl7OqspzlS8qYOt61I8wJw6zgdHUFv9/RzOr0Abr1\nO5IypfNmjOeas+exqrKcMxdMd5lSew0nDLMCcKi9k19t2cXqmgYe3FBP/b6kTOkZc6fx2UuWsqqy\nnJPKXTvC+uaEYZanGloO8WBNA6trGnh0SyOH2ruYUFLEhUvKqK6sYMXJZcxw7Qg7Bk4YZnkiIqjZ\n2cKamnpWb2jgmW3JDDuzpo7jyqo5VFdWcPbC6Ywp9oR+dnycMMxGsMMdnTxWu+fIra91TUntiDfN\nmcrNb1lCdWUFS0+Y5K4mGxBOGGYjzO79h3lwQzKh3yObG2lt62Tc6CLOX1zKJ6tPYsXScsonuXaE\nDTwnDLNhLiLY3LA/rR3RwO9e3ksEVEwew6Wnz2JVZTlvXlTqMqWWc04YZsNQe2cXv92650iSeHlP\nUjvijbMm88mVi1lVWcEbZ7lMqQ0uJwyzYaLpQBsPbWxkdU09v9zYSMvhDkqKR3Heohl8ZPlCqpdW\ncMIUdzXZ0HHCMBtCtY37WVPTwANpmdLOrqB04hjedspMqivLOX9xKeNL/L+pDQ/+l2g2iDo6u3jy\npb1HuppqdyW1I5aeMImPLV9EdWU5b5o91WVKbVhywjDLsX2H2vnlxkbW1NSzdmMjzQfbGV2U1I64\n9s3zqa4sZ/Y0146w4c8JwywHXt59IGlFbKjn8do9dHQF08aPprqynIsrK7hgSRkTXabURhj/izUb\nAJ1dwdPb9h6Z0G9T/X4ATiqfyIcvWMiqynJOnzvNZUptRHPCMDtOrYc7eGRzIw8838BDGxvY3dpG\n8Shx1oLpXHnmXFZVljNvhmtHWP5wwjA7BnVNB5O5mmoaeOyF3bR1djF5bDErlpZTXVnB8iVlTBnn\nMqWWn5wwzPrQ1RU8W9d8JEnU7ExqRywoncAHzp1HdWUFVfOnuXaEFQQnDLMeDrZ18uiWpEzpmg0N\nNLYktSOq5k3nlrcuZdWyChaVTRzqMM0GnROGGVC/7xBr0gHrR7fs4nBHFxPHFLN8SRmrlpVz0ZJy\npk1wmVIrbE4YVpAigvU79h2pZf1cXTMAs6eN4+qz5rKqsoKzFkynpNhdTWbdcp4wJF0CfBUoAr4V\nEV/qsX0ecAdQBuwBromI7em2ucC3gDlAAG+LiBdzHbPlp0Ptnfymdjern6/nwQ0N7Gw+hASnz5nK\nZ/74ZFZVVrCkwmVKzY4mpwlDUhHwDeBiYDvwhKT7IuL5jN2+DNwVEXdKWgncBrw/3XYX8MWIeEDS\nRKArl/Fa/mlsOczaDUkr4tEtuzjQ1sn4kiIuWFzKpy5ewsql5ZS6TKlZVnLdwjgL2BIRtQCS7gEu\nBTITxjLg0+nrtcCP0n2XAcUR8QBAROzPcayWByKCjfUtR7qant7WRATMnDKWy8+YRXVlBecunOHa\nEWbHIdcJYxawLWN5O3B2j32eAS4n6ba6DJgkaQawBGiS9ANgAbAa+FxEdGYeLOl64HqAuXPn5uIc\nbJhr6+ji8a27jySJ7XuTMqWnzp7Cp1YtobqynGUzXTvC7PUaDoPeNwNfl3Qd8DBQB3SSxHYBcDrw\nMvBd4Drg25kHR8TtwO0AVVVVMVhB29Da29rG2o1Jgnh40y72H+5gTPEoLlhcyg0rTmLl0nIqJrt2\nhNlAynXCqCMZsO42O113RETsIGlhkI5TXBERTZK2A09ndGf9CDiHHgnDCkNE8EJja/oAXVI7oiug\nfNIY3vGmmVQvreC8k0oZV+KuJrNcyXXCeAJYLGkBSaK4Cnhv5g6SSoE9EdEF3EJyx1T3sVMllUVE\nI7ASWJfjeG0Yae/s4okX9xx5PuLF3UmZ0mUzJ3PjipOorqzglFlTXDvCbJDkNGFERIekG4Gfk9xW\ne0dErJd0K7AuIu4DLgJukxQkXVI3pMd2SroZWKOk8/lJ4J9yGa8NveaD7Ty0sYE1NcmEfvsOdVBS\nNIpzF83gQ+cvYGVlBbOmjhvqMM0KkiLyp9u/qqoq1q1zI2SkeXFX65EKdE+8mNSOmDGhhBVLy1lV\nWcEFi0uZ4NoRZjkj6cmIqOpvP/9faIOusyv43ctJmdLVz9fzQmNSpvTkiklcf+FCqisrOG3OVNeO\nMBtmnDBsULQcaueRzbtY/Xw9azc2sPdAO8WjkjKl15wzj1WVFcyZ7jKlZsOZE4blzLY9B47M+PpY\n7W7aO4Op40ez4uRyqivLuXBJGZPHunaE2UjhhGEDpqsreGZ705HxiA2vtACwsGwCHzxvAasqKzhj\n7lSKXTvCbERywrDX5UBbB49sTmpHPLihkV37D1M0SlTNm8Zf/Ekl1ZUVLCh1mVKzfOCEYcdsZ/PB\nI89G/OqF3bR1dDFpbDEXnVzOqspyli8pY+p4144wyzdOGNaviOD3dfuSu5pq6lm/IylTOm/GeK45\nex6rKss5c8F0lyk1y3NOGNarQ+2d/PqFXTzwfAMPbqinfl9SpvSMudP47CVLWVVZzknlrh1hVkic\nMOyIhpZDPFjTwOqaBh7d0sih9i4mlBRx4ZIyqisrWHFyGTNcO8KsYDlhFLCIoGZnSzKh34YGntnW\nBMCsqeO4smoO1ZUVnL1wOmOKPaGfmTlhFJzDHZ08VrsneT6ipoG6pqR2xGlzpnLzW5ZQXVnB0hMm\nuavJzF7DCaMA7N5/mLUbG1lTU8/Dmxppbetk3Ogizl9cyierT2LF0nLKJ7l2hJn1zQkjD0UEWxr2\nszqtQPe7l/cSARWTx3Dp6bNYVVnOmxeVukypmR0TJ4w80d7ZxW+37jnylPXLe5LaEW+cNZlPrlzM\nxcsqeMOJLlNqZsfPCWMEazrQxkMbG1ldU88vNzXScqiDkuJRnLdoBh9ZvpDqpRWcMMVdTWY2MJww\nRpjaxv2sSbua1r20l86uoHTiGN72xplUV5Zz/uJSxpf4sprZwPM3yzDX0dnFky/tZc2GBlY/X0/t\nrqR2xNITJvGx5YuoriznTbOnukypmeWcE8YwtO9QOw9vamT18/U8tKmRpgPtjC5Kakdc++b5VFeW\nM3uaa0eY2eDKKmFIOgdYHxEt6fJkoDIiHs9lcIXk5d0HkgHrDfU8XpuUKZ02fjQrl5ZzcWUFFywp\nY6LLlJrZEMr2G+ibwBkZy/t7WWfHoLMreHrbXlans75uqt8PwEnlE/nwBQtZVVnO6XOnuUypmQ0b\n2SYMRUR0L0RElyT/uXuMWg938MjmRlbXNLB2QwO7W9soHiXOWjCdK8+cy6rKcubNcO0IMxuesv3S\nr5X0SZJWBcDHgdrchJRfdjQdTOZqqmngNy/spq2zi8lji1mxtJzqygqWLyljyjiXKTWz4S/bhPFR\n4GvAXwABrAGuz1VQ+WBPaxuf+/6z/OL5egAWlE7gA+fOo7qygqr501w7wsxGnKwSRkQ0AFcdzwdI\nugT4KlAEfCsivtRj+zzgDqAM2ANcExHb022dwHPpri9HxDuPJ4bB9ljtbm665yn2trZzU/Vi3nna\niSwqmzjUYZmZvS7Z3iX1f0haFq8SEf+1n+OKgG8AFwPbgSck3RcRz2fs9mXgroi4U9JK4Dbg/em2\ngxFxWjYxDhd3//ZlPv/D55g3YwLfvvZM3jhrylCHZGY2ILLtkvppxuuxwGXAjiyOOwvYEhG1AJLu\nAS4FMhPGMuDT6eu1wI+yjGlYuvPXL/KGE6dw9/Xn+DZYM8srWXWkR8T3M37+DXgPUJXFobOAbRnL\n29N1mZ4BLk9fXwZMkjQjXR4raZ2kxyS9q7cPkHR9us+6xsbGbE4nZzo6u6htbOXNi2Y4WZhZ3jne\nkdfFQPkAxXAzsFzSU8ByoA7oTLfNi4gq4L3AVyQt6nlwRNweEVURUVVWVjZAIR2fl/YcoK2zi8UV\nk4Y0DjOzXMh2DKOFZAxD6e9XgM9mcWgdMCdjeXa67oiI2EHawpA0EbgiIprSbXXp71pJDwGnAy9k\nE/NQ2FzfAsCSCg9wm1n+yfYuqeP9k/kJYLGkBSSJ4iqS1sIRkkqBPRHRBdxCcscUkqYBByLicLrP\necD/e5xxDIrMp7XNzPJN1h3t6Rf4YpJBbwAi4uG+jomIDkk3Aj8nua32johYL+lWYF1E3AdcBNwm\nKYCHgRvSwyuBf5TURdJ19qUed1cNO5vqW5gzfZynFzezvJRtl9SHgZtIupSeBs4BfgOs7O/YiLgf\nuL/Hui9kvL4XuLeX434NnJJNfMPF5vr9LC73+IWZ5adsB71vAs4EXoqIFSRjCU05i2oEau/sonbX\nfhZ7/MLM8lS2CeNQRBwCkDQmIjYAJ+curJHnpd2ttHcGS9zCMLM8lW1n+3ZJU0keqntA0l7gpdyF\nNfJsTge8l/iWWjPLU9neJXVZ+vKvJK0FpgA/694uaVpE7M1BfCPGpvr9SL5Dyszy1zHfzhMRv+xl\n9RoKvJjSpoYW5kwbz7iSoqEOxcwsJwZqju2CLwu3ub7FD+yZWV4bqITxmplsC0l7Zxdbd7V6ShAz\ny2uu4jMAXtyV3iHlFoaZ5TF3SQ2A7ilB/NCemeWzbJ/0nt7L6paIaE9fVw9cSCPPpvoW3yFlZnkv\n27ukfkcy6+xektbEVOAVSfXAn0bEkzmKb0TY0rCfudPHM3a075Ays/yVbZfUA8DbIqI0ImYAbyWp\nwvdx4H/lKriRYnNDC4vdujCzPJdtwjgnIn7evRARvwDOjYjHgDE5iWyE6OwKXtx1gEVOGGaW57Lt\nktop6bPAPenylUC9pCKgKyeRjRDb9yZV9haVOmGYWX7LtoXxXpKpzX+U/sxN1xWR1PcuWLWNrQAs\nLJswxJGYmeVWtnNJ7QI+cZTNWwYunJHnhcbkltqFZW5hmFl+y/a22iXAzcD8zGMiot8CSvmudlcr\nU8ePZvqEkqEOxcwsp7Idw/h34H8D3wI6cxfOyFPbuJ+Fpe6OMrP8l23C6IiIb+Y0khHqhcZWli8p\nG+owzMxyLttB759I+rikmZKmd//kNLIRoOVQO40th1nk8QszKwDZtjCuTX9/JmNdAAsHNpyRxXdI\nmVkhyfYuqQW5DmQkqt2V3CG1yAnDzApAnwlD0sqIeFDS5b1tj4gf5CaskaG2sZWiUWLudCcMM8t/\n/Y1hLE9/v6OXn7dn8wGSLpG0UdIWSZ/rZfs8SWskPSvpIUmze2yfLGm7pK9n83mDqbaxlTnTxlFS\n7LIiZpb/+mxhRMRfpr8/eDxvnk4d8g3gYmA78ISk+yLi+YzdvgzcFRF3SloJ3Aa8P2P73wAPH8/n\n59oLjfv9wJ6ZFYxsH9wbA1zBax/cu7WfQ88CtkREbfo+9wCXApkJYxnw6fT1WpKpR7o/94+ACuBn\nQFU2sQ6Wrq5g665Wzj+pdKhDMTMbFNn2pfyY5Iu+A2jN+OnPLGBbxvL2dF2mZ4DuMZLLgEmSZkga\nBfx/JE+YH5Wk6yWtk7SusbExi5AGRl3TQQ53dLmFYWYFI9vbamdHxCU5iuFm4OuSriPpeqojeZr8\n48D9EbFdOnoF2Ii4HbgdoKqqKnIU42vU7krype+QMrNCkW3C+LWkUyLiuWN8/zqSSn3dZqfrjoiI\nHaQtDEkTgSsioknSucAFkj4OTARKJO2PiNcMnA+FWk86aGYFJtuEcT5wnaStwGGSMq0REaf2c9wT\nwGJJC0gSxVUk06IfIakU2BMRXcAtwB0kb/6+jH2uA6qGS7KA5A6pSWOLKZ3oSQfNrDBkmzDeejxv\nHhEdkm4Efk5SO+OOiFgv6VZgXUTcB1wE3CYpSLqkbjiezxps3XdI9dVdZmaWT/p7cG9yROwDWo73\nAyLifuD+Huu+kPH6XuDeft7jn4F/Pt4YcmHrrlbOXThjqMMwMxs0/bUwvkPygN6TJHNHZf45XbBz\nSR1o62Bn8yHPIWVmBaW/B/fenv72XFIZtqZ3SC1wHW8zKyDZjmEgaRqwGBjbvS4ihuUT2LnWnTDc\nwjCzQpLtk94fBm4iuS32aeAc4DdAQZZo7Z7WfP4MJwwzKxzZPul9E3Am8FJErABOB5pyFtUwt3VX\nKydOGcu4kqKhDsXMbNBkmzAORcQhSOaViogNwMm5C2t4q93V6gf2zKzgZJswtkuaSjIx4AOSfgy8\nlLuwhq+IoLZxPwtK3R1lZoUl24p7l6Uv/0rSWmAKyQyyBWd3axsthzqcMMys4PSbMNKaFusjYilA\nRPwy51ENY75DyswKVb9dUhHRCWyUNHcQ4hn2tqZ3SC30MxhmVmCyfQ5jGrBe0m/JqIMREe/MSVTD\n2Au79lNSNIpZ08YNdShmZoMq24QxllfX8BbwtwMfzvC3tbGVeTPGUzTKkw6aWWHJNmEU9xy7kFSQ\nf2Jv3dXqAW8zK0h9jmFI+pik54CTJT2b8bMVeHZwQhw+OruCl3YfYIEHvM2sAGUzW+1/ArcBmcWL\nWiJiT86iGqbq9h6krbOLRR7wNrMC1N9stc1AM3D14IQzvNXuSsqyuoVhZoUo2ye9jT9MOugxDDMr\nRE4Yx2BJiThlAAAOjUlEQVTrrlYmjy1mxgTX8TazwuOEcQy27mplget4m1mBcsI4Blt3tbLQ3VFm\nVqCcMLLU1tHFjuaDzJsxfqhDMTMbEk4YWdrZfJAImDW1IJ9XNDNzwshW3d6DAJ5DyswKVs4ThqRL\nJG2UtEXS53rZPk/SmvQJ8ockzc5Y/ztJT0taL+mjuY61L9ubkoQxe6q7pMysMOU0YaS1NL4BvBVY\nBlwtaVmP3b4M3BURpwK3kjxVDrATODciTgPOBj4n6cRcxtuXur0HkeCEKWOHKgQzsyGV6xbGWcCW\niKiNiDbgHuDSHvssAx5MX6/t3h4RbRFxOF0/ZhBi7dP2vQepmDSWkmL34plZYcr1t98sYFvG8vZ0\nXaZngMvT15cBkyTNAJA0R9Kz6Xv8bUTsyHG8R1XXdMDjF2ZW0IbDn8s3A8slPQUsB+qAToCI2JZ2\nVZ0EXCupoufBkq6XtE7SusbGxpwFWdd00HdImVlBy3XCqAPmZCzPTtcdERE7IuLyiDgd+Hy6rqnn\nPsDvgQt6fkBE3B4RVRFRVVZWNtDxA8m05jubDjHbLQwzK2C5ThhPAIslLZBUAlwF3Je5g6RSSd1x\n3ALcka6f3V2kSdI04HxgY47j7VVDyyE6usJdUmZW0HKaMCKiA7gR+DlQA3wvItZLulVSdz3wi4CN\nkjYBFcAX0/WVwOOSngF+CXw5Ip7LZbxHc+QZDHdJmVkBy7ZE63GLiPuB+3us+0LG63uBe3s57gHg\n1FzHl4267mcw3MIwswI2HAa9h73taQvjRLcwzKyAOWFkoa7pINMnlDC+JOcNMjOzYcsJIwt1e31L\nrZmZE0YW/AyGmZkTRr8igu17/ZS3mZkTRj/2tLZxqL3LLQwzK3hOGP3ovqXWLQwzK3ROGP3ofmjP\nz2CYWaFzwuhHnQsnmZkBThj92r73IBPHFDN5nJ/BMLPC5oTRj+5baiUNdShmZkPKCaMfdXsPesDb\nzAwnjH75oT0zs4QTRh8OtnXSfLCdmVPHDnUoZmZDzgmjD00H2wCYNr5kiCMxMxt6Thh9aDrQDsDU\ncaOHOBIzs6HnhNGH7oQxZbwThpmZE0Yfmg92tzDcJWVm5oTRh+Z0DMMtDDMzJ4w+eQzDzOwPnDD6\n0HSwndFFYnxJ0VCHYmY25Jww+tB8sJ0p40o8LYiZGU4YfWo+0M4UTzpoZgY4YfSp6WAbU/3QnpkZ\nMAgJQ9IlkjZK2iLpc71snydpjaRnJT0kaXa6/jRJv5G0Pt12Za5j7anpQLsHvM3MUjlNGJKKgG8A\nbwWWAVdLWtZjty8Dd0XEqcCtwG3p+gPAByLiDcAlwFckTc1lvD0lYxhOGGZmkPsWxlnAloiojYg2\n4B7g0h77LAMeTF+v7d4eEZsiYnP6egfQAJTlON5XaT7Q7mcwzMxSuU4Ys4BtGcvb03WZngEuT19f\nBkySNCNzB0lnASXACz0/QNL1ktZJWtfY2Dhggbd3dtFyuMNPeZuZpYbDoPfNwHJJTwHLgTqgs3uj\npJnAvwAfjIiungdHxO0RURURVWVlA9cA2dc9LYhbGGZmAOT6ntE6YE7G8ux03RFpd9PlAJImAldE\nRFO6PBn4D+DzEfFYjmN9le55pDyGYWaWyHUL4wlgsaQFkkqAq4D7MneQVCqpO45bgDvS9SXAD0kG\nxO/NcZyv0XTQM9WamWXKacKIiA7gRuDnQA3wvYhYL+lWSe9Md7sI2ChpE1ABfDFd/x7gQuA6SU+n\nP6flMt5MzZ5HyszsVXL+GHNE3A/c32PdFzJe3wu8pgUREf8K/Guu4zua7mp7fnDPzCwxHAa9h6Xu\nFobHMMzMEk4YR9E9hjF5rOeSMjMDJ4yjajrQzqSxxRQX+T+RmRk4YRxV88F2P4NhZpbBCeMomg60\nefzCzCyDE8ZRNB9s97QgZmYZnDCOoumgJx40M8vkhHEUza6FYWb2Kk4YvYiIpIXhhGFmdoQTRi9a\n2zrp7ArfJWVmlsEJoxdNB9JpQTzobWZ2hBNGL5oOeKZaM7OenDB64VoYZmav5YTRi2ZX2zMzew0n\njF40HamF4TEMM7NuThi9+EMtDLcwzMy6OWH0ovlAOyXFoxg7umioQzEzGzacMHqRzCPl1oWZWSYn\njF40HfDU5mZmPTlh9KLpYJsHvM3MenDC6EXTgXYmu0vKzOxVnDB6sc/V9szMXsMJoxdNHvQ2M3uN\nnCcMSZdI2ihpi6TP9bJ9nqQ1kp6V9JCk2RnbfiapSdJPcx1nt8MdnRxo63QLw8ysh5wmDElFwDeA\ntwLLgKslLeux25eBuyLiVOBW4LaMbX8HvD+XMXbr7Aoe2tjAJ77zFADTJnjQ28wsU3GO3/8sYEtE\n1AJIuge4FHg+Y59lwKfT12uBH3VviIg1ki7KcYys39HM9Xc9SV3TQaZPKOH6CxfyjjedmOuPNTMb\nUXKdMGYB2zKWtwNn99jnGeBy4KvAZcAkSTMiYnc2HyDpeuD6dHG/pI3HGGMpsKt74SXgKeDzx/gm\nI8yrzrlAFOI5Q2GedyGeM7y+856XzU65ThjZuBn4uqTrgIeBOqAz24Mj4nbg9uP9cEnrIqLqeI8f\niXzOhaMQz7sQzxkG57xznTDqgDkZy7PTdUdExA6SFgaSJgJXRERTjuMyM7NjlOu7pJ4AFktaIKkE\nuAq4L3MHSaWSuuO4BbgjxzGZmdlxyGnCiIgO4Ebg50AN8L2IWC/pVknvTHe7CNgoaRNQAXyx+3hJ\njwD/DlRL2i7pj3MQ5nF3Z41gPufCUYjnXYjnDINw3oqIXH+GmZnlAT/pbWZmWXHCMDOzrBRswuhv\nypJ8IGmOpLWSnpe0XtJN6frpkh6QtDn9PW2oY80FSUWSnuqeWia9+eLx9Jp/N70RI29ImirpXkkb\nJNVIOrcQrrWkT6X/vn8v6W5JY/PxWku6Q1KDpN9nrOv1+irxtfT8n5V0xkDEUJAJI8spS/JBB/Bn\nEbEMOAe4IT3PzwFrImIxsCZdzkc3kdxs0e1vgX+IiJOAvcCHhiSq3Pkq8LOIWAq8ieTc8/paS5oF\nfBKoiog3AkUkd2Pm47X+Z+CSHuuOdn3fCixOf64HvjkQARRkwiBjypKIaAO6pyzJKxGxMyJ+l75u\nIfkCmUVyrnemu90JvGtoIsyddBLLPwG+lS4LWAncm+6SV+ctaQpwIfBtgIhoS59nyvtrTfI82ThJ\nxcB4YCd5eK0j4mFgT4/VR7u+l5LM0RcR8RgwVdLM1xtDoSaM3qYsmTVEsQwKSfOB04HHgYqI2Jlu\neoXkduZ88xXgz4GudHkG0JTe6g35d80XAI3A/0m74b4laQJ5fq0joo5kAtOXSRJFM/Ak+X2tMx3t\n+ubkO65QE0ZBSZ+g/z7w3yJiX+a2SO6rzqt7qyW9HWiIiCeHOpZBVAycAXwzIk4HWunR/ZSn13oa\nyV/TC4ATgQm8ttumIAzG9S3UhNHvlCX5QtJokmTxbxHxg3R1fXfzNP3dMFTx5ch5wDslvUjS3biS\npH9/atptAfl3zbcD2yPi8XT5XpIEku/XehWwNSIaI6Id+AHJ9c/na53paNc3J99xhZow+p2yJB+k\n/fbfBmoi4u8zNt0HXJu+vhb48WDHlksRcUtEzI6I+STX9sGIeB/J9PnvTnfLq/OOiFeAbZJOTldV\nk5QRyOtrTdIVdY6k8em/9+7zzttr3cPRru99wAfSu6XOAZozuq6OW8E+6S3pbST93EXAHRHxxX4O\nGXEknQ88AjzHH/ry/zvJOMb3gLkkM7q/JyJ6DqblhbSeys0R8XZJC0laHNNJZrG/JiIOD2V8A0nS\naSSD/CVALfBBkj8K8/paS/pr4EqSuwKfAj5M0l+fV9da0t0kUymVAvXAX5LUD3rN9U2T59dJuucO\nAB+MiHWvO4ZCTRhmZnZsCrVLyszMjpEThpmZZcUJw8zMsuKEYWZmWXHCMDOzrDhhmJlZVpwwzPog\n6SFJVYP4eX+XTtX9d0fZ/q48nVnZRoDi/ncxs+MhqThjArxsXQ9Mj4jOo2x/F/BTkqeZB+LzzLLm\nFoblBUnz06JB/5T+hf4LSeMyWwiSStP5pZB0naQfpUVnXpR0o6RPpzO9PiZpesbbv1/S02mBnrPS\n4yekBW1+mx5zacb73ifpQZL6BL3FqrQl8XtJz0m6Ml1/HzAReLJ7XY/j3gy8E/i7NJ5F6fl9RdI6\n4CZJZZK+L+mJ9Oe8fuJ9Q7ru6bTQzuKBuB6Wn9zCsHyyGLg6Iv5U0veAK/rZ/40kU76PBbYAn42I\n0yX9A/ABkqljAMZHxGmSLgTuSI/7PMkcVf9V0lTgt5JWp/ufAZzaxxQclwOnkRQ5KgWekPRwRLxT\n0v6IOK23gyLi12lS+WlE3AuQzABBSUR0J8XvkBQOelTSXODnQGUf8X4U+GpE/Fs6r1pRP//NrIA5\nYVg+2RoRT6evnwTm97P/2rSwVIukZuAn6frngFMz9rsbkgI2kianX7hvIZkR9+Z0n7Ek8/kAPNDP\nfE3nA3en3U71kn4JnMnxT4D53YzXq4BlaSIBmJxOb3+0eH8DfF5JwakfRMTm44zBCoAThuWTzMnl\nOoFxJBPSdXe9ju1j/66M5S5e/f9GzwnXAhBwRURszNwg6WySWhSDKfPzRgHnRMShzB3SyeheEy9Q\nI+lxkuqE90v6SEQ8mNtwbaTyGIbluxeBP0pfv7uP/frSPcZwPsk00c0kXT2fSL+IkXT6MbzfI8CV\nkooklZGUVv1tlse2AJP62P4L4BPdC+kMthwt3nQG39qI+BrJ1NinYnYUThiW774MfEzSUyTjBcfj\nUHr8/wY+lK77G2A08Kyk9elytn4IPAs8AzwI/HlazyIb9wCfSQeuF/Wy/ZNAVTqA/TzJGEVf8b4H\n+L2kp0nGZu46hvOwAuPpzc3MLCtuYZiZWVY86G2WI5JOAf6lx+rDEXF2Fsd+HvgvPVb/ez5WhrSR\nw11SZmaWFXdJmZlZVpwwzMwsK04YZmaWFScMMzPLyv8FOFV9547xvhkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x108857748>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model 'model_ensemble' AUC : 0.9723967898113478\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEXCAYAAAC+mHPKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XucXWV97/HPN5MbkECuIuRCEgxgBIQwJrGCIFUL1oKA\nVVAotNrUIuqpYgvHc7ykx4M95bTqkVpRUfACIipGSkXkIlRNyIRLuBkIk4RMwmXI5EpI5vY7f6xn\nwsrOXHYms2bP7Pm+X695zV63vX8rG57frOdZ6/kpIjAzM+vJsEoHYGZmg4MThpmZlcUJw8zMyuKE\nYWZmZXHCMDOzsjhhmJlZWZwwbECSVCNpu6TpfbnvQCXpNEmPV/DzfyXpg5X6fBscnDCsT6QGu+On\nXdIrueV9bogioi0ixkTEs325776SNF7SdyU9L2mrpJWSPt3XnxMR90bEG/r6fQEk/ZekS0rWvV3S\nmtznvzMiftDD+wyXFJJmFBGnDXzDKx2AVYeIGNPxOjVEH46IX3e1v6ThEdHaH7Htp68CNcAxwFbg\naOD1ffkBg+jfYr8MlfOsZr7CsH4h6X9J+pGkGyVtAy6U9GZJSyRtlvScpK9KGpH23+OvWUnfT9v/\nU9I2Sb+XNHNf903bz5T0lKQtkv6fpN+W/gWe8ybghxGxOSLaI+LJiPhp7r2Ok/RrSU3pKuTv0/rR\nKYbnJK2X9C+SRqZtb5e0RtJ/l/Q88M3Sv/glNUj6pKRHU5w3ShqV235l+rz1kv56f//yz1+FSDpK\n0n3pc1+S9MO0233p9+PpyvG8tP9HJK2StFHSrZIOS+s7vpdLJa0C/iDpG5L+qeSzb5f0sd7Gbv3H\nCcP60znAD4FDgB8BrcAngEnAW4AzgL/p5vgPAP8TmAA8C/zjvu4r6TXAzcCn0+euBuZ18z5LgKsk\nXSJpdn6DpEOAXwO/AA4DjgLuTZs/C9QCxwMnpvO7Mnf4VGAMMB24tIvPfh/wDmAWcBJwUfrcdwMf\nA96WPvP0buLvjS8C/wGMT3Fek9a/Nf1+Q+oC/ImkdwKLgPcCU4ANQGnX1llkifc44HrgAklK53Io\ncBpwYx+fgxXACcP6039FxC/SX+qvRMSyiFgaEa0RUQ9cC5zazfG3RERdRLSQNUon9GLfdwMPR8TP\n07Z/BV7q5n0uJUtuHweelPR0aiQhawifjYivRMSuiNgaEQ+kbR8EPh8RjRHxIlmjelHufVvT9uaI\neKWLz/5yRDwfERuB23Ln8D7g2+lq52XgC93E3+Hf0pXcZkmbgVu72bcFmAEcFhE7I+K33ez7QeBb\nEfFwROwErgBOlTQ1t8//johN6Tv/HbCTLEkAXAD8OiK6+w5sgHDCsP60Lr8g6RhJ/9ExoEzWqE7q\n5vjnc693kP2Fvq/7Hp6PI7LZNxu6epOI2BER/ysi5gITgZ8CP0lXF9OAZ7o49HBgbW55Ldlf4B1e\niIjmbuIv+xxKXnfl0ogY1/EDvKebfT8FjADqUpfYxd3su8d5RsRWYBN7nmtpfDcAF6bXFwLfKyN+\nGwCcMKw/lU6N/A3gMeB1EXEwWTeOCo7hObJuFgBS18iUrnd/VURsAa4ia7hnkDWER3ax+wbgiNzy\ndGB9/u3Kjnhve5wDWeLqMxHxXER8OCIOAz4KXJvGgDqLeY/zlDSWrCuru3P9HnCOpBPJ/v1+0Zfx\nW3GcMKySxgJbgJclvZ7uxy/6ym3AXEl/Jmk42RjK5K52lvQ5SbWSRkoaTdY11QQ8DSwGpku6TNIo\nSQdL6hgPuRH4rKRJkiaTjad8v4/O4WbgQ5KOlnRgeu8+I+l9kjqS6GayBr8tItqAjWRjKh1uTLEc\nnwblrwLuj4jurtrWAo+QjWf8OHVl2SDghGGV9CngYmAb2dXGj4r+wIh4AXg/8C9kjd+RwEPArm4O\nuz7tu4Gs7/1PU1fVFrJB6fOAF4CneHUM5gtkjeJjwApgKVlj2hfn8Avg62R3LT0NdIwxdHcO+2I+\nsEzSy2RdcB/NPePyOeCHaSzk3Ij4JVlX4s/Irnymk41r9OR6skFwd0cNInIBJRvKJNWQJYL3RsT9\nlY6nNyQdBzwIjIqI9krHUw5JpwPfBmaFG6FBw1cYNuRIOkPSuNSF8j/J7gp6oIfDBhRJ56RusgnA\nl4CfD6JkMZKsK/CbThaDixOGDUUnA/VAI/AnwDkR0VfdOf3lo2S3A68iu031o5UNpzzpamgT2fMx\nX61wOLaP3CVlZmZl8RWGmZmVpaomH5w0aVLMmDGj0mGYmRUmAl5paWX7rjZe3tXKjuY22iMYM2o4\nMycd1Kv3XL58+UsR0eXt5R2qKmHMmDGDurq6SodhZtZndra08ci6zSxd3cTS1RtZvnYTO1vaGQac\ndOgY5s+cyPxZE5g3cwKvGTu6V58haW3Pe1VZwjAzG+x2trTx4NpNLFndxNL6jTy0bjPNre1IcPSh\nYzn/TdNZMGsCb5oxgYljRvX8hn2o8IQh6QzgK2Q1Bb4VEV8q2X4EcB3Z07ZNwIUR0SDpbWQTw3U4\nBjg/IrqbNM3MbFDZ0dzK8rWbWFqfXUE8sm4LzW3tDBPMOfxgLlpwBPNnZlcQ4w4cWdFYC00Y6aGo\na8iehm0ge3p0cUQ8kdvtauCGiLg+PcxzFXBRRNxDmp0z3Wu+CvhVkfGamRVt+65W6tY0sSQliEcb\nttDaHtQME8cefjCXvGUGC2ZNoHbGBA4ePaLS4e6h6CuMecCqNHU1km4CzgbyCWMO8Mn0+h46n3b5\nvcB/RsSOAmM1M+tzW15poW5NUzYGUb+RxzZspa09GD5MHD/1EP76rbOYPzNLEGNGDexRgqKjm8Ke\nUxs3kM1Tk/cIcC5Zt9U5wFhJE1MNgA7nk839sxdJC4GFANOnT++jsM3MemfzjmYeWJ0liCX1G3ni\nua1EwMiaYbxx2iFcetqRzJ85kblHjOPAkQM7QZQaCNFeDnwtlYe8j2xa5LaOjanc43HAHZ0dHBHX\nkhXeoba21k8hmlm/2rh91x4JYuUL27IEMXwYc6eP4+Onz2b+rAnMnT6e0SNqKh3ufik6Yaxnz7n6\np7LnPPlExAayKwwkjQHOi4jNuV3eB/wsVUczM6uoF7ft3D1AvbS+iadf3A7A6BHDOOmI8XzyuKOY\nP2sib5x2CKOGD+4EUarohLEMmJ2Kr6wn61r6QH4HSZOApjRx2pVkd0zlXcCetZDNzPrN81t2snT1\nxt2D1PWNLwNw0MgaTpoxgfecOIUFsyZw3JRxjBxe3ZNnFJowIqJV0mVk3Uk1wHUR8bikRUBdRCwm\nqy9wlaQg65LaPYmapBlkVyi/KTJOM7MO6ze/wtL6jSyp38jS1U2s3ZjdazN21HBqZ4zn/bXTmD9r\nIscefjDDa6o7QZSqqskHa2trw096m1m5IoJ1Ta+wJHUvLV29kYZNrwBwyAEjeNOMCSyYNYH5Mycy\n5/CDqRlWdAXhypC0PCJqe9pvIAx6m5n1i4hgzcYdLE1XD0vqN/LclqxC7PgDRzBv5gQ+dPJM5s+c\nyDGvHcuwKk0QveWEYWZVKyJ4pnF7Gn/InoN4cVtW+mTSmJG752GaP3Mis18zxgmiB04YZlY12tuD\np1/cvvsOpqWrN/LS9mYAXjN2FPNnTWT+zAksmDWRIycfhOQEsS+cMMxs0GpvD558fuvu5PDA6iY2\n7cjuwD/8kNGcMnsy82dOYP6sicyYeKATxH5ywjCzQaOtPXhiw9Z0m2uWILbubAVg6vgDOP2YQ1kw\nK7uCmDr+ACeIPuaEYWYDVktbO4+t37J7/KFuzSa27coSxIyJB3LmsYdlYxCzJjJl3AEVjrb6OWGY\n2YDR3NrOo+s37x6krlvTxI7mbKagWZMP4t1vPHz3ba6vPaR3xYKs95wwzKxidrW28fCze1eTAzjq\n0DGcN3fqfleTs77jhGFm/WZnSxsPPvtqsaCHnt3MrgFSTc565oRhZoXpqZrchQOompz1zAnDzPpM\nRzW5jkHqFYOompz1zAnDzHqtmqrJWc/8DZpZ2fLV5Jau3sgTG7bSnqsm97enHsmCWYOzmpz1zN+o\nmXWpp2pyH6uianLWMycMM9utcduuPeZheuqFoVNNznrmhGE2hL2wdSdL6ruuJnf2CUOnmpz1rPCE\nIekM4CtkFfe+FRFfKtl+BFlZ1slAE3BhRDSkbdOBb5FV3QvgXRGxpuiYzapVRzW5jiuINa4mZ/ug\n0IQhqQa4BngH0AAsk7Q4Ip7I7XY1cENEXC/pdOAq4KK07QbgixFxp6QxQHuR8ZpVk4igYdMr/L6+\n62py2XMQ1V1NzvpO0VcY84BVEVEPIOkm4GwgnzDmAJ9Mr+8Bbk37zgGGR8SdABGxveBYzQa10mpy\nS+s3ssHV5KwPFZ0wpgDrcssNwPySfR4BziXrtjoHGCtpInAUsFnST4GZwK+BKyKiLX+wpIXAQoDp\n06cXcQ5mA1JWTe5lluQSRGk1uY+4mpz1oYEw6H058DVJlwD3AeuBNrLYTgFOBJ4FfgRcAnw7f3BE\nXAtcC1BbWxv9FbRZf9u7mlwTL23PEoSryVl/KDphrCcbsO4wNa3bLSI2kF1hkMYpzouIzZIagIdz\n3Vm3AgsoSRhm1aq9PfjD89v2KBa0ZzW5Sa4mZ/2q6ISxDJgtaSZZojgf+EB+B0mTgKaIaAeuJLtj\nquPYcZImR0QjcDpQV3C8ZhWzZzW5JpataWLLK1mCcDU5GwgKTRgR0SrpMuAOsttqr4uIxyUtAuoi\nYjFwGnCVpCDrkvpoOrZN0uXAXcr+z1gOfLPIeM36U2tbO49t2JqNQXRSTe6MN7zW1eRsQFFE9XT7\n19bWRl2dL0JsYCqtJrd8TRMv56rJzZ850dXkrCIkLY+I2p72GwiD3mZVaVdrG4+s28LS+o0sWb2R\nB9du5pWWLEEcdegYznU1ORtknDDM+khP1eTe/6Zpu4sFuZqcDUZOGGa9tKO5lQfXbt59F5OryVm1\nc8IwK5OrydlQ54Rh1oWtO1M1ufomlqxu4rH1W1xNzoY0/1dulrianFn3/F+9DVlNLzfzQHpIztXk\nzHrmhGFDhqvJme0fJwyrWh3V5DoGqZ9xNTmz/eKEYVWjp2pyf147jQWuJmfWa04YNih1VJPbfQWx\neiPrmlxNzqxIThg2KEQEazfu2KOLqbSa3F+9xdXkzIrkhGEDUkc1ufwg9QtbXU3OrJKcMGxAcDU5\ns4HPCcMqIl9Nbml9Ew+saaLp5WbA1eTMBionDOsXPVWTe9vRr2H+rAm82dXkzAaswhOGpDOAr5BV\n3PtWRHypZPsRZGVZJwNNwIUR0ZC2tQGPpl2fjYizio7X+kZHNbmlaZB62Zomtu10NTmzwazQhCGp\nBrgGeAfQACyTtDginsjtdjVwQ0RcL+l04CrgorTtlYg4ocgYrW+0tLWzomHL7ruYSqvJvfv4w11N\nzmyQK/oKYx6wKiLqASTdBJwN5BPGHOCT6fU9wK0Fx2R9IF9NbunqJpav3eRqcmZVruiEMQVYl1tu\nAOaX7PMIcC5Zt9U5wFhJEyNiIzBaUh3QCnwpIvZKJpIWAgsBpk+f3vdnYICryZnZwBj0vhz4mqRL\ngPuA9UBb2nZERKyXNAu4W9KjEfFM/uCIuBa4FqC2tjb6L+zqlq8mt7S+iYfXbXY1ObMhruiEsR6Y\nlluemtbtFhEbyK4wkDQGOC8iNqdt69Pvekn3AicCeyQM6xvbd7WyfO2mbAzC1eTMrBNFJ4xlwGxJ\nM8kSxfnAB/I7SJoENEVEO3Al2R1TSBoP7IiIXWmftwD/p+B4hwxXkzOzfVVoSxARrZIuA+4gu632\nuoh4XNIioC4iFgOnAVdJCrIuqY+mw18PfENSOzCMbAzjib0+xMqyZUcLD6xpSncxuZqcme07RVRP\nt39tbW3U1dVVOowBIV9NbunqJv7w/NY9qsnNnznR1eTMDABJyyOitqf9/KdklWjctivVo97IkvpO\nqsm93dXkzGz/OGEMUq4mZ2b9zQljkNiw+ZXs6uEZV5Mzs8pwwhiAXE3OzAYiJ4wBoKOaXMdDcktc\nTc7MBiAnjAp5pnF7ekjO1eTMbHBwwqiAHyxdy2d+9hjganJmNng4YVTArQ+t56hDx/CNi2pdTc7M\nBg3fTtPPtu1s4aFnN/P21x/KzEm+mjCzwcMJo58tqW+itT04ZfbkSodiZrZPnDD62f1PN3LgyBrm\nHjGu0qGYme0TJ4x+dv/TL7Fg1kRPz2Fmg44TRj9a17SD1S+9zCmzJ1U6FDOzfeaE0Y/uf/olAI9f\nmNmg5ITRj+5/upHDDxnNkZMPqnQoZmb7rPCEIekMSSslrZJ0RSfbj5B0l6QVku6VNLVk+8GSGiR9\nrehYi9Ta1s5vV73EKbMn+1ZaMxuUCk0YkmqAa4AzgTnABZLmlOx2NXBDRBwPLAKuKtn+j2SV+Aa1\nFeu3sHVnK6cc5fELMxucir7CmAesioj6iGgGbgLOLtlnDnB3en1Pfrukk4BDgV8VHGfh7l3ZiARv\nOdIJw8wGp6ITxhRgXW65Ia3LewQ4N70+BxgraaKkYcD/BS4vOMbC7Wxp44dLn+Xk101i/EEjKx2O\nmVmvlJUwJC2QNDa3fLCk+X0Uw+XAqZIeAk4F1gNtwKXA7RHR0ENsCyXVSaprbGzso5D61o+XN/DS\n9l1cetrrKh2KmVmvlTv54NeBubnl7Z2s68x6YFpueWpat1tEbCBdYUgaA5wXEZslvRk4RdKlwBhg\npKTtEXFFyfHXAtcC1NbWRpnn029a2tr593ufYe70cSyYNaHS4ZiZ9Vq5CUMRsbsxjoh2SeUcuwyY\nLWkmWaI4H/jAHm8sTQKaIqIduBK4Ln3GB3P7XALUliaLwWDxwxtYv/kVFp39Bt8dZWaDWrljGPWS\nPi5pRPr5BFDf00ER0QpcBtwBPAncHBGPS1ok6ay022nASklPkQ1wf3Gfz2KAam8P/u3eVRzz2rGc\nfsxrKh2Omdl+KfcK4yPAV4H/AQRwF7CwnAMj4nbg9pJ1n829vgW4pYf3+C7w3TJjHTBuf+w5nml8\nma9ecKKvLsxs0CsrYUTEi2TdSVaGbTtb+Mqvn+a7v1vDkZMP4k+PO6zSIZmZ7beyEoak75BdWewh\nIv6qzyMahHY0t/KH57ftnlzwB0uf5aXtu3h/7TQ+/SdHU+Oa3GZWBcrtkrot93o02fMSG/o+nMFn\n+dom/vb7D/Litl27182dPo5v/kUtJ0xzzQszqx7ldkn9JL8s6UbgvwqJaABZ9eI2Ft32JAtPmcXJ\nJVOSRwTfX7KWRbc9weHjDuDfLzyJIycfxNTxB3LASNe6MLPqU+4VRqnZQFXf9nPbig38wy0reLm5\njebWtr0SxqLbnuA7v13D246ezJfPP5FDDhhRoUjNzPpHuWMY28jGMJR+Pw/8Q4FxVdRVtz/JN+6r\nZ+70cRw75RBu+P1a1jXtYNqEAwGob9zOd367hgvmTeeL7zmWYR6jMLMhoKznMCJibEQcnPt9VGk3\nVbV4cdtOvnFfPWefcDg3LXwzC986C4BbH3r1AfUbfr+WETXik+84ysnCzIaMsicflDRe0jxJb+34\nKTKwStnZ3A7AW2dPZuTwYUwdfyALZk3gpw+tJyLYurOFH9et48+OP5zJY0dVOFozs/5T7uSDHyar\nSXEH8IX0+/PFhVU5zW1tAIwY/uo/zXlzp7L6pZd58NlN3FLXwMvNbVz8RzMqFKGZWWWUe4XxCeBN\nwNqIeBtwIrC5sKgqqLk1e9xkZM2rXU1nHncYB4yo4cd1DVz/+zXMnT6ON/qWWTMbYsq9S2pnROyU\nhKRREfEHSUcXGlmFtLRlXVIjal7NpWNGDefMY1/LzXXraA/41Dur8tTNzLpV7hVGg6RxwK3AnZJ+\nDqwtLqzK6UgYI4fv+U9z7typtAccevAozjz2tZUIzcysosp9cO+c9PLzku4BDgF+2bFd0viI2FRA\nfP2uuXXvKwyANx85kfkzJ/Bnbzx8r21mZkPBPj+4FxG/6WT1XfRcTGlQaO6kSwqgZpj40d+8uRIh\nmZkNCH31p3LVPIzQ0pYNeo8a7qsIM7O8vmoVB1xp1N7qqkvKzGyoK7xVlHSGpJWSVknaq8SqpCMk\n3SVphaR7JU3NrX9Q0sOSHpf0kaJjhfxdUlVz0WRm1icK7ZKSVANcA5wJzAEukDSnZLergRsi4nhg\nEXBVWv8c8OaIOAGYD1wh6fA+irdLXY1hmJkNdeU+6T2hk5/89Kx/3MWh84BVEVEfEc3ATcDZJfvM\nAe5Or+/p2B4RzRHRUWRiVLmx7q+OKwyPYZiZ7ancVvFBoBF4Cng6vV6TuoxOioimLo6bAqzLLTek\ndXmPAOem1+cAYyVNBJA0TdKK9B7/FBF7FW2StFBSnaS6xsbGMk+nax7DMDPrXLmt4p3AuyJiUkRM\nJOtiug24FPi3/YzhcuBUSQ8BpwLrgTaAiFiXuqpeB1ws6dDSgyPi2oiojYjayZMn72couTEMX2GY\nme2h3FZxQUTc0bEQEb8iG19YQtZd1JX1wLTc8tS0breI2BAR50bEicBn0rrNpfsAjwGnlBlvr3Xc\nVutBbzOzPZWbMJ6T9A/pzqUjJP098EIa1G7v5rhlwGxJMyWNBM4HFud3kDRJUkccVwLXpfVTJR2Q\nXo8HTgZWln1mvdTRJTXSXVJmZnsot1X8ANnVwa3pZ3paVwO8r6uDIqIVuIxsOvQngZsj4nFJiySd\nlXY7DVgp6SngUOCLaf3rgaWSHgF+A1wdEY/uw7n1SnNbOyNqhOQrDDOzvHLnknoJ+FgXm1f1cOzt\nwO0l6z6be30LcEsnx90JHF9OfH2ppbXdA95mZp0ot6b3UWSD0zPyx0TE6cWEVTktbe17zVRrZmbl\nTz74Y+DfgW+R7mCqVs1t4SsMM7NOlJswWiPi64VGMkA0t7Z7wNvMrBPltoy/kHSppMPyT3sXGlmF\ntKRBbzMz21O5VxgXp9+fzq0LYFbfhlN5HsMwM+tcuXdJzSw6kIGi2XdJmZl1qtuEIen0iLhb0rmd\nbY+InxYTVuVkz2E4YZiZlerpCuNUsplk/6yTbQFUXcJoafOgt5lZZ7pNGBHxufT7L/snnMpraQsO\nGFFT6TDMzAacch/cGwWcx94P7i0qJqzKaW5t5+DR5d4LYGY2dJTbMv4c2AIsB3b1sO+g1uIxDDOz\nTpWbMKZGxBmFRjJANPu2WjOzTpXbMv5O0nGFRjJAeNDbzKxz5V5hnAxcImk1WZeUgEjV8KqKn8Mw\nM+tcuQnjzEKjGEBa2oIRwz01iJlZqZ4e3Ds4IrYC2/opnopraW1nZI1vqzUzK9VT38sP0+/lQF36\nvTy33CNJZ0haKWmVpCs62X6EpLskrZB0r6Spaf0Jkn4v6fG07f1ln9V+aG5r9xWGmVknenpw793p\nd6/mkko1v68B3gE0AMskLY6IJ3K7XQ3cEBHXSzoduAq4CNgB/EVEPC3pcGC5pDsiYnNvYilHRGR3\nSXkMw8xsL2U/oSZpPDAbGN2xLiLu6+GwecCqiKhP73ETcDaQTxhzgE+m1/eQ1QwnIp7Kfc4GSS8C\nk4HCEkZbexCBB73NzDpRVsso6cPAfcAdwBfS78+XcegUYF1uuSGty3sE6Jjc8BxgrKSJJZ8/DxgJ\nPNNJbAsl1Umqa2xsLCOkrrW0BYCfwzAz60S5LeMngDcBayPibcCJ9N1f+pcDp0p6iGyyw/XkysBK\nOgz4HvCXEdFeenBEXBsRtRFRO3ny5P0KpLk1e3tfYZiZ7a3cLqmdEbFTEpJGRcQfJB1dxnHrgWm5\n5alp3W4RsYF0hSFpDHBexziFpIOB/wA+ExFLyoy115rbsoQx0hX3zMz2Um7CaJA0jmx84U5Jm4C1\nZRy3DJgtaSZZojgf+EB+B0mTgKZ09XAlcF1aPxL4GdmA+C1lxrlfWjoShrukzMz2Um7FvXPSy89L\nugc4BPhlGce1SrqMbMyjBrguIh6XtAioi4jFwGnAVZKCbJzko+nw9wFvBSZKuiStuyQiHi7rzHqh\nI2G4S8rMbG89Jox0a+zjEXEMQET8Zl8+ICJuB24vWffZ3OtbgL2uICLi+8D39+Wz9pfHMMzMutZj\nyxgRbcBKSdP7IZ6KavYVhplZl8odwxgPPC7pAeDljpURcVYhUVVIx221ozyGYWa2l3ITxmjg3bll\nAf/U9+FUlscwzMy6Vm7CGF46diHpgALiqahXxzB8W62ZWameZqv9W+BSYJakFblNY4HfFhlYJewe\nw3CXlJnZXnq6wvgh8J9kEwLmZ5rdFhFNhUVVIS2tHQ/uOWGYmZXqabbaLcAW4IL+CaeyPJeUmVnX\n3DLmNLdlU1h50NvMbG9uGXNaWrMrDA96m5ntzQkjp9lzSZmZdcktY06zB73NzLrkljHHD+6ZmXXN\nLWOOpzc3M+uaW8ac5nRb7fBhHvQ2MyvlhJHT3NrOyJphSE4YZmalnDByWtrafUutmVkXCk8Yks6Q\ntFLSKklXdLL9CEl3SVoh6V5JU3Pbfilps6Tbio4TsoTh8Qszs84V2jqman3XAGcCc4ALJM0p2e1q\nsrrdxwOLyOat6vDPwEVFxpiXXWE4YZiZdabo1nEesCoi6iOiGbgJOLtknznA3en1PfntEXEXsK3g\nGHfb1eqEYWbWlaJbxynAutxyQ1qX9whwbnp9DjBW0sRyP0DSQkl1kuoaGxv3K9iWtnCXlJlZFwZC\n63g5cKqkh4BTgfVAW7kHR8S1EVEbEbWTJ0/er0Ba0l1SZma2t3Ir7vXWemBabnlqWrdbRGwgXWFI\nGgOcFxGbC46rUy1t7YwY7rukzMw6U/Sf08uA2ZJmShoJnA8szu8gaZKkjjiuBK4rOKYuNXvQ28ys\nS4W2jhHRClwG3AE8CdwcEY9LWiTprLTbacBKSU8BhwJf7Dhe0v3Aj4E/ltQg6U+KjLfZXVJmZl0q\nukuKiLgduL1k3Wdzr28Bbuni2FOKjW5PLW3tHDSq8H8SM7NByX9O57hLysysa24dc1paw1ODmJl1\nwQkjJ5sapKbSYZiZDUhOGDnNnnzQzKxLThg5vkvKzKxrbh1zPPmgmVnX3DrmeC4pM7OuuXXM8W21\nZmZdc+sSdoFzAAALqklEQVSYREQaw/Cgt5lZZ5wwktb2AHCXlJlZF9w6Ji1t7QDukjIz64Jbx6Sl\nNbvCcMIwM+ucW8dkV1tWs2mEu6TMzDrl1jFpacuuMEb5CsPMrFNuHZOW1jSG4Yp7ZmadKjxhSDpD\n0kpJqyRd0cn2IyTdJWmFpHslTc1tu1jS0+nn4iLjbPagt5lZtwptHSXVANcAZwJzgAskzSnZ7Wrg\nhog4HlgEXJWOnQB8DpgPzAM+J2l8UbE2tzphmJl1p+jWcR6wKiLqI6IZuAk4u2SfOcDd6fU9ue1/\nAtwZEU0RsQm4EzijqEA7bqv1cxhmZp0runWcAqzLLTekdXmPAOem1+cAYyVNLPPYPtMx6O3Zas3M\nOjcQWsfLgVMlPQScCqwH2so9WNJCSXWS6hobG3sdhLukzMy6V3TruB6YlluemtbtFhEbIuLciDgR\n+Exat7mcY9O+10ZEbUTUTp48udeBvvqkt++SMjPrTNEJYxkwW9JMSSOB84HF+R0kTZLUEceVwHXp\n9R3AOyWNT4Pd70zrCtHsMQwzs24V2jpGRCtwGVlD/yRwc0Q8LmmRpLPSbqcBKyU9BRwKfDEd2wT8\nI1nSWQYsSusKsXvQ211SZmadGl70B0TE7cDtJes+m3t9C3BLF8dex6tXHIXyGIaZWffcOia+rdbM\nrHtuHZPmNs9Wa2bWHbeOScdcUh7DMDPrnFvHZPdcUp580MysU04Yia8wzMy659YxaWlrR4KaYb7C\nMDPrjBNGsqutnRE1w5CcMMzMOuOEkbS0hrujzMy64RYyaWlr9zMYZmbdcAuZtLS1e+JBM7NuOGEk\nza3tfmjPzKwbbiGTZndJmZl1yy1k0tLW7kFvM7NuuIVMWtrCXVJmZt1wC5lkYxge9DYz64oTRuIx\nDDOz7hXeQko6Q9JKSaskXdHJ9umS7pH0kKQVkt6V1o+U9B1Jj0p6RNJpRcbZ0ua7pMzMulNoCymp\nBrgGOBOYA1wgaU7Jbv+DrHTriWQ1v/8trf9rgIg4DngH8H9ztb/7nAe9zcy6V3QLOQ9YFRH1EdEM\n3AScXbJPAAen14cAG9LrOcDdABHxIrAZqC0qUD+HYWbWvaJbyCnAutxyQ1qX93ngQkkNZLW/P5bW\nPwKcJWm4pJnAScC00g+QtFBSnaS6xsbGXgfa0hYewzAz68ZAaCEvAL4bEVOBdwHfS11P15ElmDrg\ny8DvgLbSgyPi2oiojYjayZMn9zoIX2GYmXVveMHvv549rwqmpnV5HwLOAIiI30saDUxK3VB/17GT\npN8BTxUVaHaXlG+rNTPrStF/Ui8DZkuaKWkk2aD24pJ9ngX+GEDS64HRQKOkAyUdlNa/A2iNiCeK\nCtSD3mZm3Sv0CiMiWiVdBtwB1ADXRcTjkhYBdRGxGPgU8E1Jf0c2AH5JRISk1wB3SGonuyq5qMhY\nW9wlZWbWraK7pIiI28kGs/PrPpt7/QTwlk6OWwMcXXR8HVraghEe9DYz65JbSCAiaPaDe2Zm3XIL\nSXZ1ATDKVxhmZl1yC0k24A148kEzs244YZBPGP7nMDPriltIsof2wAnDzKw7biHJHtoDPDWImVk3\n3ELy6qC3H9wzM+uaW0g8hmFmVg5FRKVj6DOSGoG1+3jYJOClAsIZyHzOQ8dQPO+heM6wf+d9RET0\nOHtrVSWM3pBUFxGF1dkYiHzOQ8dQPO+heM7QP+ftPhgzMyuLE4aZmZXFCQOurXQAFeBzHjqG4nkP\nxXOGfjjvIT+GYWZm5fEVhpmZlcUJw8zMyjJkE4akMyStlLRK0hWVjqcIkqZJukfSE5Iel/SJtH6C\npDslPZ1+j690rEWQVCPpIUm3peWZkpam7/xHqWxw1ZA0TtItkv4g6UlJbx4K37Wkv0v/fT8m6UZJ\no6vxu5Z0naQXJT2WW9fp96vMV9P5r5A0ty9iGJIJQ1INcA1wJjAHuEDSnMpGVYhW4FMRMQdYAHw0\nnecVwF0RMRu4Ky1Xo08AT+aW/wn414h4HbAJ+FBFoirOV4BfRsQxwBvJzr2qv2tJU4CPA7URcSxZ\nKejzqc7v+rvAGSXruvp+zwRmp5+FwNf7IoAhmTCAecCqiKiPiGbgJuDsCsfU5yLiuYh4ML3eRtaA\nTCE71+vTbtcD76lMhMWRNBX4U+BbaVnA6cAtaZeqOm9JhwBvBb4NEBHNEbGZIfBdk5WaPkDScOBA\n4Dmq8LuOiPuAppLVXX2/ZwM3RGYJME7SYfsbw1BNGFOAdbnlhrSuakmaAZwILAUOjYjn0qbngUMr\nFFaRvgz8PdCelicCmyOiNS1X23c+E2gEvpO64b4l6SCq/LuOiPXA1cCzZIliC7Cc6v6u87r6fgtp\n44ZqwhhSJI0BfgL8t4jYmt8W2X3VVXVvtaR3Ay9GxPJKx9KPhgNzga9HxInAy5R0P1Xpdz2e7K/p\nmcDhwEHs3W0zJPTH9ztUE8Z6YFpueWpaV3UkjSBLFj+IiJ+m1S90XJ6m3y9WKr6CvAU4S9Iasu7G\n08n698elbguovu+8AWiIiKVp+RayBFLt3/XbgdUR0RgRLcBPyb7/av6u87r6fgtp44ZqwlgGzE53\nUowkGyRbXOGY+lzqt/828GRE/Etu02Lg4vT6YuDn/R1bkSLiyoiYGhEzyL7buyPig8A9wHvTblV1\n3hHxPLBO0tFp1R8DT1Dl3zVZV9QCSQem/947zrtqv+sSXX2/i4G/SHdLLQC25Lquem3IPukt6V1k\n/dw1wHUR8cUKh9TnJJ0M3A88yqt9+f+dbBzjZmA62XTw74uI0sG0qiDpNODyiHi3pFlkVxwTgIeA\nCyNiVyXj60uSTiAb5B8J1AN/SfZHYVV/15K+ALyf7K7Ah4APk/XXV9V3LelG4DSyacxfAD4H3Eon\n329Knl8j657bAfxlRNTtdwxDNWGYmdm+GapdUmZmto+cMMzMrCxOGGZmVhYnDDMzK4sThpmZlcUJ\nw8zMyuKEYdYNSfdKqu3Hz/vnNFX3P3ex/T1VOrOyDQLDe97FzHpD0vDcBHjlWghMiIi2Lra/B7iN\n7Gnmvvg8s7L5CsOqgqQZqWjQN9Nf6L+SdED+CkHSpDS/FJIukXRrKjqzRtJlkj6ZZnpdImlC7u0v\nkvRwKtAzLx1/UCpo80A65uzc+y6WdDdZfYLOYlW6knhM0qOS3p/WLwbGAMs71pUc90fAWcA/p3iO\nTOf3ZUl1wCckTZb0E0nL0s9beoj3DWndw6nQzuy++D6sOvkKw6rJbOCCiPhrSTcD5/Ww/7FkU76P\nBlYB/xARJ0r6V+AvyKaOATgwIk6Q9FbgunTcZ8jmqPorSeOAByT9Ou0/Fzi+myk4zgVOICtyNAlY\nJum+iDhL0vaIOKGzgyLidymp3BYRtwBkM0AwMiI6kuIPyQoH/Zek6cAdwOu7ifcjwFci4gdpXrWa\nHv7NbAhzwrBqsjoiHk6vlwMzetj/nlRYapukLcAv0vpHgeNz+90IWQEbSQenBvedZDPiXp72GU02\nnw/AnT3M13QycGPqdnpB0m+AN9H7CTB/lHv9dmBOSiQAB6fp7buK9/fAZ5QVnPppRDzdyxhsCHDC\nsGqSn1yuDTiAbEK6jq7X0d3s355bbmfP/zdKJ1wLQMB5EbEyv0HSfLJaFP0p/3nDgAURsTO/Q5qM\nbq94gSclLSWrTni7pL+JiLuLDdcGK49hWLVbA5yUXr+3m/260zHGcDLZNNFbyLp6PpYaYiSduA/v\ndz/wfkk1kiaTlVZ9oMxjtwFju9n+K+BjHQtpBlu6ijfN4FsfEV8lmxr7eMy64IRh1e5q4G8lPUQ2\nXtAbO9Px/w58KK37R2AEsELS42m5XD8DVgCPAHcDf5/qWZTjJuDTaeD6yE62fxyoTQPYT5CNUXQX\n7/uAxyQ9TDY2c8M+nIcNMZ7e3MzMyuIrDDMzK4sHvc0KIuk44Hslq3dFxPwyjv0M8Oclq39cjZUh\nbfBwl5SZmZXFXVJmZlYWJwwzMyuLE4aZmZXFCcPMzMry/wErRpo7UZYYqAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1090aef98>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for m in models:\n",
    "    print(\"Model '{}' AUC : {}\".format(m, models[m].auc()))\n",
    "    models[m].plot(metric='auc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Misc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O cluster uptime:</td>\n",
       "<td>57 mins 50 secs</td></tr>\n",
       "<tr><td>H2O cluster version:</td>\n",
       "<td>3.10.4.8</td></tr>\n",
       "<tr><td>H2O cluster version age:</td>\n",
       "<td>5 days </td></tr>\n",
       "<tr><td>H2O cluster name:</td>\n",
       "<td>H2O_from_python_pierre_2e44fe</td></tr>\n",
       "<tr><td>H2O cluster total nodes:</td>\n",
       "<td>1</td></tr>\n",
       "<tr><td>H2O cluster free memory:</td>\n",
       "<td>27.69 Gb</td></tr>\n",
       "<tr><td>H2O cluster total cores:</td>\n",
       "<td>4</td></tr>\n",
       "<tr><td>H2O cluster allowed cores:</td>\n",
       "<td>4</td></tr>\n",
       "<tr><td>H2O cluster status:</td>\n",
       "<td>locked, healthy</td></tr>\n",
       "<tr><td>H2O connection url:</td>\n",
       "<td>http://localhost:54321</td></tr>\n",
       "<tr><td>H2O connection proxy:</td>\n",
       "<td>None</td></tr>\n",
       "<tr><td>H2O internal security:</td>\n",
       "<td>False</td></tr>\n",
       "<tr><td>Python version:</td>\n",
       "<td>3.6.0 final</td></tr></table></div>"
      ],
      "text/plain": [
       "--------------------------  -----------------------------\n",
       "H2O cluster uptime:         57 mins 50 secs\n",
       "H2O cluster version:        3.10.4.8\n",
       "H2O cluster version age:    5 days\n",
       "H2O cluster name:           H2O_from_python_pierre_2e44fe\n",
       "H2O cluster total nodes:    1\n",
       "H2O cluster free memory:    27.69 Gb\n",
       "H2O cluster total cores:    4\n",
       "H2O cluster allowed cores:  4\n",
       "H2O cluster status:         locked, healthy\n",
       "H2O connection url:         http://localhost:54321\n",
       "H2O connection proxy:\n",
       "H2O internal security:      False\n",
       "Python version:             3.6.0 final\n",
       "--------------------------  -----------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "h2o.cluster().show_status()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'stopbefore' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-39-5cd44630cf3f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mstopbefore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'stopbefore' is not defined"
     ]
    }
   ],
   "source": [
    "stopbefore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "h2o.cluster().shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}