Feature Selection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy.stats as stats\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import sklearn.metrics as metrics\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('online_shoppers_intention.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Administrative</th>\n",
       "      <th>Administrative_Duration</th>\n",
       "      <th>Informational</th>\n",
       "      <th>Informational_Duration</th>\n",
       "      <th>ProductRelated</th>\n",
       "      <th>ProductRelated_Duration</th>\n",
       "      <th>BounceRates</th>\n",
       "      <th>ExitRates</th>\n",
       "      <th>PageValues</th>\n",
       "      <th>SpecialDay</th>\n",
       "      <th>Month</th>\n",
       "      <th>OperatingSystems</th>\n",
       "      <th>Browser</th>\n",
       "      <th>Region</th>\n",
       "      <th>TrafficType</th>\n",
       "      <th>VisitorType</th>\n",
       "      <th>Weekend</th>\n",
       "      <th>Revenue</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Returning_Visitor</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>64.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>Returning_Visitor</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>Returning_Visitor</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2.666667</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>Returning_Visitor</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10</td>\n",
       "      <td>627.500000</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>Returning_Visitor</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Administrative  Administrative_Duration  Informational  \\\n",
       "0               0                      0.0              0   \n",
       "1               0                      0.0              0   \n",
       "2               0                      0.0              0   \n",
       "3               0                      0.0              0   \n",
       "4               0                      0.0              0   \n",
       "\n",
       "   Informational_Duration  ProductRelated  ProductRelated_Duration  \\\n",
       "0                     0.0               1                 0.000000   \n",
       "1                     0.0               2                64.000000   \n",
       "2                     0.0               1                 0.000000   \n",
       "3                     0.0               2                 2.666667   \n",
       "4                     0.0              10               627.500000   \n",
       "\n",
       "   BounceRates  ExitRates  PageValues  SpecialDay Month  OperatingSystems  \\\n",
       "0         0.20       0.20         0.0         0.0   Feb                 1   \n",
       "1         0.00       0.10         0.0         0.0   Feb                 2   \n",
       "2         0.20       0.20         0.0         0.0   Feb                 4   \n",
       "3         0.05       0.14         0.0         0.0   Feb                 3   \n",
       "4         0.02       0.05         0.0         0.0   Feb                 3   \n",
       "\n",
       "   Browser  Region  TrafficType        VisitorType  Weekend  Revenue  \n",
       "0        1       1            1  Returning_Visitor    False    False  \n",
       "1        2       1            2  Returning_Visitor    False    False  \n",
       "2        1       9            3  Returning_Visitor    False    False  \n",
       "3        2       2            4  Returning_Visitor    False    False  \n",
       "4        3       1            4  Returning_Visitor     True    False  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data1 = data.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Encoding the Categorical Columns to Numerical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "LB = LabelEncoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data1['Month'] = LB.fit_transform(data['Month'])\n",
    "data1['Weekend'] = LB.fit_transform(data['Weekend'])\n",
    "data1['VisitorType'] = LB.fit_transform(data['VisitorType'])\n",
    "data1['Revenue'] = LB.fit_transform(data['Revenue'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Administrative</th>\n",
       "      <th>Administrative_Duration</th>\n",
       "      <th>Informational</th>\n",
       "      <th>Informational_Duration</th>\n",
       "      <th>ProductRelated</th>\n",
       "      <th>ProductRelated_Duration</th>\n",
       "      <th>BounceRates</th>\n",
       "      <th>ExitRates</th>\n",
       "      <th>PageValues</th>\n",
       "      <th>SpecialDay</th>\n",
       "      <th>Month</th>\n",
       "      <th>OperatingSystems</th>\n",
       "      <th>Browser</th>\n",
       "      <th>Region</th>\n",
       "      <th>TrafficType</th>\n",
       "      <th>VisitorType</th>\n",
       "      <th>Weekend</th>\n",
       "      <th>Revenue</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>64.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2.666667</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10</td>\n",
       "      <td>627.500000</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Administrative  Administrative_Duration  Informational  \\\n",
       "0               0                      0.0              0   \n",
       "1               0                      0.0              0   \n",
       "2               0                      0.0              0   \n",
       "3               0                      0.0              0   \n",
       "4               0                      0.0              0   \n",
       "\n",
       "   Informational_Duration  ProductRelated  ProductRelated_Duration  \\\n",
       "0                     0.0               1                 0.000000   \n",
       "1                     0.0               2                64.000000   \n",
       "2                     0.0               1                 0.000000   \n",
       "3                     0.0               2                 2.666667   \n",
       "4                     0.0              10               627.500000   \n",
       "\n",
       "   BounceRates  ExitRates  PageValues  SpecialDay  Month  OperatingSystems  \\\n",
       "0         0.20       0.20         0.0         0.0      2                 1   \n",
       "1         0.00       0.10         0.0         0.0      2                 2   \n",
       "2         0.20       0.20         0.0         0.0      2                 4   \n",
       "3         0.05       0.14         0.0         0.0      2                 3   \n",
       "4         0.02       0.05         0.0         0.0      2                 3   \n",
       "\n",
       "   Browser  Region  TrafficType  VisitorType  Weekend  Revenue  \n",
       "0        1       1            1            2        0        0  \n",
       "1        2       1            2            2        0        0  \n",
       "2        1       9            3            2        0        0  \n",
       "3        2       2            4            2        0        0  \n",
       "4        3       1            4            2        1        0  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = data1.drop('Revenue',axis=1)\n",
    "Y = data1.Revenue"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Univariate Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    The scikit-learn library provides the SelectKBest class that can be used with a suite of different statistical tests to select a specific number of features.\n",
    "    We have used Chi square test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = SelectKBest(score_func=chi2)\n",
    "fit = test.fit(X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Features</th>\n",
       "      <th>Scores</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ProductRelated_Duration</td>\n",
       "      <td>877404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>PageValues</td>\n",
       "      <td>175127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Administrative_Duration</td>\n",
       "      <td>41754.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Informational_Duration</td>\n",
       "      <td>35059.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ProductRelated</td>\n",
       "      <td>19317.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Administrative</td>\n",
       "      <td>1133.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Informational</td>\n",
       "      <td>357.982</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Month</td>\n",
       "      <td>86.1637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>SpecialDay</td>\n",
       "      <td>53.7971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>VisitorType</td>\n",
       "      <td>37.5475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>BounceRates</td>\n",
       "      <td>29.6543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ExitRates</td>\n",
       "      <td>28.9851</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Browser</td>\n",
       "      <td>8.87329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Weekend</td>\n",
       "      <td>8.12046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Region</td>\n",
       "      <td>3.03757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>TrafficType</td>\n",
       "      <td>1.28319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>OperatingSystems</td>\n",
       "      <td>1.03713</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Features   Scores\n",
       "5   ProductRelated_Duration   877404\n",
       "8                PageValues   175127\n",
       "1   Administrative_Duration  41754.8\n",
       "3    Informational_Duration  35059.8\n",
       "4            ProductRelated  19317.3\n",
       "0            Administrative  1133.97\n",
       "2             Informational  357.982\n",
       "10                    Month  86.1637\n",
       "9                SpecialDay  53.7971\n",
       "15              VisitorType  37.5475\n",
       "6               BounceRates  29.6543\n",
       "7                 ExitRates  28.9851\n",
       "12                  Browser  8.87329\n",
       "16                  Weekend  8.12046\n",
       "13                   Region  3.03757\n",
       "14              TrafficType  1.28319\n",
       "11         OperatingSystems  1.03713"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ff = pd.DataFrame([data1.columns[:-1],fit.scores_],index=['Features','Scores']).T\n",
    "ff.sort_values('Scores',ascending = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "LR = LogisticRegression()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  K Fold Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV,KFold\n",
    "kf = KFold(n_splits=5,shuffle=True,random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = data1[['ProductRelated_Duration','PageValues','Administrative_Duration','Informational_Duration',\n",
    "          'ProductRelated','Administrative','Informational','Month','SpecialDay','VisitorType',\n",
    "          'BounceRates', 'ExitRates']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = data1.Revenue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Accuracy: 0.88  F1: 0.50 ROC-AOC: 0.68\n"
     ]
    }
   ],
   "source": [
    "F1=[]\n",
    "roc_auc = []\n",
    "accuracy=[]\n",
    "for train,test in kf.split(X,Y):\n",
    "    xtrain,xtest=X.iloc[train,:],X.iloc[test,:]\n",
    "    ytrain,ytest=Y.iloc[train],Y.iloc[test]\n",
    "    LR.fit(xtrain,ytrain)\n",
    "    y_predict = LR.predict(xtest)\n",
    "    F1.append(metrics.f1_score(ytest,y_predict))\n",
    "    fpr,tpr,_=metrics.roc_curve(ytest,y_predict)\n",
    "    roc_auc.append(metrics.auc(fpr,tpr))\n",
    "    accuracy.append(metrics.accuracy_score(ytest,y_predict))\n",
    "print(' Accuracy: %0.02f  F1: %0.02f ROC-AOC: %0.02F'\n",
    "      %(np.mean(accuracy),np.mean(F1),np.mean(roc_auc)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "RT = RandomForestClassifier(random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Accuracy: 0.90  F1: 0.61 ROC-AOC: 0.75\n"
     ]
    }
   ],
   "source": [
    "F1=[]\n",
    "roc_auc = []\n",
    "accuracy=[]\n",
    "for train,test in kf.split(X,Y):\n",
    "    xtrain,xtest=X.iloc[train,:],X.iloc[test,:]\n",
    "    ytrain,ytest=Y.iloc[train],Y.iloc[test]\n",
    "    RT.fit(xtrain,ytrain)\n",
    "    y_predict = RT.predict(xtest)\n",
    "    F1.append(metrics.f1_score(ytest,y_predict))\n",
    "    fpr,tpr,_=metrics.roc_curve(ytest,y_predict)\n",
    "    roc_auc.append(metrics.auc(fpr,tpr))\n",
    "    accuracy.append(metrics.accuracy_score(ytest,y_predict))\n",
    "print(' Accuracy: %0.02f  F1: %0.02f ROC-AOC: %0.02F'\n",
    "      %(np.mean(accuracy),np.mean(F1),np.mean(roc_auc)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Recursive Feature Elimination"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    It uses the model accuracy to identify which attributes (and combination of attributes) contribute the most to predicting the target attribute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import RFE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = data1.drop('Revenue',axis=1)\n",
    "Y = data1.Revenue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Features</th>\n",
       "      <th>Support</th>\n",
       "      <th>Rankings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>PageValues</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Month</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>SpecialDay</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>VisitorType</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ExitRates</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>BounceRates</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Weekend</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Informational</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>OperatingSystems</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Administrative</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Browser</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Region</td>\n",
       "      <td>False</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ProductRelated</td>\n",
       "      <td>False</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>TrafficType</td>\n",
       "      <td>False</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Informational_Duration</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Administrative_Duration</td>\n",
       "      <td>False</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ProductRelated_Duration</td>\n",
       "      <td>False</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Features Support Rankings\n",
       "8                PageValues    True        1\n",
       "10                    Month    True        1\n",
       "9                SpecialDay    True        1\n",
       "15              VisitorType    True        1\n",
       "7                 ExitRates    True        1\n",
       "6               BounceRates    True        1\n",
       "16                  Weekend    True        1\n",
       "2             Informational    True        1\n",
       "11         OperatingSystems   False        2\n",
       "0            Administrative   False        3\n",
       "12                  Browser   False        4\n",
       "13                   Region   False        5\n",
       "4            ProductRelated   False        6\n",
       "14              TrafficType   False        7\n",
       "3    Informational_Duration   False        8\n",
       "1   Administrative_Duration   False        9\n",
       "5   ProductRelated_Duration   False       10"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = LogisticRegression()\n",
    "rfe = RFE(model)\n",
    "fit = rfe.fit(X, Y)\n",
    "ff = pd.DataFrame([data1.columns[:-1],fit.support_,fit.ranking_],index=['Features','Support','Rankings']).T\n",
    "ff.sort_values('Rankings')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "LR = LogisticRegression()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  K Fold Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "kf = KFold(n_splits=5,shuffle=True,random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = data1[['PageValues','Month','SpecialDay','VisitorType','ExitRates','BounceRates',\n",
    "           'Weekend','Informational']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = data1.Revenue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Accuracy: 0.88  F1: 0.49 ROC-AOC: 0.67\n"
     ]
    }
   ],
   "source": [
    "F1=[]\n",
    "roc_auc = []\n",
    "accuracy=[]\n",
    "for train,test in kf.split(X,Y):\n",
    "    xtrain,xtest=X.iloc[train,:],X.iloc[test,:]\n",
    "    ytrain,ytest=Y.iloc[train],Y.iloc[test]\n",
    "    LR.fit(xtrain,ytrain)\n",
    "    y_predict = LR.predict(xtest)\n",
    "    F1.append(metrics.f1_score(ytest,y_predict))\n",
    "    fpr,tpr,_=metrics.roc_curve(ytest,y_predict)\n",
    "    roc_auc.append(metrics.auc(fpr,tpr))\n",
    "    accuracy.append(metrics.accuracy_score(ytest,y_predict))\n",
    "print(' Accuracy: %0.02f  F1: %0.02f ROC-AOC: %0.02F'\n",
    "      %(np.mean(accuracy),np.mean(F1),np.mean(roc_auc)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "RT = RandomForestClassifier(random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Accuracy: 0.88  F1: 0.58 ROC-AOC: 0.73\n"
     ]
    }
   ],
   "source": [
    "F1=[]\n",
    "roc_auc = []\n",
    "accuracy=[]\n",
    "for train,test in kf.split(X,Y):\n",
    "    xtrain,xtest=X.iloc[train,:],X.iloc[test,:]\n",
    "    ytrain,ytest=Y.iloc[train],Y.iloc[test]\n",
    "    RT.fit(xtrain,ytrain)\n",
    "    y_predict = RT.predict(xtest)\n",
    "    F1.append(metrics.f1_score(ytest,y_predict))\n",
    "    fpr,tpr,_=metrics.roc_curve(ytest,y_predict)\n",
    "    roc_auc.append(metrics.auc(fpr,tpr))\n",
    "    accuracy.append(metrics.accuracy_score(ytest,y_predict))\n",
    "print(' Accuracy: %0.02f  F1: %0.02f ROC-AOC: %0.02F'\n",
    "      %(np.mean(accuracy),np.mean(F1),np.mean(roc_auc)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Importance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Bagged decision trees like Random Forest and Extra Trees can be used to estimate the importance of features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import ExtraTreesClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = data1.drop('Revenue',axis=1)\n",
    "Y = data1.Revenue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Features</th>\n",
       "      <th>Importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>PageValues</td>\n",
       "      <td>0.33371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ExitRates</td>\n",
       "      <td>0.0709816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ProductRelated_Duration</td>\n",
       "      <td>0.0682977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ProductRelated</td>\n",
       "      <td>0.0637527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>BounceRates</td>\n",
       "      <td>0.0578388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Month</td>\n",
       "      <td>0.0544314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Administrative</td>\n",
       "      <td>0.0504058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Administrative_Duration</td>\n",
       "      <td>0.0486242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Region</td>\n",
       "      <td>0.0451208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>TrafficType</td>\n",
       "      <td>0.0435808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>OperatingSystems</td>\n",
       "      <td>0.0343297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Browser</td>\n",
       "      <td>0.0332433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Informational_Duration</td>\n",
       "      <td>0.0285435</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Informational</td>\n",
       "      <td>0.0281921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Weekend</td>\n",
       "      <td>0.0156704</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>VisitorType</td>\n",
       "      <td>0.0150854</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>SpecialDay</td>\n",
       "      <td>0.00819191</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Features  Importance\n",
       "8                PageValues     0.33371\n",
       "7                 ExitRates   0.0709816\n",
       "5   ProductRelated_Duration   0.0682977\n",
       "4            ProductRelated   0.0637527\n",
       "6               BounceRates   0.0578388\n",
       "10                    Month   0.0544314\n",
       "0            Administrative   0.0504058\n",
       "1   Administrative_Duration   0.0486242\n",
       "13                   Region   0.0451208\n",
       "14              TrafficType   0.0435808\n",
       "11         OperatingSystems   0.0343297\n",
       "12                  Browser   0.0332433\n",
       "3    Informational_Duration   0.0285435\n",
       "2             Informational   0.0281921\n",
       "16                  Weekend   0.0156704\n",
       "15              VisitorType   0.0150854\n",
       "9                SpecialDay  0.00819191"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = ExtraTreesClassifier()\n",
    "model.fit(X, Y)\n",
    "ff = pd.DataFrame([data1.columns[:-1],model.feature_importances_],index=['Features','Importance']).T\n",
    "ff.sort_values('Importance',ascending = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "LR = LogisticRegression()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  K Fold Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "kf = KFold(n_splits=5,shuffle=True,random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = data1[['PageValues','ExitRates','BounceRates','ProductRelated',\n",
    "           'Month','Administrative','TrafficType',\n",
    "           'Region']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = data1[['PageValues','ExitRates','BounceRates','ProductRelated','ProductRelated_Duration',\n",
    "           'Month','Administrative_Duration','Administrative','TrafficType',\n",
    "           'Region']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = data1.Revenue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Accuracy: 0.88  F1: 0.49 ROC-AOC: 0.67\n"
     ]
    }
   ],
   "source": [
    "F1=[]\n",
    "roc_auc = []\n",
    "accuracy=[]\n",
    "for train,test in kf.split(X,Y):\n",
    "    xtrain,xtest=X.iloc[train,:],X.iloc[test,:]\n",
    "    ytrain,ytest=Y.iloc[train],Y.iloc[test]\n",
    "    LR.fit(xtrain,ytrain)\n",
    "    y_predict = LR.predict(xtest)\n",
    "    F1.append(metrics.f1_score(ytest,y_predict))\n",
    "    fpr,tpr,_=metrics.roc_curve(ytest,y_predict)\n",
    "    roc_auc.append(metrics.auc(fpr,tpr))\n",
    "    accuracy.append(metrics.accuracy_score(ytest,y_predict))\n",
    "print(' Accuracy: %0.02f  F1: %0.02f ROC-AOC: %0.02F'\n",
    "      %(np.mean(accuracy),np.mean(F1),np.mean(roc_auc)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "RT = RandomForestClassifier(random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Accuracy: 0.90  F1: 0.62 ROC-AOC: 0.75\n"
     ]
    }
   ],
   "source": [
    "F1=[]\n",
    "roc_auc = []\n",
    "accuracy=[]\n",
    "for train,test in kf.split(X,Y):\n",
    "    xtrain,xtest=X.iloc[train,:],X.iloc[test,:]\n",
    "    ytrain,ytest=Y.iloc[train],Y.iloc[test]\n",
    "    RT.fit(xtrain,ytrain)\n",
    "    y_predict = RT.predict(xtest)\n",
    "    F1.append(metrics.f1_score(ytest,y_predict))\n",
    "    fpr,tpr,_=metrics.roc_curve(ytest,y_predict)\n",
    "    roc_auc.append(metrics.auc(fpr,tpr))\n",
    "    accuracy.append(metrics.accuracy_score(ytest,y_predict))\n",
    "print(' Accuracy: %0.02f  F1: %0.02f ROC-AOC: %0.02F'\n",
    "      %(np.mean(accuracy),np.mean(F1),np.mean(roc_auc)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inference:\n",
    "        From all the three feature selection models both extra bag classifier and Univariate selection has given the best scores\n",
    "        Extra bag classifier has given importance to the variables that are highly correlated with the Target variable.\n",
    "        It can be used as a final selection model"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}