ch02-missing-data-imputation/Recipe-08-Adding-a-missing-value-indicator-variable.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adding a missing value indicator variable\n",
    "\n",
    "In this recipe, we will add binary variables to indicate that a value is missing using pandas, Scikit-learn and Feature-Engine, all open source Python libraries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# to split the data sets\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# to impute missing data with feature-engine\n",
    "from feature_engine.missing_data_imputers import AddNaNBinaryImputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A1</th>\n",
       "      <th>A2</th>\n",
       "      <th>A3</th>\n",
       "      <th>A4</th>\n",
       "      <th>A5</th>\n",
       "      <th>A6</th>\n",
       "      <th>A7</th>\n",
       "      <th>A8</th>\n",
       "      <th>A9</th>\n",
       "      <th>A10</th>\n",
       "      <th>A11</th>\n",
       "      <th>A12</th>\n",
       "      <th>A13</th>\n",
       "      <th>A14</th>\n",
       "      <th>A15</th>\n",
       "      <th>A16</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>b</td>\n",
       "      <td>30.83</td>\n",
       "      <td>0.000</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>1.25</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>1</td>\n",
       "      <td>f</td>\n",
       "      <td>g</td>\n",
       "      <td>202.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>a</td>\n",
       "      <td>58.67</td>\n",
       "      <td>4.460</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>q</td>\n",
       "      <td>h</td>\n",
       "      <td>3.04</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>6</td>\n",
       "      <td>f</td>\n",
       "      <td>g</td>\n",
       "      <td>43.0</td>\n",
       "      <td>560</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>a</td>\n",
       "      <td>24.50</td>\n",
       "      <td>NaN</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>q</td>\n",
       "      <td>h</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>f</td>\n",
       "      <td>g</td>\n",
       "      <td>280.0</td>\n",
       "      <td>824</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>b</td>\n",
       "      <td>27.83</td>\n",
       "      <td>1.540</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>3.75</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>5</td>\n",
       "      <td>t</td>\n",
       "      <td>g</td>\n",
       "      <td>100.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>b</td>\n",
       "      <td>20.17</td>\n",
       "      <td>5.625</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>1.71</td>\n",
       "      <td>t</td>\n",
       "      <td>f</td>\n",
       "      <td>0</td>\n",
       "      <td>f</td>\n",
       "      <td>s</td>\n",
       "      <td>120.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  A1     A2     A3 A4 A5 A6 A7    A8   A9  A10  A11 A12 A13    A14  A15  A16\n",
       "0  b  30.83  0.000  u  g  w  v  1.25    t    t    1   f   g  202.0    0    1\n",
       "1  a  58.67  4.460  u  g  q  h  3.04    t    t    6   f   g   43.0  560    1\n",
       "2  a  24.50    NaN  u  g  q  h   NaN  NaN  NaN    0   f   g  280.0  824    1\n",
       "3  b  27.83  1.540  u  g  w  v  3.75    t    t    5   t   g  100.0    3    1\n",
       "4  b  20.17  5.625  u  g  w  v  1.71    t    f    0   f   s  120.0    0    1"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load data\n",
    "data = pd.read_csv('creditApprovalUCI.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((483, 15), (207, 15))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's separate into training and testing sets\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    data.drop('A16', axis=1), data['A16'], test_size=0.3, random_state=0)\n",
    "\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "A1     0.008282\n",
       "A2     0.022774\n",
       "A3     0.140787\n",
       "A4     0.008282\n",
       "A5     0.008282\n",
       "A6     0.008282\n",
       "A7     0.008282\n",
       "A8     0.140787\n",
       "A9     0.140787\n",
       "A10    0.140787\n",
       "A11    0.000000\n",
       "A12    0.000000\n",
       "A13    0.000000\n",
       "A14    0.014493\n",
       "A15    0.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find the percentage of missing data within those variables\n",
    "\n",
    "X_train.isnull().mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add missing indicator with pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A1</th>\n",
       "      <th>A2</th>\n",
       "      <th>A3</th>\n",
       "      <th>A4</th>\n",
       "      <th>A5</th>\n",
       "      <th>A6</th>\n",
       "      <th>A7</th>\n",
       "      <th>A8</th>\n",
       "      <th>A9</th>\n",
       "      <th>A10</th>\n",
       "      <th>...</th>\n",
       "      <th>A13</th>\n",
       "      <th>A14</th>\n",
       "      <th>A15</th>\n",
       "      <th>A1_NA</th>\n",
       "      <th>A3_NA</th>\n",
       "      <th>A4_NA</th>\n",
       "      <th>A5_NA</th>\n",
       "      <th>A6_NA</th>\n",
       "      <th>A7_NA</th>\n",
       "      <th>A8_NA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>596</td>\n",
       "      <td>a</td>\n",
       "      <td>46.08</td>\n",
       "      <td>3.000</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>c</td>\n",
       "      <td>v</td>\n",
       "      <td>2.375</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>...</td>\n",
       "      <td>g</td>\n",
       "      <td>396.0</td>\n",
       "      <td>4159</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>303</td>\n",
       "      <td>a</td>\n",
       "      <td>15.92</td>\n",
       "      <td>2.875</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>q</td>\n",
       "      <td>v</td>\n",
       "      <td>0.085</td>\n",
       "      <td>f</td>\n",
       "      <td>f</td>\n",
       "      <td>...</td>\n",
       "      <td>g</td>\n",
       "      <td>120.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>204</td>\n",
       "      <td>b</td>\n",
       "      <td>36.33</td>\n",
       "      <td>2.125</td>\n",
       "      <td>y</td>\n",
       "      <td>p</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>0.085</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>...</td>\n",
       "      <td>g</td>\n",
       "      <td>50.0</td>\n",
       "      <td>1187</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>351</td>\n",
       "      <td>b</td>\n",
       "      <td>22.17</td>\n",
       "      <td>0.585</td>\n",
       "      <td>y</td>\n",
       "      <td>p</td>\n",
       "      <td>ff</td>\n",
       "      <td>ff</td>\n",
       "      <td>0.000</td>\n",
       "      <td>f</td>\n",
       "      <td>f</td>\n",
       "      <td>...</td>\n",
       "      <td>g</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>118</td>\n",
       "      <td>b</td>\n",
       "      <td>57.83</td>\n",
       "      <td>7.040</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>m</td>\n",
       "      <td>v</td>\n",
       "      <td>14.000</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>...</td>\n",
       "      <td>g</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1332</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    A1     A2     A3 A4 A5  A6  A7      A8 A9 A10  ...  A13    A14   A15  \\\n",
       "596  a  46.08  3.000  u  g   c   v   2.375  t   t  ...    g  396.0  4159   \n",
       "303  a  15.92  2.875  u  g   q   v   0.085  f   f  ...    g  120.0     0   \n",
       "204  b  36.33  2.125  y  p   w   v   0.085  t   t  ...    g   50.0  1187   \n",
       "351  b  22.17  0.585  y  p  ff  ff   0.000  f   f  ...    g  100.0     0   \n",
       "118  b  57.83  7.040  u  g   m   v  14.000  t   t  ...    g  360.0  1332   \n",
       "\n",
       "     A1_NA  A3_NA  A4_NA  A5_NA  A6_NA  A7_NA  A8_NA  \n",
       "596      0      0      0      0      0      0      0  \n",
       "303      0      0      0      0      0      0      0  \n",
       "204      0      0      0      0      0      0      0  \n",
       "351      0      0      0      0      0      0      0  \n",
       "118      0      0      0      0      0      0      0  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add missing indicator\n",
    "\n",
    "for var in ['A1', 'A3', 'A4', 'A5', 'A6', 'A7', 'A8']:\n",
    "\n",
    "    X_train[var+'_NA'] = np.where(X_train[var].isnull(), 1, 0)\n",
    "    X_test[var+'_NA'] = np.where(X_test[var].isnull(), 1, 0)\n",
    "\n",
    "    \n",
    "# check the new missing indicator variables\n",
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.14078674948240166, 0.14078674948240166)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the mean of the missing indicator should be the same as the \n",
    "# percentage of missing values in the original variable\n",
    "\n",
    "X_train['A3'].isnull().mean(), X_train['A3_NA'].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adding missing indicator with Feature Engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's separate into training and testing set\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    data.drop('A16', axis=1), data['A16'], test_size=0.3, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AddNaNBinaryImputer(variables=['A1', 'A2', 'A3', 'A4', 'A5', 'A6', 'A7', 'A8',\n",
       "                               'A9', 'A10', 'A11', 'A12', 'A13', 'A14', 'A15'])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's create a random sample imputer\n",
    "\n",
    "imputer = AddNaNBinaryImputer()\n",
    "\n",
    "imputer.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Sole\\Anaconda3\\envs\\feml\\lib\\site-packages\\sklearn\\utils\\validation.py:933: FutureWarning: Passing attributes to check_is_fitted is deprecated and will be removed in 0.23. The attributes argument is ignored.\n",
      "  \"argument is ignored.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# transform the data - replace the missing values\n",
    "\n",
    "X_train = imputer.transform(X_train)\n",
    "X_test = imputer.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A1</th>\n",
       "      <th>A2</th>\n",
       "      <th>A3</th>\n",
       "      <th>A4</th>\n",
       "      <th>A5</th>\n",
       "      <th>A6</th>\n",
       "      <th>A7</th>\n",
       "      <th>A8</th>\n",
       "      <th>A9</th>\n",
       "      <th>A10</th>\n",
       "      <th>...</th>\n",
       "      <th>A6_na</th>\n",
       "      <th>A7_na</th>\n",
       "      <th>A8_na</th>\n",
       "      <th>A9_na</th>\n",
       "      <th>A10_na</th>\n",
       "      <th>A11_na</th>\n",
       "      <th>A12_na</th>\n",
       "      <th>A13_na</th>\n",
       "      <th>A14_na</th>\n",
       "      <th>A15_na</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>596</td>\n",
       "      <td>a</td>\n",
       "      <td>46.08</td>\n",
       "      <td>3.000</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>c</td>\n",
       "      <td>v</td>\n",
       "      <td>2.375</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>303</td>\n",
       "      <td>a</td>\n",
       "      <td>15.92</td>\n",
       "      <td>2.875</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>q</td>\n",
       "      <td>v</td>\n",
       "      <td>0.085</td>\n",
       "      <td>f</td>\n",
       "      <td>f</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>204</td>\n",
       "      <td>b</td>\n",
       "      <td>36.33</td>\n",
       "      <td>2.125</td>\n",
       "      <td>y</td>\n",
       "      <td>p</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>0.085</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>351</td>\n",
       "      <td>b</td>\n",
       "      <td>22.17</td>\n",
       "      <td>0.585</td>\n",
       "      <td>y</td>\n",
       "      <td>p</td>\n",
       "      <td>ff</td>\n",
       "      <td>ff</td>\n",
       "      <td>0.000</td>\n",
       "      <td>f</td>\n",
       "      <td>f</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>118</td>\n",
       "      <td>b</td>\n",
       "      <td>57.83</td>\n",
       "      <td>7.040</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>m</td>\n",
       "      <td>v</td>\n",
       "      <td>14.000</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    A1     A2     A3 A4 A5  A6  A7      A8 A9 A10  ...  A6_na A7_na A8_na  \\\n",
       "596  a  46.08  3.000  u  g   c   v   2.375  t   t  ...      0     0     0   \n",
       "303  a  15.92  2.875  u  g   q   v   0.085  f   f  ...      0     0     0   \n",
       "204  b  36.33  2.125  y  p   w   v   0.085  t   t  ...      0     0     0   \n",
       "351  b  22.17  0.585  y  p  ff  ff   0.000  f   f  ...      0     0     0   \n",
       "118  b  57.83  7.040  u  g   m   v  14.000  t   t  ...      0     0     0   \n",
       "\n",
       "     A9_na  A10_na  A11_na  A12_na  A13_na  A14_na  A15_na  \n",
       "596      0       0       0       0       0       0       0  \n",
       "303      0       0       0       0       0       0       0  \n",
       "204      0       0       0       0       0       0       0  \n",
       "351      0       0       0       0       0       0       0  \n",
       "118      0       0       0       0       0       0       0  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check that null values were replaced\n",
    "X_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adding missing indicator with Scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.impute import MissingIndicator\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('creditApprovalUCI.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    data.drop('A16', axis=1), data['A16'], test_size=0.3, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MissingIndicator(error_on_new=True, features='missing-only', missing_values=nan,\n",
       "                 sparse='auto')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indicator = MissingIndicator(error_on_new=True, features='missing-only')\n",
    "indicator.fit(X_train)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 13], dtype=int64)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# we can see the features with na:\n",
    "# the result shows the column index in the NumPy array\n",
    "\n",
    "indicator.features_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>A1</th>\n",
       "      <th>A2</th>\n",
       "      <th>A3</th>\n",
       "      <th>A4</th>\n",
       "      <th>A5</th>\n",
       "      <th>A6</th>\n",
       "      <th>A7</th>\n",
       "      <th>A8</th>\n",
       "      <th>A9</th>\n",
       "      <th>...</th>\n",
       "      <th>A2_NA</th>\n",
       "      <th>A3_NA</th>\n",
       "      <th>A4_NA</th>\n",
       "      <th>A5_NA</th>\n",
       "      <th>A6_NA</th>\n",
       "      <th>A7_NA</th>\n",
       "      <th>A8_NA</th>\n",
       "      <th>A9_NA</th>\n",
       "      <th>A10_NA</th>\n",
       "      <th>A14_NA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>596</td>\n",
       "      <td>a</td>\n",
       "      <td>46.08</td>\n",
       "      <td>3.000</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>c</td>\n",
       "      <td>v</td>\n",
       "      <td>2.375</td>\n",
       "      <td>t</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>303</td>\n",
       "      <td>a</td>\n",
       "      <td>15.92</td>\n",
       "      <td>2.875</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>q</td>\n",
       "      <td>v</td>\n",
       "      <td>0.085</td>\n",
       "      <td>f</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>204</td>\n",
       "      <td>b</td>\n",
       "      <td>36.33</td>\n",
       "      <td>2.125</td>\n",
       "      <td>y</td>\n",
       "      <td>p</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>0.085</td>\n",
       "      <td>t</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>351</td>\n",
       "      <td>b</td>\n",
       "      <td>22.17</td>\n",
       "      <td>0.585</td>\n",
       "      <td>y</td>\n",
       "      <td>p</td>\n",
       "      <td>ff</td>\n",
       "      <td>ff</td>\n",
       "      <td>0.000</td>\n",
       "      <td>f</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>118</td>\n",
       "      <td>b</td>\n",
       "      <td>57.83</td>\n",
       "      <td>7.040</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>m</td>\n",
       "      <td>v</td>\n",
       "      <td>14.000</td>\n",
       "      <td>t</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   index A1     A2     A3 A4 A5  A6  A7      A8 A9  ...  A2_NA  A3_NA  A4_NA  \\\n",
       "0    596  a  46.08  3.000  u  g   c   v   2.375  t  ...  False  False  False   \n",
       "1    303  a  15.92  2.875  u  g   q   v   0.085  f  ...  False  False  False   \n",
       "2    204  b  36.33  2.125  y  p   w   v   0.085  t  ...  False  False  False   \n",
       "3    351  b  22.17  0.585  y  p  ff  ff   0.000  f  ...  False  False  False   \n",
       "4    118  b  57.83  7.040  u  g   m   v  14.000  t  ...  False  False  False   \n",
       "\n",
       "   A5_NA  A6_NA  A7_NA  A8_NA  A9_NA  A10_NA  A14_NA  \n",
       "0  False  False  False  False  False   False   False  \n",
       "1  False  False  False  False  False   False   False  \n",
       "2  False  False  False  False  False   False   False  \n",
       "3  False  False  False  False  False   False   False  \n",
       "4  False  False  False  False  False   False   False  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# with Sklearn we need to join the missing indicators dataframe\n",
    "# to the original X_train\n",
    "\n",
    "# let's create a column name for each of the new MissingIndicators\n",
    "indicator_cols = [c+'_NA' for c in X_train.columns[indicator.features_]]\n",
    "\n",
    "# and now let's concatenate the original dataset with the missing indicators\n",
    "X_train = pd.concat([\n",
    "    X_train.reset_index(),\n",
    "    pd.DataFrame(indicator.transform(X_train), columns = indicator_cols)],\n",
    "    axis=1)\n",
    "\n",
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "feml",
   "language": "python",
   "name": "feml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {
    "height": "697.556px",
    "left": "0px",
    "right": "1370.33px",
    "top": "110.444px",
    "width": "336.333px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}