renumbered chapters, expanded reqs.txt, improved ch4 var transf

Author: Soledad Galli

ch1-variable-characteristics/DataPrep_Titanic.ipynb renamed to ch01-variable-characteristics/DataPrep_Titanic.ipynb

@@ -180,6 +180,7 @@
     "data = pd.DataFrame(boston_dataset.data,\n",
     "                      columns=boston_dataset.feature_names)\n",
     "\n",
+    "# display top 5 rows (not in book recipe)\n",
     "data.head()"
    ]
   },
@@ -202,7 +203,7 @@
     }
    ],
    "source": [
-    "# plot the histogram to inspect variable distributions\n",
+    "# plot the histogram to inspect variable distributions (not in book)\n",
     "\n",
     "data.hist(bins=30, figsize=(12,12))\n",
     "plt.show()"
@@ -268,6 +269,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# make a copy of the dataframe where we will store the modified\n",
+    "# variables\n",
+    "\n",
     "data_tf = data.copy()"
    ]
   },
@@ -277,6 +281,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# apply log to a set of variables\n",
     "data_tf[['LSTAT', 'NOX', 'DIS', 'RM']] = np.log(\n",
     "    data[['LSTAT', 'NOX', 'DIS', 'RM']])"
    ]
@@ -300,6 +305,8 @@
     }
    ],
    "source": [
+    "# use diagnostic plot function to address variable transformation\n",
+    "\n",
     "diagnostic_plots(data_tf, 'LSTAT')"
    ]
   },
@@ -316,6 +323,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# start the transformer with NumPy log as argument\n",
     "transformer = FunctionTransformer(np.log, validate=True)"
    ]
   },
@@ -325,8 +333,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# make a list of columns to transform\n",
     "cols = ['LSTAT', 'NOX', 'DIS', 'RM']\n",
     "\n",
+    "# transform subset of dataframe\n",
     "data_t = transformer.transform(data[cols])"
    ]
   },
@@ -336,6 +346,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# capture returned NumPy array in pandas dataframe\n",
     "data_tf = pd.DataFrame(data_t, columns=cols)"
    ]
   },
@@ -358,14 +369,16 @@
     }
    ],
    "source": [
+    "# use diagnostic plot function to evaluate transformation\n",
+    "# (not in book)\n",
     "diagnostic_plots(data_tf, 'LSTAT')"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Logarithmic transformation with Feature Engine"
+    "## Logarithmic transformation with Feature-engine"
    ]
   },
   {
@@ -385,8 +398,10 @@
     }
    ],
    "source": [
+    "# initialize the transformer\n",
     "lt = LogTransformer(variables=['LSTAT', 'NOX', 'DIS', 'RM'])\n",
     "\n",
+    "# fit transformer to data set\n",
     "lt.fit(data)"
    ]
   },
@@ -396,6 +411,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# transform variables within our data set\n",
     "data_tf = lt.transform(data)"
    ]
   },
@@ -418,14 +434,19 @@
     }
    ],
    "source": [
+    "# use diagnostic plot function to evaluate transformation\n",
+    "# (not in book)\n",
+    "\n",
     "diagnostic_plots(data_tf, 'LSTAT')"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Logarithm with Scikit-learn selecting a group of variables"
+    "## Logarithm with Scikit-learn selecting a group of variables\n",
+    "\n",
+    "### Not in book"
    ]
   },
   {
@@ -450,11 +471,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# load data set\n",
     "boston_dataset = load_boston()\n",
     "\n",
     "# create a dataframe with the independent variables\n",
     "data = pd.DataFrame(boston_dataset.data,\n",
-    "                      columns=boston_dataset.feature_names)"
+    "                    columns=boston_dataset.feature_names)"
    ]
   },
   {
@@ -463,10 +485,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# list of variables to transform\n",
     "cols = ['LSTAT', 'NOX', 'DIS', 'RM']\n",
     "\n",
-    "log_transformer = Pipeline(steps=[('log_transformer', FunctionTransformer(np.log, validate=True))])\n",
+    "# set transformer within a pipeline\n",
+    "log_transformer = Pipeline(steps=[('log_transformer',\n",
+    "                                   FunctionTransformer(np.log, validate=True))])\n",
     "\n",
+    "# set pipeline within ColumnTransformer to select features\n",
     "preprocessor = ColumnTransformer(transformers=[\n",
     "    ('log_transformer', log_transformer, cols)], remainder='passthrough')"
    ]
@@ -477,8 +503,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# fit to the data (the entire dataframe this time)\n",
     "preprocessor.fit(data)\n",
     "\n",
+    "# transform the data\n",
     "data_tf = preprocessor.transform(data)"
    ]
   },
@@ -1641,15 +1669,9 @@
     }
    ],
    "source": [
+    "# visualize the data as a pandas dataframe\n",
     "pd.DataFrame(data_tf)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

ch1-variable-characteristics/Recipe-1-indetifying-variables-types.ipynb renamed to ch01-variable-characteristics/Recipe-1-indetifying-variables-types.ipynb

@@ -180,6 +180,7 @@
     "data = pd.DataFrame(boston_dataset.data,\n",
     "                      columns=boston_dataset.feature_names)\n",
     "\n",
+    "# display top 5 rows (not in book recipe)\n",
     "data.head()"
    ]
   },
@@ -202,7 +203,7 @@
     }
    ],
    "source": [
-    "# plot the histogram to inspect variable distributions\n",
+    "# plot the histogram to inspect variable distributions (not in book)\n",
     "\n",
     "data.hist(bins=30, figsize=(12,12))\n",
     "plt.show()"
@@ -250,7 +251,7 @@
     }
    ],
    "source": [
-    "# check original distribution\n",
+    "# check original distribution of the variable DIS\n",
     "\n",
     "diagnostic_plots(data, 'DIS')"
    ]
@@ -268,6 +269,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# make a copy of the dataframe where we will store the modified\n",
+    "# variables\n",
+    "\n",
     "data_tf = data.copy()"
    ]
   },
@@ -277,6 +281,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# apply the reciprocal to a set of variables\n",
+    "\n",
     "data_tf[['LSTAT', 'NOX', 'DIS', 'RM']] = np.reciprocal(data[['LSTAT', 'NOX', 'DIS', 'RM']])"
    ]
   },
@@ -299,6 +305,8 @@
     }
    ],
    "source": [
+    "# use diagnostic plot function to address variable transformation\n",
+    "\n",
     "diagnostic_plots(data_tf, 'DIS')"
    ]
   },
@@ -315,6 +323,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# start the transformer with NumPy reciprocal as argument\n",
+    "\n",
     "transformer = FunctionTransformer(np.reciprocal, validate=True)"
    ]
   },
@@ -324,8 +334,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# make a list of columns to transform\n",
     "cols = ['LSTAT', 'NOX', 'DIS', 'RM']\n",
     "\n",
+    "# transform subset of dataframe\n",
     "data_t = transformer.transform(data[cols])"
    ]
   },
@@ -335,6 +347,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# capture returned NumPy array in pandas dataframe\n",
+    "\n",
     "data_tf = pd.DataFrame(data_t, columns=cols)"
    ]
   },
@@ -357,14 +371,17 @@
     }
    ],
    "source": [
+    "# use diagnostic plot function to evaluate transformation\n",
+    "# (not in book)\n",
+    "\n",
     "diagnostic_plots(data_tf, 'DIS')"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Reciprocal transformation with Feature Engine"
+    "## Reciprocal transformation with Feature-engine"
    ]
   },
   {
@@ -384,7 +401,10 @@
     }
    ],
    "source": [
+    "# initialize the transformer\n",
     "rt = ReciprocalTransformer(variables = ['LSTAT', 'NOX', 'DIS', 'RM'])\n",
+    "\n",
+    "# fit transformer to the entire dataframe\n",
     "rt.fit(data)"
    ]
   },
@@ -394,6 +414,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# transform the indicated variables within our data set\n",
     "data_tf = rt.transform(data)"
    ]
   },
@@ -416,15 +437,11 @@
     }
    ],
    "source": [
+    "# use diagnostic plot function to evaluate transformation\n",
+    "# (not in book)\n",
+    "\n",
     "diagnostic_plots(data_tf, 'DIS')"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {