2-1_Basic_NN-MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Section 2-1 - Basic Neural Network - MNIST"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this section, we'll apply techniques developed so far on the MNIST dataset. The MNIST dataset consists of hand-drawn digits from zero to nine. Each image is 28 pixels in height and 28 pixels in width, with the pixel value an integer between 0 and 255."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from time import time\n",
    "\n",
    "np.random.seed(1337)\n",
    "\n",
    "df = pd.read_csv('data/mnist.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For illustrative purposes, the first example is shown with pixel values between 0 and 10."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,9,3,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,9,9,3,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,9,9,6,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,9,9,8,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,7,9,9,2,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,8,9,9,3,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,9,9,9,6,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,9,9,6,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,0,0,8,9,9,9,3,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,0,1,9,9,9,0,0,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,0,7,9,9,7,0,0,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,6,9,9,7,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,7,9,9,2,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,7,9,9,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,3,9,9,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,3,9,9,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,8,9,9,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,3,9,9,9,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,3,9,9,9,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,8,8,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
      "[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n"
     ]
    }
   ],
   "source": [
    "for item in df.iloc[0, 1:].values.reshape(28,28)/26:\n",
    "    print(''.join(str([int(term) for term in item]).split(' ')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A hand-drawn 1 can clearly be seen from the visualization in grayscale."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGdCAYAAABU0qcqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAZVUlEQVR4nO3db0yV9/3/8dfx39G2cBgiHE5Fi9rqUpVlThmxRRqJwBbjvyXa9YYuRqPDZsraLiyr2G0Jm0u6pguzu6VrVrUzmZp6w8WiHLMNbbQaY7YSIWxgBGxNOAexoIHP74a/nm+Pggqew5sDz0fyScq5rsN599o1n704xwuPc84JAIAhNsZ6AADA6ESAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACAiXHWA9yrt7dX165dU1JSkjwej/U4AIABcs6po6NDgUBAY8b0f50z7AJ07do1ZWVlWY8BAHhMzc3Nmjp1ar/bh92P4JKSkqxHAADEwMP+PI9bgKqqqvTMM89o4sSJys3N1SeffPJIz+PHbgAwMjzsz/O4BOjDDz9UWVmZKioq9OmnnyonJ0dFRUW6fv16PF4OAJCIXBwsWrTIlZaWRr7u6elxgUDAVVZWPvS5oVDISWKxWCxWgq9QKPTAP+9jfgV0+/ZtnT9/XoWFhZHHxowZo8LCQtXW1t63f3d3t8LhcNQCAIx8MQ/QF198oZ6eHmVkZEQ9npGRodbW1vv2r6yslM/niyw+AQcAo4P5p+DKy8sVCoUiq7m52XokAMAQiPnfA0pLS9PYsWPV1tYW9XhbW5v8fv99+3u9Xnm93liPAQAY5mJ+BTRhwgQtWLBA1dXVkcd6e3tVXV2tvLy8WL8cACBBxeVOCGVlZVq/fr2+853vaNGiRXrnnXfU2dmpH/3oR/F4OQBAAopLgNauXavPP/9cO3fuVGtrq771rW/p+PHj930wAQAwenmcc856iK8Lh8Py+XzWYwAAHlMoFFJycnK/280/BQcAGJ0IEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACAiXHWAwDAQJw6dWrAzykoKBjUaw3mecFgcFCvNRpxBQQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmOBmpADMDOWNRTH8cAUEADBBgAAAJmIeoF27dsnj8UStOXPmxPplAAAJLi7vAT3//PP6+OOP/+9FxvFWEwAgWlzKMG7cOPn9/nh8awDACBGX94CuXLmiQCCgGTNm6JVXXlFTU1O/+3Z3dyscDkctAMDIF/MA5ebmat++fTp+/Lj27NmjxsZGvfjii+ro6Ohz/8rKSvl8vsjKysqK9UgAgGHI45xz8XyB9vZ2TZ8+XW+//bY2btx43/bu7m51d3dHvg6Hw0QIGCWG+98DGsxrBYPB2A+SoEKhkJKTk/vdHvdPB6SkpOi5555TfX19n9u9Xq+8Xm+8xwAADDNx/3tAN2/eVENDgzIzM+P9UgCABBLzAL322msKBoP673//q3/9619atWqVxo4dq5dffjnWLwUASGAx/xHc1atX9fLLL+vGjRuaMmWKXnjhBZ05c0ZTpkyJ9UsBABJYzAN08ODBWH9LAAmgoqJiwM8Zqg8U1NTUDOp5fKAgvrgXHADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgIu6/EXWgwuGwfD6f9RjAqLZkyZIBP2ewN/wcCh6Px3qEUelhvxGVKyAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYGGc9AIDhZzjf2bqgoMB6BMQIV0AAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAluRgokiCVLlgz4OcP5pqLS4G4sGgwGYz8ITHAFBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCY4GakgIGReGPRwczHjUVHN66AAAAmCBAAwMSAA3T69GktX75cgUBAHo9HR44cidrunNPOnTuVmZmpSZMmqbCwUFeuXInVvACAEWLAAers7FROTo6qqqr63L579269++67eu+993T27Fk9+eSTKioqUldX12MPCwAYOQb8IYSSkhKVlJT0uc05p3feeUe/+MUvtGLFCknS+++/r4yMDB05ckTr1q17vGkBACNGTN8DamxsVGtrqwoLCyOP+Xw+5ebmqra2ts/ndHd3KxwORy0AwMgX0wC1trZKkjIyMqIez8jIiGy7V2VlpXw+X2RlZWXFciQAwDBl/im48vJyhUKhyGpubrYeCQAwBGIaIL/fL0lqa2uLerytrS2y7V5er1fJyclRCwAw8sU0QNnZ2fL7/aquro48Fg6HdfbsWeXl5cXypQAACW7An4K7efOm6uvrI183Njbq4sWLSk1N1bRp07R9+3b9+te/1rPPPqvs7Gy9+eabCgQCWrlyZSznBgAkuAEH6Ny5c3rppZciX5eVlUmS1q9fr3379umNN95QZ2enNm/erPb2dr3wwgs6fvy4Jk6cGLupAQAJz+Occ9ZDfF04HJbP57MeA4irYfZ/u/sM5saiX/8PU0CSQqHQA9/XN/8UHABgdCJAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAICJAf86BgDRhvudrQdj165d1iNgFOAKCABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwwc1IgRHM4/FYjwD0iysgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAENyMFvsY5Zz1Cv2pqaqxHAGKKKyAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQ3I8WIdOrUKesRHmgwNxZ96aWXYj8IYIgrIACACQIEADAx4ACdPn1ay5cvVyAQkMfj0ZEjR6K2b9iwQR6PJ2oVFxfHal4AwAgx4AB1dnYqJydHVVVV/e5TXFyslpaWyDpw4MBjDQkAGHkG/CGEkpISlZSUPHAfr9crv98/6KEAACNfXN4DqqmpUXp6umbPnq2tW7fqxo0b/e7b3d2tcDgctQAAI1/MA1RcXKz3339f1dXV+u1vf6tgMKiSkhL19PT0uX9lZaV8Pl9kZWVlxXokAMAwFPO/B7Ru3brIP8+bN0/z58/XzJkzVVNTo6VLl963f3l5ucrKyiJfh8NhIgQAo0DcP4Y9Y8YMpaWlqb6+vs/tXq9XycnJUQsAMPLFPUBXr17VjRs3lJmZGe+XAgAkkAH/CO7mzZtRVzONjY26ePGiUlNTlZqaqrfeektr1qyR3+9XQ0OD3njjDc2aNUtFRUUxHRwAkNgGHKBz585F3ZPqq/dv1q9frz179ujSpUv685//rPb2dgUCAS1btky/+tWv5PV6Yzc1ACDhDThABQUFcs71u/3vf//7Yw0EjAa7du2yHgEwx73gAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYCLmv5IbiLVTp04N+DkFBQWxHySGrxUMBmM/CJBguAICAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAExwM1IMqYqKigE/ZyhvLFpTUzPg53BjUWBwuAICAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEx4nHPOeoivC4fD8vl81mPgESxZsmTAzxnMzT4HY7Cv89JLL8V2EGAUC4VCSk5O7nc7V0AAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAluRopBG2anTpSCgoJBPS8YDMZ2EGAU42akAIBhiQABAEwMKECVlZVauHChkpKSlJ6erpUrV6quri5qn66uLpWWlmry5Ml66qmntGbNGrW1tcV0aABA4htQgILBoEpLS3XmzBmdOHFCd+7c0bJly9TZ2RnZZ8eOHfroo4906NAhBYNBXbt2TatXr4754ACAxPZYH0L4/PPPlZ6ermAwqPz8fIVCIU2ZMkX79+/XD37wA0nSZ599pm9+85uqra3Vd7/73Yd+Tz6EkDj4EAKAB4nrhxBCoZAkKTU1VZJ0/vx53blzR4WFhZF95syZo2nTpqm2trbP79Hd3a1wOBy1AAAj36AD1Nvbq+3bt2vx4sWaO3euJKm1tVUTJkxQSkpK1L4ZGRlqbW3t8/tUVlbK5/NFVlZW1mBHAgAkkEEHqLS0VJcvX9bBgwcfa4Dy8nKFQqHIam5ufqzvBwBIDOMG86Rt27bp2LFjOn36tKZOnRp53O/36/bt22pvb4+6Cmpra5Pf7+/ze3m9Xnm93sGMAQBIYAO6AnLOadu2bTp8+LBOnjyp7OzsqO0LFizQ+PHjVV1dHXmsrq5OTU1NysvLi83EAIARYUBXQKWlpdq/f7+OHj2qpKSkyPs6Pp9PkyZNks/n08aNG1VWVqbU1FQlJyfr1VdfVV5e3iN9Ag4AMHoMKEB79uyRdP9HXPfu3asNGzZIkn7/+99rzJgxWrNmjbq7u1VUVKQ//vGPMRkWADByDChAj/L3PiZOnKiqqipVVVUNeigMrSVLlliPAGAU4l5wAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMDGo34gKDHf3/sqQRxUMBmM7CIB+cQUEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJjwOOec9RBfFw6H5fP5rMfAIxiqU2cwNxblpqKAvVAopOTk5H63cwUEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJgYZz0AEpfH47EeAUAC4woIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmBhQgCorK7Vw4UIlJSUpPT1dK1euVF1dXdQ+BQUF8ng8UWvLli0xHRoAkPgGFKBgMKjS0lKdOXNGJ06c0J07d7Rs2TJ1dnZG7bdp0ya1tLRE1u7du2M6NAAg8Q3oN6IeP3486ut9+/YpPT1d58+fV35+fuTxJ554Qn6/PzYTAgBGpMd6DygUCkmSUlNTox7/4IMPlJaWprlz56q8vFy3bt3q93t0d3crHA5HLQDAKOAGqaenx33/+993ixcvjnr8T3/6kzt+/Li7dOmS+8tf/uKefvppt2rVqn6/T0VFhZPEYrFYrBG2QqHQAzsy6ABt2bLFTZ8+3TU3Nz9wv+rqaifJ1dfX97m9q6vLhUKhyGpubjY/aCwWi8V6/PWwAA3oPaCvbNu2TceOHdPp06c1derUB+6bm5srSaqvr9fMmTPv2+71euX1egczBgAggQ0oQM45vfrqqzp8+LBqamqUnZ390OdcvHhRkpSZmTmoAQEAI9OAAlRaWqr9+/fr6NGjSkpKUmtrqyTJ5/Np0qRJamho0P79+/W9731PkydP1qVLl7Rjxw7l5+dr/vz5cfkXAAAkqIG876N+fs63d+9e55xzTU1NLj8/36Wmpjqv1+tmzZrlXn/99Yf+HPDrQqGQ+c8tWSwWi/X462F/9nv+f1iGjXA4LJ/PZz0GAOAxhUIhJScn97ude8EBAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwMuwA556xHAADEwMP+PB92Aero6LAeAQAQAw/789zjhtklR29vr65du6akpCR5PJ6obeFwWFlZWWpublZycrLRhPY4DndxHO7iONzFcbhrOBwH55w6OjoUCAQ0Zkz/1znjhnCmRzJmzBhNnTr1gfskJyeP6hPsKxyHuzgOd3Ec7uI43GV9HHw+30P3GXY/ggMAjA4ECABgIqEC5PV6VVFRIa/Xaz2KKY7DXRyHuzgOd3Ec7kqk4zDsPoQAABgdEuoKCAAwchAgAIAJAgQAMEGAAAAmEiZAVVVVeuaZZzRx4kTl5ubqk08+sR5pyO3atUsejydqzZkzx3qsuDt9+rSWL1+uQCAgj8ejI0eORG13zmnnzp3KzMzUpEmTVFhYqCtXrtgMG0cPOw4bNmy47/woLi62GTZOKisrtXDhQiUlJSk9PV0rV65UXV1d1D5dXV0qLS3V5MmT9dRTT2nNmjVqa2szmjg+HuU4FBQU3Hc+bNmyxWjiviVEgD788EOVlZWpoqJCn376qXJyclRUVKTr169bjzbknn/+ebW0tETWP/7xD+uR4q6zs1M5OTmqqqrqc/vu3bv17rvv6r333tPZs2f15JNPqqioSF1dXUM8aXw97DhIUnFxcdT5ceDAgSGcMP6CwaBKS0t15swZnThxQnfu3NGyZcvU2dkZ2WfHjh366KOPdOjQIQWDQV27dk2rV682nDr2HuU4SNKmTZuizofdu3cbTdwPlwAWLVrkSktLI1/39PS4QCDgKisrDacaehUVFS4nJ8d6DFOS3OHDhyNf9/b2Or/f7373u99FHmtvb3der9cdOHDAYMKhce9xcM659evXuxUrVpjMY+X69etOkgsGg865u//bjx8/3h06dCiyz3/+8x8nydXW1lqNGXf3HgfnnFuyZIn7yU9+YjfUIxj2V0C3b9/W+fPnVVhYGHlszJgxKiwsVG1treFkNq5cuaJAIKAZM2bolVdeUVNTk/VIphobG9Xa2hp1fvh8PuXm5o7K86Ompkbp6emaPXu2tm7dqhs3bliPFFehUEiSlJqaKkk6f/687ty5E3U+zJkzR9OmTRvR58O9x+ErH3zwgdLS0jR37lyVl5fr1q1bFuP1a9jdjPReX3zxhXp6epSRkRH1eEZGhj777DOjqWzk5uZq3759mj17tlpaWvTWW2/pxRdf1OXLl5WUlGQ9nonW1lZJ6vP8+GrbaFFcXKzVq1crOztbDQ0N+vnPf66SkhLV1tZq7Nix1uPFXG9vr7Zv367Fixdr7ty5ku6eDxMmTFBKSkrUviP5fOjrOEjSD3/4Q02fPl2BQECXLl3Sz372M9XV1elvf/ub4bTRhn2A8H9KSkoi/zx//nzl5uZq+vTp+utf/6qNGzcaTobhYN26dZF/njdvnubPn6+ZM2eqpqZGS5cuNZwsPkpLS3X58uVR8T7og/R3HDZv3hz553nz5ikzM1NLly5VQ0ODZs6cOdRj9mnY/wguLS1NY8eOve9TLG1tbfL7/UZTDQ8pKSl67rnnVF9fbz2Kma/OAc6P+82YMUNpaWkj8vzYtm2bjh07plOnTkX9+ha/36/bt2+rvb09av+Rej70dxz6kpubK0nD6nwY9gGaMGGCFixYoOrq6shjvb29qq6uVl5enuFk9m7evKmGhgZlZmZaj2ImOztbfr8/6vwIh8M6e/bsqD8/rl69qhs3boyo88M5p23btunw4cM6efKksrOzo7YvWLBA48ePjzof6urq1NTUNKLOh4cdh75cvHhRkobX+WD9KYhHcfDgQef1et2+ffvcv//9b7d582aXkpLiWltbrUcbUj/96U9dTU2Na2xsdP/85z9dYWGhS0tLc9evX7ceLa46OjrchQsX3IULF5wk9/bbb7sLFy64//3vf845537zm9+4lJQUd/ToUXfp0iW3YsUKl52d7b788kvjyWPrQceho6PDvfbaa662ttY1Nja6jz/+2H372992zz77rOvq6rIePWa2bt3qfD6fq6mpcS0tLZF169atyD5btmxx06ZNcydPnnTnzp1zeXl5Li8vz3Dq2HvYcaivr3e//OUv3blz51xjY6M7evSomzFjhsvPzzeePFpCBMg55/7whz+4adOmuQkTJrhFixa5M2fOWI805NauXesyMzPdhAkT3NNPP+3Wrl3r6uvrrceKu1OnTjlJ963169c75+5+FPvNN990GRkZzuv1uqVLl7q6ujrboePgQcfh1q1bbtmyZW7KlClu/Pjxbvr06W7Tpk0j7j/S+vr3l+T27t0b2efLL790P/7xj903vvEN98QTT7hVq1a5lpYWu6Hj4GHHoampyeXn57vU1FTn9XrdrFmz3Ouvv+5CoZDt4Pfg1zEAAEwM+/eAAAAjEwECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABg4v8BybhKo7JU7yoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.imshow(df.iloc[0, 1:].values.reshape(28,28), cmap=plt.get_cmap('gray', 5))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are 42,000 images. We similarly split the images 80:20 into training and test sets, and scale the data through division by 255. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = df.iloc[:33600, :]\n",
    "\n",
    "X_train = df_train.iloc[:, 1:].values / 255.\n",
    "y_train = df_train['label'].values\n",
    "y_train_onehot = pd.get_dummies(df_train['label']).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = df.iloc[33600:, :]\n",
    "\n",
    "X_test = df_test.iloc[:, 1:].values / 255.\n",
    "y_test = df_test['label'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.2s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "building tree 1 of 100\n",
      "building tree 2 of 100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.4s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "building tree 3 of 100\n",
      "building tree 4 of 100\n",
      "building tree 5 of 100\n",
      "building tree 6 of 100\n",
      "building tree 7 of 100\n",
      "building tree 8 of 100\n",
      "building tree 9 of 100\n",
      "building tree 10 of 100\n",
      "building tree 11 of 100\n",
      "building tree 12 of 100\n",
      "building tree 13 of 100\n",
      "building tree 14 of 100\n",
      "building tree 15 of 100\n",
      "building tree 16 of 100\n",
      "building tree 17 of 100\n",
      "building tree 18 of 100\n",
      "building tree 19 of 100\n",
      "building tree 20 of 100\n",
      "building tree 21 of 100\n",
      "building tree 22 of 100\n",
      "building tree 23 of 100\n",
      "building tree 24 of 100\n",
      "building tree 25 of 100\n",
      "building tree 26 of 100\n",
      "building tree 27 of 100\n",
      "building tree 28 of 100\n",
      "building tree 29 of 100\n",
      "building tree 30 of 100\n",
      "building tree 31 of 100\n",
      "building tree 32 of 100\n",
      "building tree 33 of 100\n",
      "building tree 34 of 100\n",
      "building tree 35 of 100\n",
      "building tree 36 of 100\n",
      "building tree 37 of 100\n",
      "building tree 38 of 100\n",
      "building tree 39 of 100\n",
      "building tree 40 of 100\n",
      "building tree 41 of 100\n",
      "building tree 42 of 100\n",
      "building tree 43 of 100\n",
      "building tree 44 of 100\n",
      "building tree 45 of 100\n",
      "building tree 46 of 100\n",
      "building tree 47 of 100\n",
      "building tree 48 of 100\n",
      "building tree 49 of 100\n",
      "building tree 50 of 100\n",
      "building tree 51 of 100\n",
      "building tree 52 of 100\n",
      "building tree 53 of 100\n",
      "building tree 54 of 100\n",
      "building tree 55 of 100\n",
      "building tree 56 of 100\n",
      "building tree 57 of 100\n",
      "building tree 58 of 100\n",
      "building tree 59 of 100\n",
      "building tree 60 of 100\n",
      "building tree 61 of 100\n",
      "building tree 62 of 100\n",
      "building tree 63 of 100\n",
      "building tree 64 of 100\n",
      "building tree 65 of 100\n",
      "building tree 66 of 100\n",
      "building tree 67 of 100\n",
      "building tree 68 of 100\n",
      "building tree 69 of 100\n",
      "building tree 70 of 100\n",
      "building tree 71 of 100\n",
      "building tree 72 of 100\n",
      "building tree 73 of 100\n",
      "building tree 74 of 100\n",
      "building tree 75 of 100\n",
      "building tree 76 of 100\n",
      "building tree 77 of 100\n",
      "building tree 78 of 100\n",
      "building tree 79 of 100\n",
      "building tree 80 of 100\n",
      "building tree 81 of 100\n",
      "building tree 82 of 100\n",
      "building tree 83 of 100\n",
      "building tree 84 of 100\n",
      "building tree 85 of 100\n",
      "building tree 86 of 100\n",
      "building tree 87 of 100\n",
      "building tree 88 of 100\n",
      "building tree 89 of 100\n",
      "building tree 90 of 100\n",
      "building tree 91 of 100\n",
      "building tree 92 of 100\n",
      "building tree 93 of 100\n",
      "building tree 94 of 100\n",
      "building tree 95 of 100\n",
      "building tree 96 of 100\n",
      "building tree 97 of 100\n",
      "building tree 98 of 100\n",
      "building tree 99 of 100\n",
      "building tree 100 of 100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:   19.2s finished\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "accuracy 0.965\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 100 out of 100 | elapsed:    0.2s finished\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "model = RandomForestClassifier(random_state=0, verbose=3)\n",
    "model = model.fit(X_train, df_train['label'].values)\n",
    "\n",
    "y_prediction = model.predict(X_test)\n",
    "print(\"\\naccuracy\", np.sum(y_prediction == df_test['label'].values) / float(len(y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1-layer Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-24 10:36:11.623274: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/12\n",
      "1050/1050 [==============================] - 1s 738us/step - loss: 0.9533 - accuracy: 0.7783\n",
      "Epoch 2/12\n",
      "1050/1050 [==============================] - 1s 714us/step - loss: 0.5329 - accuracy: 0.8669\n",
      "Epoch 3/12\n",
      "1050/1050 [==============================] - 1s 708us/step - loss: 0.4576 - accuracy: 0.8801\n",
      "Epoch 4/12\n",
      "1050/1050 [==============================] - 1s 718us/step - loss: 0.4206 - accuracy: 0.8875\n",
      "Epoch 5/12\n",
      "1050/1050 [==============================] - 1s 714us/step - loss: 0.3972 - accuracy: 0.8918\n",
      "Epoch 6/12\n",
      "1050/1050 [==============================] - 1s 717us/step - loss: 0.3809 - accuracy: 0.8953\n",
      "Epoch 7/12\n",
      "1050/1050 [==============================] - 1s 727us/step - loss: 0.3684 - accuracy: 0.8982\n",
      "Epoch 8/12\n",
      "1050/1050 [==============================] - 1s 713us/step - loss: 0.3586 - accuracy: 0.9007\n",
      "Epoch 9/12\n",
      "1050/1050 [==============================] - 1s 715us/step - loss: 0.3506 - accuracy: 0.9027\n",
      "Epoch 10/12\n",
      "1050/1050 [==============================] - 1s 714us/step - loss: 0.3440 - accuracy: 0.9047\n",
      "Epoch 11/12\n",
      "1050/1050 [==============================] - 1s 711us/step - loss: 0.3382 - accuracy: 0.9062\n",
      "Epoch 12/12\n",
      "1050/1050 [==============================] - 1s 714us/step - loss: 0.3332 - accuracy: 0.9069\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " dense (Dense)               (32, 10)                  7850      \n",
      "                                                                 \n",
      " activation (Activation)     (32, 10)                  0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 7,850\n",
      "Trainable params: 7,850\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "\n",
      "time taken 9.667016744613647 seconds\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation\n",
    "\n",
    "start = time()\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(10))\n",
    "model.add(Activation(\"softmax\"))\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy'])\n",
    "\n",
    "model.fit(X_train, y_train_onehot, epochs=12)\n",
    "\n",
    "model.summary()\n",
    "\n",
    "print('\\ntime taken %s seconds' % str(time() - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "263/263 [==============================] - 0s 560us/step\n",
      "\n",
      "accuracy 0.909047619047619\n"
     ]
    }
   ],
   "source": [
    "y_prediction = np.argmax(model.predict(X_test), axis=-1)\n",
    "print(\"\\naccuracy\", np.sum(y_prediction == y_test) / float(len(y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2-layer Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.7052 - accuracy: 0.8178\n",
      "Epoch 2/12\n",
      "1050/1050 [==============================] - 1s 999us/step - loss: 0.3979 - accuracy: 0.8877\n",
      "Epoch 3/12\n",
      "1050/1050 [==============================] - 1s 995us/step - loss: 0.3534 - accuracy: 0.8993\n",
      "Epoch 4/12\n",
      "1050/1050 [==============================] - 1s 999us/step - loss: 0.3312 - accuracy: 0.9051\n",
      "Epoch 5/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.3170 - accuracy: 0.9093\n",
      "Epoch 6/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.3072 - accuracy: 0.9117\n",
      "Epoch 7/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.3000 - accuracy: 0.9153\n",
      "Epoch 8/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.2937 - accuracy: 0.9165\n",
      "Epoch 9/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.2889 - accuracy: 0.9182\n",
      "Epoch 10/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.2847 - accuracy: 0.9182\n",
      "Epoch 11/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.2813 - accuracy: 0.9211\n",
      "Epoch 12/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.2778 - accuracy: 0.9212\n",
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " dense_1 (Dense)             (32, 100)                 78500     \n",
      "                                                                 \n",
      " dense_2 (Dense)             (32, 10)                  1010      \n",
      "                                                                 \n",
      " activation_1 (Activation)   (32, 10)                  0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 79,510\n",
      "Trainable params: 79,510\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "\n",
      "time taken 13.243598699569702 seconds\n"
     ]
    }
   ],
   "source": [
    "start = time()\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(100))\n",
    "model.add(Dense(10))\n",
    "model.add(Activation(\"softmax\"))\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy'])\n",
    "\n",
    "model.fit(X_train, y_train_onehot, epochs=12)\n",
    "\n",
    "model.summary()\n",
    "\n",
    "print('\\ntime taken %s seconds' % str(time() - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "263/263 [==============================] - 0s 765us/step\n",
      "\n",
      "accuracy 0.9204761904761904\n"
     ]
    }
   ],
   "source": [
    "y_prediction = np.argmax(model.predict(X_test), axis=-1)\n",
    "print(\"\\naccuracy\", np.sum(y_prediction == y_test) / float(len(y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3-layer Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.6318 - accuracy: 0.8294\n",
      "Epoch 2/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.3598 - accuracy: 0.8961\n",
      "Epoch 3/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.3256 - accuracy: 0.9061\n",
      "Epoch 4/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.3080 - accuracy: 0.9110\n",
      "Epoch 5/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.2976 - accuracy: 0.9146\n",
      "Epoch 6/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.2893 - accuracy: 0.9176\n",
      "Epoch 7/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.2827 - accuracy: 0.9187\n",
      "Epoch 8/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.2788 - accuracy: 0.9202\n",
      "Epoch 9/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.2744 - accuracy: 0.9222\n",
      "Epoch 10/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.2709 - accuracy: 0.9222\n",
      "Epoch 11/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.2677 - accuracy: 0.9241\n",
      "Epoch 12/12\n",
      "1050/1050 [==============================] - 1s 1ms/step - loss: 0.2649 - accuracy: 0.9251\n",
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " dense_3 (Dense)             (32, 100)                 78500     \n",
      "                                                                 \n",
      " dense_4 (Dense)             (32, 100)                 10100     \n",
      "                                                                 \n",
      " dense_5 (Dense)             (32, 10)                  1010      \n",
      "                                                                 \n",
      " activation_2 (Activation)   (32, 10)                  0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 89,610\n",
      "Trainable params: 89,610\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "\n",
      "time taken 13.777081727981567 seconds\n"
     ]
    }
   ],
   "source": [
    "start = time()\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(100))\n",
    "model.add(Dense(100))\n",
    "model.add(Dense(10))\n",
    "model.add(Activation(\"softmax\"))\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy'])\n",
    "\n",
    "model.fit(X_train, y_train_onehot, epochs=12)\n",
    "\n",
    "model.summary()\n",
    "\n",
    "print('\\ntime taken %s seconds' % str(time() - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "263/263 [==============================] - 0s 735us/step\n",
      "\n",
      "accuracy 0.9204761904761904\n"
     ]
    }
   ],
   "source": [
    "y_prediction = np.argmax(model.predict(X_test), axis=-1)\n",
    "print(\"\\naccuracy\", np.sum(y_prediction == y_test) / float(len(y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As in the previous section, we were able to drive loss down even further with additional layers. While we see improvements in accuracy, it's still not enough to beat the benchmark. We'll look into more advanced techniques to enhance model performance in next section."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}