mnist_0-9.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://pytorch.org/tutorials/beginner/blitz/neural_networks_tutorial.html#sphx-glr-beginner-blitz-neural-networks-tutorial-py\n",
    "#Special thanks to Balint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "!pip install -Uqq fastbook\n",
    "import fastbook\n",
    "fastbook.setup_book()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from fastai.vision.all import *\n",
    "from fastbook import *\n",
    "import os\n",
    "\n",
    "matplotlib.rc('image', cmap='Greys')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = untar_data(URLs.MNIST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Path('/home/sjmec1/.fastai/data/mnist_png')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "Path.BASE_PATH = path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(#3) [Path('export.pkl'),Path('testing'),Path('training')]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path.ls()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Putting It All Together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data(dirname,x):\n",
    "    targetDir = path/dirname/x\n",
    "    targetImages = targetDir.ls().sorted()\n",
    "    targetTensors = [tensor(Image.open(o)) for o in targetImages]\n",
    "    targetStacked = torch.stack(targetTensors).float()/255\n",
    "    return targetStacked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dataset_len(dirname,x):\n",
    "    return len((path/dirname/x).ls())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x = torch.cat([get_data(\"training\",\"0\"), \n",
    "                     get_data(\"training\",\"1\"),\n",
    "                     get_data(\"training\",\"2\"), \n",
    "                     get_data(\"training\",\"3\"),\n",
    "                     get_data(\"training\",\"4\"), \n",
    "                     get_data(\"training\",\"5\"),\n",
    "                     get_data(\"training\",\"6\"), \n",
    "                     get_data(\"training\",\"7\"),\n",
    "                     get_data(\"training\",\"8\"), \n",
    "                     get_data(\"training\",\"9\")]).view(-1, 28*28)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y = tensor([1]*get_dataset_len(\"training\",\"0\") + \n",
    "                  [0]*get_dataset_len(\"training\",\"1\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"2\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"3\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"4\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"5\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"6\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"7\") + \n",
    "                  [0]*get_dataset_len(\"training\",\"8\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"9\")).unsqueeze(1) \n",
    "train_y = torch.cat((train_y,\n",
    "           tensor([0]*get_dataset_len(\"training\",\"0\") + \n",
    "                  [1]*get_dataset_len(\"training\",\"1\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"2\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"3\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"4\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"5\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"6\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"7\") + \n",
    "                  [0]*get_dataset_len(\"training\",\"8\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"9\")).unsqueeze(1)),1)\n",
    "train_y = torch.cat((train_y,\n",
    "           tensor([0]*get_dataset_len(\"training\",\"0\") + \n",
    "                  [0]*get_dataset_len(\"training\",\"1\") +\n",
    "                  [1]*get_dataset_len(\"training\",\"2\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"3\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"4\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"5\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"6\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"7\") + \n",
    "                  [0]*get_dataset_len(\"training\",\"8\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"9\")).unsqueeze(1)),1)\n",
    "train_y = torch.cat((train_y,\n",
    "           tensor([0]*get_dataset_len(\"training\",\"0\") + \n",
    "                  [0]*get_dataset_len(\"training\",\"1\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"2\") +\n",
    "                  [1]*get_dataset_len(\"training\",\"3\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"4\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"5\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"6\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"7\") + \n",
    "                  [0]*get_dataset_len(\"training\",\"8\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"9\")).unsqueeze(1)),1)\n",
    "train_y = torch.cat((train_y,\n",
    "           tensor([0]*get_dataset_len(\"training\",\"0\") + \n",
    "                  [0]*get_dataset_len(\"training\",\"1\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"2\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"3\") +\n",
    "                  [1]*get_dataset_len(\"training\",\"4\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"5\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"6\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"7\") + \n",
    "                  [0]*get_dataset_len(\"training\",\"8\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"9\")).unsqueeze(1)),1)\n",
    "train_y = torch.cat((train_y,\n",
    "           tensor([0]*get_dataset_len(\"training\",\"0\") + \n",
    "                  [0]*get_dataset_len(\"training\",\"1\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"2\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"3\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"4\") +\n",
    "                  [1]*get_dataset_len(\"training\",\"5\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"6\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"7\") + \n",
    "                  [0]*get_dataset_len(\"training\",\"8\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"9\")).unsqueeze(1)),1)\n",
    "train_y = torch.cat((train_y,\n",
    "           tensor([0]*get_dataset_len(\"training\",\"0\") + \n",
    "                  [0]*get_dataset_len(\"training\",\"1\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"2\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"3\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"4\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"5\") +\n",
    "                  [1]*get_dataset_len(\"training\",\"6\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"7\") + \n",
    "                  [0]*get_dataset_len(\"training\",\"8\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"9\")).unsqueeze(1)),1)\n",
    "train_y = torch.cat((train_y,\n",
    "           tensor([0]*get_dataset_len(\"training\",\"0\") + \n",
    "                  [0]*get_dataset_len(\"training\",\"1\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"2\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"3\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"4\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"5\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"6\") +\n",
    "                  [1]*get_dataset_len(\"training\",\"7\") + \n",
    "                  [0]*get_dataset_len(\"training\",\"8\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"9\")).unsqueeze(1)),1)\n",
    "train_y = torch.cat((train_y,\n",
    "           tensor([0]*get_dataset_len(\"training\",\"0\") + \n",
    "                  [0]*get_dataset_len(\"training\",\"1\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"2\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"3\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"4\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"5\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"6\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"7\") + \n",
    "                  [1]*get_dataset_len(\"training\",\"8\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"9\")).unsqueeze(1)),1)\n",
    "train_y = torch.cat((train_y,\n",
    "           tensor([0]*get_dataset_len(\"training\",\"0\") + \n",
    "                  [0]*get_dataset_len(\"training\",\"1\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"2\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"3\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"4\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"5\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"6\") +\n",
    "                  [0]*get_dataset_len(\"training\",\"7\") + \n",
    "                  [0]*get_dataset_len(\"training\",\"8\") +\n",
    "                  [1]*get_dataset_len(\"training\",\"9\")).unsqueeze(1)),1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([60000, 784]), torch.Size([60000, 10]))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x.shape,train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_x = torch.cat([get_data(\"testing\",\"0\"), \n",
    "                     get_data(\"testing\",\"1\"),\n",
    "                     get_data(\"testing\",\"2\"), \n",
    "                     get_data(\"testing\",\"3\"),\n",
    "                     get_data(\"testing\",\"4\"), \n",
    "                     get_data(\"testing\",\"5\"),\n",
    "                     get_data(\"testing\",\"6\"), \n",
    "                     get_data(\"testing\",\"7\"),\n",
    "                     get_data(\"testing\",\"8\"), \n",
    "                     get_data(\"testing\",\"9\")]).view(-1, 28*28)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_y = tensor([1]*get_dataset_len(\"testing\",\"0\") + \n",
    "                  [0]*get_dataset_len(\"testing\",\"1\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"2\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"3\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"4\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"5\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"6\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"7\") + \n",
    "                  [0]*get_dataset_len(\"testing\",\"8\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"9\")).unsqueeze(1) \n",
    "valid_y = torch.cat((valid_y,\n",
    "           tensor([0]*get_dataset_len(\"testing\",\"0\") + \n",
    "                  [1]*get_dataset_len(\"testing\",\"1\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"2\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"3\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"4\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"5\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"6\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"7\") + \n",
    "                  [0]*get_dataset_len(\"testing\",\"8\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"9\")).unsqueeze(1)),1)\n",
    "valid_y = torch.cat((valid_y,\n",
    "           tensor([0]*get_dataset_len(\"testing\",\"0\") + \n",
    "                  [0]*get_dataset_len(\"testing\",\"1\") +\n",
    "                  [1]*get_dataset_len(\"testing\",\"2\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"3\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"4\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"5\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"6\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"7\") + \n",
    "                  [0]*get_dataset_len(\"testing\",\"8\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"9\")).unsqueeze(1)),1)\n",
    "valid_y = torch.cat((valid_y,\n",
    "           tensor([0]*get_dataset_len(\"testing\",\"0\") + \n",
    "                  [0]*get_dataset_len(\"testing\",\"1\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"2\") +\n",
    "                  [1]*get_dataset_len(\"testing\",\"3\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"4\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"5\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"6\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"7\") + \n",
    "                  [0]*get_dataset_len(\"testing\",\"8\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"9\")).unsqueeze(1)),1)\n",
    "valid_y = torch.cat((valid_y,\n",
    "           tensor([0]*get_dataset_len(\"testing\",\"0\") + \n",
    "                  [0]*get_dataset_len(\"testing\",\"1\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"2\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"3\") +\n",
    "                  [1]*get_dataset_len(\"testing\",\"4\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"5\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"6\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"7\") + \n",
    "                  [0]*get_dataset_len(\"testing\",\"8\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"9\")).unsqueeze(1)),1)\n",
    "valid_y = torch.cat((valid_y,\n",
    "           tensor([0]*get_dataset_len(\"testing\",\"0\") + \n",
    "                  [0]*get_dataset_len(\"testing\",\"1\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"2\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"3\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"4\") +\n",
    "                  [1]*get_dataset_len(\"testing\",\"5\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"6\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"7\") + \n",
    "                  [0]*get_dataset_len(\"testing\",\"8\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"9\")).unsqueeze(1)),1)\n",
    "valid_y = torch.cat((valid_y,\n",
    "           tensor([0]*get_dataset_len(\"testing\",\"0\") + \n",
    "                  [0]*get_dataset_len(\"testing\",\"1\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"2\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"3\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"4\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"5\") +\n",
    "                  [1]*get_dataset_len(\"testing\",\"6\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"7\") + \n",
    "                  [0]*get_dataset_len(\"testing\",\"8\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"9\")).unsqueeze(1)),1)\n",
    "valid_y = torch.cat((valid_y,\n",
    "           tensor([0]*get_dataset_len(\"testing\",\"0\") + \n",
    "                  [0]*get_dataset_len(\"testing\",\"1\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"2\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"3\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"4\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"5\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"6\") +\n",
    "                  [1]*get_dataset_len(\"testing\",\"7\") + \n",
    "                  [0]*get_dataset_len(\"testing\",\"8\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"9\")).unsqueeze(1)),1)\n",
    "valid_y = torch.cat((valid_y,\n",
    "           tensor([0]*get_dataset_len(\"testing\",\"0\") + \n",
    "                  [0]*get_dataset_len(\"testing\",\"1\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"2\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"3\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"4\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"5\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"6\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"7\") + \n",
    "                  [1]*get_dataset_len(\"testing\",\"8\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"9\")).unsqueeze(1)),1)\n",
    "valid_y = torch.cat((valid_y,\n",
    "           tensor([0]*get_dataset_len(\"testing\",\"0\") + \n",
    "                  [0]*get_dataset_len(\"testing\",\"1\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"2\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"3\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"4\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"5\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"6\") +\n",
    "                  [0]*get_dataset_len(\"testing\",\"7\") + \n",
    "                  [0]*get_dataset_len(\"testing\",\"8\") +\n",
    "                  [1]*get_dataset_len(\"testing\",\"9\")).unsqueeze(1)),1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([10000, 784]), torch.Size([10000, 10]))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_x.shape,valid_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Simple Net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_params(size, std=1.0): return (torch.randn(size)*std).requires_grad_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mnist_loss(predictions, targets):\n",
    "    #predictions = predictions.sigmoid()\n",
    "    #return torch.where(targets==1, 1-predictions, predictions).mean()\n",
    "    return torch.where(targets==1, 1-predictions, predictions).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pick_a_file(n):\n",
    "    if n == 0:   x = '/home/sjmec1/.fastai/data/mnist_png/testing/0/1001.png'\n",
    "    elif n == 1: x = '/home/sjmec1/.fastai/data/mnist_png/testing/1/1004.png'\n",
    "    elif n == 2: x = '/home/sjmec1/.fastai/data/mnist_png/testing/2/1002.png'\n",
    "    elif n == 3: x = '/home/sjmec1/.fastai/data/mnist_png/testing/3/1020.png'\n",
    "    elif n == 4: x = '/home/sjmec1/.fastai/data/mnist_png/testing/4/1010.png'\n",
    "    elif n == 5: x = '/home/sjmec1/.fastai/data/mnist_png/testing/5/1003.png'\n",
    "    elif n == 6: x = '/home/sjmec1/.fastai/data/mnist_png/testing/6/100.png'\n",
    "    elif n == 7: x = '/home/sjmec1/.fastai/data/mnist_png/testing/7/0.png'\n",
    "    elif n == 8: x = '/home/sjmec1/.fastai/data/mnist_png/testing/8/1007.png'\n",
    "    elif n == 9: x = '/home/sjmec1/.fastai/data/mnist_png/testing/9/1000.png'\n",
    " \n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "simple_net = nn.Sequential(\n",
    "    nn.Linear(28*28,30),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(30,10),\n",
    "    nn.Softmax(dim=1)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_accuracy(xb, yb):\n",
    "    #preds = xb.sigmoid()\n",
    "    #correct = (preds>0.5) == yb\n",
    "    #return correct.float().mean()\n",
    "    correct = torch.argmax(xb, dim=1) == torch.argmax(yb, dim=1)\n",
    "    return correct.float().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SuggestedLRs(lr_min=5.754399462603033e-05, lr_steep=1.0964781722577754e-06)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEVCAYAAAAsHqjeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU1fn48c+TfU8IhLCGyE5YghIVRAWLgmu1brUoSrVqtdbW9ttvbSuKS21rW/utPyktLYriVhewat0VEcQtiEHCEtawZmELSSD78/tjJjgOk43MzJ0kz/v1mpc3555773MH5Mm559xzRFUxxhhj/CnM6QCMMcZ0PpZcjDHG+J0lF2OMMX5nycUYY4zfWXIxxhjjd5ZcjDHG+J0lF2OMMX4XtOQiIqkislhEKkWkUESmN1P3DhEpEpEyEXlMRKLd5dEiMt99fLmIrBKR87yOjRORv4nIXvfxHwb63owxxnxTMFsuc4AaIB24GpgrIiO9K4nINOBOYAqQCQwE7nXvjgB2AJOAZGAW8LyIZHqcYh6QCoxw//cOv9+JMcaYZkkw3tAXkXjgADBKVQvcZQuBXap6p1fdZ4Btqvpr989TgKdVtVcT514N3KuqL4nIMOBzoJ+qHgrcHRljjGlOsFouQ4H6xsTilgcc03Jxl+V51UsXke7eFUUk3X3ufHfRqUAhcK/7sdhXInKZP27AGGNM60UE6ToJQJlXWRmQ2Iq6jduJwL7GQhGJBJ4GnlDV9e7ifsAo4CWgDzAB+K+IrFXVdZ4XEZGbgJsA4uPjxw0fPvw4bssYY7qulStX7lXVNF/7gpVcKoAkr7IkoLwVdRu3j9YVkTBgIa4+nNs86h4BaoEHVLUOWCoiS4CpwDeSi6rOw9U/Q05Ojubm5rbxlowxpmsTkcKm9gXrsVgBECEiQzzKsvn6cZanfPc+z3rFqroPQEQEmI9rYMBlqlrrUXe1X6M2xhhzXIKSXFS1ElgE3Cci8SIyEbgYV+vD25PADSKSJSLdgLuABR775+IaCXaRqh7xOvZDYDvwKxGJcF9nMvCWP+/HGGNM84I5FPlWIBYoAZ4FblHVfBHJEJEKEckAUNU3gYeAJbg65wuBewBEZABwMzAWKHIfVyEiV7uPrcWVtM7H1VfzT+Bajz4ZY4wxQRCUocihzvpcjDGm7URkparm+Npn078YY4zxO0suxhhj/M6SizGmy6irb6Cqtt7pMLqEYL3nYowxjjlSU88zn23nH0s3c+BwDSdmdGPCwO6cNbwnY/unOB1ep2TJxRjTaakqCz8p5JH3NrG3opoJA7szul8yn2zZx/97fyN/fW8j52Sl8+vzR3BCj3inw+1ULLkYYzqlhgblvtfWsmDFNk4b1J2/XX0Sp5yQenR/2ZFanv60kDnvb2LqX5Yy87RMfj51GDGR4Q5G3XlYcjHGdDr1DcqdL63mhZU7ueH0E7jrghG4Jvf4WnJsJLdOHszl4/rx57cK+OeyrWzdW8nca8YRGW7d0e1l36AxplOpqWvg9udW8cLKnfxkyhCficVTz8QY/nD5GO6/eCTvrivhZ8/nUd9g7/+1l7VcjDGdRlVtPbc8tZIlG0r5zfkjuPHMga0+dsaETCpr6vn9G+uJjwrnd5eObjYpmeZZcjHGdArlVbX84IlcPtu2n99dOprvnZLR5nP8cNIgKqrqeHTJJkrLq7nnopFkdI8LQLSdnyUXY0yHlLfjIO+uKyYmMpzYyHBe/nIXa3cf4q9Xnci3s/sc93l/PnUoybGR/OXdAs7+y1JuPnMgt04eTGyUdfS3hSUXY0yHs6mkgqv/9SkV1XVHy2Iiw/jHjHFMGZHernOLCDeeOZCLsvvwuzfW8f/e38T760t47qbxJMZEtjf0LsMmrsQmrjSmIymvquWSOR9x8HAtr/74dFLjo6iqrScyPIz4aP//vvzO2mJ++NRKJgzszmMzTyYqwsZBNbKJK40xnUJDg/Lz5/PYtu8wj04/iT4pscREhpMSFxWQxAJwTlY6v790NMs37eWXL63GfiFvHXssZozpMOYs2cTba4uZdWEWEwZ1D9p1r8jpT/GhKv70dgE9E6O587zhNpKsBZZcjDEdwtwPNvPndwq4ZGwfrp+YGfTr/+iswRQdquIfH25hb0UND146iugI6+RviiUXY0xIU1X+9PYG5izZzLez+/DHK7IdaTWICPdfPIoeCdH837sbKdxXyT9mjKN7QnTQY+kIgtbnIiKpIrJYRCpFpFBEpjdT9w4RKRKRMhF5TESi3eXRIjLffXy5iKwSkfOaOMc9IqIicnag7skYE1iqyr2vrmXOks1875T+/OW7Yx2dmkVE+OnZQ/l/3zuRr3aVcfGcj9hYXO5YPKEsmH9Kc4AaIB24GpgrIiO9K4nINOBOYAqQCQwE7nXvjgB2AJOAZGAW8LyIZHqdYxBwObDH/7dhjAmWjzbtY8GKbXx/YiYPfmc04WGh0c9xUXYfnr95AtV1DVw6dwUrNu91OqSQE5TkIiLxwGXALFWtUNXlwCvADB/VrwPmq2q+qh4A7gdmAqhqparOVtVtqtqgqq8BW4FxXud4FPglrmRmjOmgnvx4G6nxUfzy3NDrQM/un8LiW0+jV1IM1z32GYu+2Ol0SCElWC2XoUC9qhZ4lOUBx7Rc3GV5XvXSReSYoSEiku4+d75H2RVAjaq+3lxAInKTiOSKSG5paWnr78QYExS7Dx7h3XXFfPfk/iE7DX6/bnG8eMtpnJyZys+ez2P2K/kcrqlr+cAuIFjJJQEo8yorAxJbUbdx+xt1RSQSeBp4QlXXu8sSgAeBn7YUkKrOU9UcVc1JS0tr1U0YY4LnmU+3o8D045gjLJiSYyNZ8P1TmHlaJgtWbOO8vy7jky37nA7LccFKLhVAkldZEuCrJ8y7buP20boiEgYsxPXY6zaPuvcCC1V1a3sDNsY4p6augec+386U4T3pnxr6E0dGRYQx+9sjefbG8ajCVfM+4dH3NzodlqOClVwKgAgRGeJRlo3H4ywP+e59nvWKVXUfgLgevM7HNTDgMlWt9ag7BbjdPdKsCOiPq8P/l/67FWNMoL2xZg97K2qYMSHT6VDaZMKg7rz50zO4KLsPf3q7gCUbSpwOyTFBSS6qWgksAu4TkXgRmQhcjKv14e1J4AYRyRKRbsBdwAKP/XOBEcBFqnrE69gpwChgrPuzG7gZ10g1Y0wH8dQnhWR2j+OMwT2cDqXN4qIi+OPlYxjeK5GfP59HUVmV0yE5IphDkW8FYoES4FngFlXNF5EMEakQkQwAVX0TeAhYAhS6P/cAiMgAXMliLFDkPq5CRK52H7tPVYsaP0A9cEBVK4J4n8aYdli35xCfbzvANeMHEBYiQ4/bKiYynEenn8SRmnpuf24VdfUNTocUdEF7Q19V9wOX+CjfjqsT37PsYeBhH3ULgVb/bVPVzDYHaoxx1HOfbScqIozLx/VzOpR2GdwzgQcuGcXPX8jjkfc28rOpw5wOKahsVmRjTMioqq3n5S93c+7IXqTERTkdTrtdNq4fl4/rxyPvb+LlVbucDieobG4xY0zIeHttMWVHarkyp7/TofjNA5eMYsf+w/zixTx6JERz+pCO1490PKzlYowJGS/k7qBvSiynBXE6/UCLiQxn3rU5DOyRwA+fWkn+bu9X/jonSy7GmJCw88Bhlm/ayxU5/TpsR35TkmMjWXD9ySTGRDDz8c8pOdT5R5BZcjHGhIQXV7rm5uroHflN6Z0cy4Lvn0J5VS3/82LnX9HSkosxxnENDcoLuTs5fXAP+nUL/Tfyj9ewXon85vwRfFhQypMfFzodTkBZcjHGOO7jLfvYdfAIV3SijvymXDN+AJOHpfHg6+vYVNJ514Kx5GKMcZSq8velm0mOjWRqVrrT4QSciPDQ5WOIj47gJ899SU1d53zB0pKLMcZR//lyN8s27uWOs4eE7NT6/tYzMYbfXTqa/N2HeOS9zjnBpSUXY4xj9lfWcN9razkxI6XDTVLZXtNG9uLycf2Yu3QzeTsOOh2O31lyMcY45oHX1lJeVcvvLx0TMksYB9OsC7NIS4jm5y/kUVVb73Q4fmXJxRjjiA8LSlm0ahc/nDSIYb18rRvY+SXHRvKHy8ewqaSCv7xT0PIBHYglF2NM0NXUNTDrP2sYmBbPj84a7HQ4jpo0NI3vnZLBvGVbWFm43+lw/MaSizEm6J7P3UHhvsPMuiCry3TiN+c3F4ygb0osP35mFaXl1U6H4xeWXIwxQVVVW88j723k5MxuTB6W5nQ4ISEhOoK/XzOO/YdruOWplVTXdfz+F0suxpigemLFNkrKq/nFtOG4Vi03AKP6JvPHy7PJLTzAPf/J7/DTwwQtuYhIqogsFpFKESkUkenN1L1DRIpEpExEHhORaHd5tIjMdx9fLiKrROQ8j+PGi8g7IrJfREpF5AUR6R2M+zPGtOxQVS1zl25m8rA0Tjkh1elwQs5F2X247azBPPf5Dp5Ysc3pcNolmC2XOUANkA5cDcwVkZHelURkGnAnMAXIBAYC97p3RwA7gElAMjALeF5EMt37uwHz3McNAMqBxwNwL8aY4/CvZVs5eLiW/+liqzK2xc/OGcrZI9K577W1vPHVHqfDOW5BSS4iEg9cBsxS1QpVXQ68AszwUf06YL6q5qvqAeB+YCaAqlaq6mxV3aaqDar6GrAVGOfe/4aqvqCqh1T1MPAoMDHgN2iMadGByhrmL9vCBaN7M6pvstPhhKywMOGR741lbP8UfvLcl6zYtNfpkI5LsFouQ4F6VfUcyJ0HHNNycZfledVLF5FjVg8SkXT3ufObuO6ZzewzxgTRwk8Kqayp5ydnD3E6lJAXFxXBYzNPJrNHHDc+mcvqnR3vDf5gJZcEwHv5tTLA15tT3nUbt79RV0QigaeBJ1R1vfdJRGQMcDfwC18BichNIpIrIrmlpaWtugljzPGpqq1nwYptTBnek6HpXfOFybZKiYviyetPJSUuipmPf05RWcdaYCxYyaUCSPIqS8LVJ9JS3cbto3VFJAxYiKsP5zbvE4jIYOAN4CequsxXQKo6T1VzVDUnLc2GQxoTSC+s3Mn+yhpunjTI6VA6lF7JMTxx/Skcrqnj14u/6lAjyIKVXAqACBHxbA9n4/uRVb57n2e9YlXdByCusYvzcQ0MuExVaz0PFpEBwLvA/aq60H+3YIw5HvUNyj8/3MKJGSmcnNnN6XA6nME9E/jfacN5f33J0dU6O4KgJBdVrQQWAfeJSLyITAQuxtX68PYkcIOIZIlIN+AuYIHH/rnACOAiVT3ieaCI9AXeB+ao6t/9fyfGmLZ6c00R2/cf5uYzB9p7Lcdp5mmZnJKZyn2vrWVP2ZGWDwgBwRyKfCsQC5QAzwK3qGq+iGSISIWIZACo6pvAQ8ASoND9uQeOtkpuBsYCRe7jKkTkavc1foBr6PI9HvsqgniPxhgPjQuBndAjnnOyejkdTocVFib88Yox1NUrd77UMR6PBS25qOp+Vb1EVeNVNUNVn3GXb1fVBFXd7lH3YVVNV9UkVf2+qla7ywtVVVQ1xn1M4+dp9/573fs99yUE6x6NMd/06db9fLWrjBvPGNglp9T3pwHd47nzvOEsLSjlpS92OR1Oi2z6F2NMwDz72XaSYiK49KS+TofSKcwYP4CTMlL43evrOHi4xulwmmXJxRgTEGWHa3ljTREXj+1rMx/7SViY8MAlozl4pJaH3trgdDjNsuRijAmIV/J2UVPXwHdP7u90KJ1KVp8kZp6WybOfbWfV9gNOh9MkSy7GmIB4PncnI3onMbKP9ytupr3uOGcoPROjuevlNdTVNzgdjk+WXIwxfrd29yG+2lXGd3P62fDjAEiIjuDuC0eSv/sQj3201elwfLLkYozxu+dzdxAVHsbFY60jP1DOH92LqVnp/OHNDXxYEHpTWFlyMcb4VXVdPS9/uYupI9PpFh/ldDidlojwl++OZUjPBH709BdsKvE1m5ZzLLkYY/zqnbXFHDxcy5U51pEfaPHREcyfeTLRkeFcvyCX/ZWhMzzZkosxxq/+u3oP6UnRTBzcw+lQuoS+KbHMu3YcRYequP3ZVSHz9r4lF2OM39TWN7B8417OGtbT3sgPopMyujHrghEs37SX178qcjocwJKLMcaPvig8QHl1HZOH2TIWwTb91AEM75XIg6+vo6q23ulwLLkYY/zng4JSIsLEHok5IDxMuOeikew6eIR/frjF6XAsuRhj/OeDDaWMG9CNxJhIp0PpkiYM6s55o3rxtw82O75ypSUXY4xfFB+qYt2eQ0we1tPpULq0X58/gnpV/vDmMau/B5UlF2OMXyzd4HqRb9JQ629xUv/UOG484wQWr9rFml1ljsVhycUY4xdLC0pJT4pmRO9Ep0Pp8m6eNIjk2EgefqfAsRgsuRhj2q2uvoFlG0uZNDTN5hILAUkxkfxw0iDeX1/CykJnZk4OWnIRkVQRWSwilSJSKCLTm6l7h4gUiUiZiDwmItHu8mgRme8+vlxEVonIeV7HThGR9SJyWESWuJdGNsYE0KodBzlUVWf9LSHkutMG0CMhij+/7cy6L8FsucwBaoB04GpgroiM9K4kItOAO4EpQCYwELjXvTsC2AFMApKBWcDzIpLpPrYHsMhdngrkAv8O0P0YY9w+2FBCuA1BDilxURHcOnkwKzbvY8WmvUG/flCSi4jEA5cBs1S1QlWXA68AM3xUvw6Yr6r5qnoAuB+YCaCqlao6W1W3qWqDqr4GbAXGuY+9FMhX1RdUtQqYDWSLyPBA3p8xXdmaXWX8+/OdjBvQjeRYG4IcSqafmkGvpBj+/E5B0KeFCVbLZShQr6qevUt5wDEtF3dZnle9dBHp7l1RRNLd5873dayqVgKbm7iOMaad3sov4oq/f0xUuHD/xaOcDsd4iYkM58dTBrOy8ADvrSsJ6rWDlVwSAO8xcWWAr2El3nUbt79RV0QigaeBJ1R1fRPHNnkdEblJRHJFJLe0NPTWQjAm1M37cDM/fGolQ3sl8vJtExnWy0aJhaIrc/ozMC2eB19fR01d8FatDFZyqQC81zpNAnwtQOBdt3H7aF0RCQMW4urDue14rqOq81Q1R1Vz0tJsXL4xbbGhqJwHX1/PuSN78e+bxtMzMcbpkEwTIsPDmHVBFlv2VrLwk8KgXTdYyaUAiBCRIR5l2Xz9OMtTvnufZ71iVd0HIK5xjvNxDQy4TFVrmzrW3dczqInrGGOO0zOfFhIVHsZvvzOamMhwp8MxLZg8LI0zh6bx13cLgrbmS1CSi7vvYxFwn4jEi8hE4GJcrQ9vTwI3iEiWiHQD7gIWeOyfC4wALlLVI17HLgZGichlIhID3A2s9nhsZoxpp8M1dSz6Yhfnje5Fqq002SGICLMuGEFlTT1/CdKLlcEcinwrEAuUAM8Ct6hqvohkiEiFiGQAqOqbwEPAEqDQ/bkHwP3Oys3AWKDIfVyFiFztPrYU16i03wIHgFOBq4J4j8Z0eq/l7aG8uo6rT7VXyDqSIemJXHNqBk9/WsiGosAviSyhsmqZk3JycjQ3N9fpMIzpEC6e8xGV1XW8c8eZ9jZ+B3OgsoZJf1zC2IxuPHn9Ke0+n4isVNUcX/ts+hdjTKut2VVG3o6DXH1qhiWWDqhbfBQ//tYQPiwoZdnGwI6SteRijGm1Zz7bTnREGJee2M/pUMxxuva0AfTrFstv/7uO+obAPbmy5GKMaZWK6jr+s2oXF2X3ITnO3sTvqKIjwvnfc4ezvqicRV/sDNh1LLkYY1rl3bXFVNbUc9XJ/Z0OxbTTRWN6k90/hT+/XcCRmvqAXMOSizGmVZYWlJIaH8VJGd2cDsW0k4jwm/NHUHSoivnLtwTkGpZcjDEtamhQPiwo5cwhPQgLs478zuCUE1KZmpXO++tLAjKpZYTfz2iM6XTydx9iX2UNk4bZVEmdyUOXjyExJjIgI/8suRhjWrS0wDWj7hlDLLl0JilxgZthwR6LGWNatLSglNF9k+mREO10KKaDsORijGlW2ZFavth+kElDrdViWs+SizGmWR9t2kt9gzLZ+ltMG1hyMcY0a+mGUhJjIhjbP8XpUEwHYsnFGNMkVWVpQSlnDOlBRLj9c2Far9V/W0TkLBE5wb3dW0SeEJHHRKRX4MIzxjipoLiCokNV1t9i2qwtv4r8DWicJ+DPQCSgwDx/B2WMCQ0fbHANQT7Tkotpo7a859JXVbeLSAQwDRiAaw373QGJzBjjuHfWFjOidxK9k2OdDsV0MG1puRwSkXRgErBWVSvc5TY9qjGdUGl5NSu3H2DayHSnQzEdUFuSy/8DPgeeBua4yyYCrVqfXkRSRWSxiFSKSKGITG+m7h0iUiQiZe5+nWiPfbeJSK6IVIvIAh/HXiki60SkXETWisglbbhHY4zbu+uKUYVpI61b1bRdq5OLqv4BOBuYqKrPuYt3AT9o5Snm4HqMlg5cDcwVkZHelURkGnAnMAXIBAYC93pU2Q08ADzm49i+wFPAz4Ak4BfAMyLSs5UxGmPc3sovIiM1juG9Ep0OxXRAbRpbqKoFqroZXKPHgF6q+lVLx4lIPHAZMEtVK1R1OfAKMMNH9euA+aqar6oHgPuBmR4xLFLVl4F9Po7tBxxU1TfU5b9AJTCoLfdpTFdXXlXLik37mJqVbssZm+PSlqHIS0Vkonv7l8BzwLMi8utWHD4UqFfVAo+yPOCYlou7LM+rXrqIdG/FdXKBdSLybREJdz8SqwZW+7ifm9yP13JLSwO7lrQxHc0HG0qpqW9g2ih7JGaOT1taLqOAT9zbNwKTgfHAD1txbAJQ5lVWBvhqb3vXbdxusW2uqvXAk8AzuJLKM8DNqlrpo+48Vc1R1Zy0NBtmaYynt9cW090WBjPt0JbkEgaoiAwCRFXXqeoOoDV/+ypw9YF4SgLKW1G3cdtX3W8QkbOBh3AlvihcI9v+JSJjWxGjMQaorqtnyfoSzslKJ9wWBjPHqS3JZTnwKPAnYDGAO9HsbcWxBUCEiAzxKMsG8n3UzXfv86xXrKq++li8jQU+VNVcVW1Q1c+BT3ENRDDGtMKKzfuoqK6zUWKmXdqSXGYCB3H1X8x2lw0H/trSge7HUouA+0Qk3t13czGw0Ef1J4EbRCRLRLoBdwELGneKSISIxADhQLiIxLhf7ATXUOkzGlsqInIicAY++lyMMb69nV9MfFQ4Ewa1ppvTGN9a/Ya+u+Xwa6+y/7bhWrfiGj5cgmuk1y2qmi8iGcBaIEtVt6vqmyLyELAEiAVeAu7xOM9dXj9fg2uo8mxVXSois4EX3S98lgIPqurbbYjTmC5LVXlvXTGThqURExnudDimA2t1chGRSFz/sM8A+uB632Qh8FtVrWnpeFXdDxzzQqOqbsfVie9Z9jDwcBPnmc3XLSdf+x/F9fjOGNNG+bsPUVJezbeG21v5pn3aMrfYQ8ApuEaHFeKaW2wWrg73O/wfmjEm2N5fX4IItjCYabe2JJcrgGyPjvUNIvIFrvdQLLkY0wm8t76E7H4p9EiIbrmyMc1oS4d+U2MSbayiMZ1AaXk1q3ce5FvDbbYk035tSS4vAK+KyDQRGSEi5wIvA88HJjRjTDB9sKEEVSy5GL9oy2Ox/8XVoT8HV4f+LlxTwFj72ZhOYMmGEtKTohnZx/t9Z2Pari1DkWuAu90fANzvm1TiSjzGmA6qpq6BZQV7uWBMb5uo0vhFm2ZF9kGxPhdjOrzcbfspr66zR2LGb9qbXMCVYIwxHdj760uICg9j4uAeTodiOokWH4uJyLea2R3lx1iMMQ55f0MJpw5MJT66Ld2wxjStNX+T5rewf7s/AjHGOGNTSQVbSiu5bkKm06GYTqTF5KKqJwQjEGOMM97KLwKwWZCNX/mjz8UY04G9sWYPJ2ak0Cs5xulQTCdiycWYLmzH/sOs2XWIc63VYvzMkosxXVjjI7FzR1lyMf5lycWYLuzNNUWM6J3EgO7xTodiOhlLLsZ0USWHqli5/QDnWavFBEDQkouIpIrIYhGpFJFCEZneTN07RKRIRMpE5DERifbYd5uI5IpItYgs8HFsnIj8TUT2uo//MEC3ZEyH9tbaYlTtkZgJjGC2XOYANUA6cDUwV0RGelcSkWnAncAUIBMYiGsZ40a7gQdwLZnsyzwgFRjh/q+tNWOMD2+tKWJgj3iG9ExoubIxbRSU5CIi8cBlwCxVrVDV5cAruJZM9nYdMF9V81X1AHA/MLNxp6ouUtWXgX3eB4rIMODbwE2qWqqq9aq60v93ZEzHdvBwDR9v2ce5o3rZRJUmIILVchkK1KtqgUdZHnBMy8VdludVL11EurfiOqfiWoL5Xvdjsa9E5LLjDdqYzmppQSn1Dco5WelOh2I6qWAllwSgzKusDEhsRd3GbV91vfUDRrmP6QPcBjwhIiO8K4rITe6+m9zS0tJWnNqYzmPZxr2kxEUypl+K06GYTipYyaUC8F6BKAkob0Xdxm1fdb0dAWqBB1S1RlWXAkuAqd4VVXWequaoak5aWlorTm1M56CqLNtYyumDexAeZo/ETGAEK7kUABEiMsSjLBvI91E3373Ps16xqh7Tx+LD6uMP0ZiuoaC4guJD1Zw5xH6pMoETlOSiqpXAIuA+EYkXkYnAxcBCH9WfBG4QkSwR6YZraeUFjTtFJMK9AmY4EC4iMSLSOAHnh7hmaf6Vu95EYDLwVoBuzZgOZ9lG12Pg04fY2i0mcII5FPlWIBYoAZ4FblHVfBHJEJEKEckAUNU3gYdwPc4qdH/u8TjPXbgef90JXOPevst9bC2upHU+rn6XfwLXqur6wN+eMR3Dhxv3MrhnAn1SYp0OxXRiQVsZSFX3A5f4KN+OqxPfs+xh4OEmzjMbmN3MdfKBCe0I1ZhOq6q2nk+37GP6qRlOh2I6OZv+xZguJHfbAarrGqy/xQScJRdjupBlG0uJDBdOHZjqdCimk7PkYkwX8uHGveQMSCUuKmhPxE0XZcnFmC6ipLyKdXsOceZQeyRmAs+SizFdxPKNewE4w4YgmyCw5GJMF/He+hJ6JEST1dt7sgxj/M+SSztU19Xzr2Vb2FN2xOlQjGlWdV09H6wv4ZysnoTZlC8mCCy5tLuwW+cAABxLSURBVEPJoWoeenMDf3xrg9OhGNOsjzfvo7KmnqlZtjCYCQ5LLu3QPzWO608/gUVf7GL1zoNOh2NMk95eW0xcVDgTBrVm5Qpj2s+SSzv96KxBdI+P4oHX1qGqTodjzDEaGpR31xYzaWgaMZHhTodjughLLu2UGBPJz6YO5bNt+3lzTZHT4RhzjNW7yigpr2bqSFsYzASPJRc/+G5Of4alJ/K7N9ZTXVcPuNbMsJaMCQVv5xcRHiacNayn06GYLsRe0/WDiPAw7rpwBDPmf8aFjyynuq6BkvIq+neLY8H1p9DXZp81DnpnbTGnZKaSEhfldCimC7GWi5+cMSSN70/MpHtCFGP7pzD9lAEUHariqnkfs/PAYafDM13U1r2VbCypsEdiJuis5eJH91w08hs/X3JiH67516dcNe8Tnr1xPP1T4xyKzHRV76x19QOek2XJxQSXtVwCaEy/FJ7+wXjKq+q4at4nVFbXOR2S6WKWrC9leK9E+nWzX2xMcFlyCbDR/ZL561Vj2XXwCMvcczsZEwy19Q2s2nGA8QPt3RYTfEFLLiKSKiKLRaRSRApFZHozde8QkSIRKRORx0Qk2mPfbSKSKyLVIrKgmXPcIyIqImf7+VbabOLgHiRGR7C0oMTpUEwXkr/7EFW1DZycaWu3mOALZstlDlADpANXA3NFZKR3JRGZBtwJTAEygYHAvR5VdgMPAI81dSERGQRcDuzxU+ztEhkexsTBPfhgQ6kNTzZBk7ttPwA5md0cjsR0RUFJLiISD1wGzFLVClVdDrwCzPBR/Tpgvqrmq+oB4H5gZuNOVV2kqi8D+5q55KPAL3Els5AweVgae8qqKCiucDoU00V8vm0/GalxpCfFOB2K6YKC1XIZCtSraoFHWR5wTMvFXZbnVS9dRFr14FhErgBqVPX14w02ECYNcy3Q9MEGezRmAk9VWVl4wFotxjHBSi4JQJlXWRmQ2Iq6jdu+6n6DiCQADwI/bUXdm9x9N7mlpaUtVW+33smxDO+VyNKCwF/LmG37DrO3ooacAdbfYpwRrORSAXivUJQElLeibuO2r7re7gUWqurWliqq6jxVzVHVnLS04Cz7OmloGp9v20+FDUk2Afa5u7/lZGu5GIcEK7kUABEiMsSjLBvI91E3373Ps16xqjbXx9JoCnC7e6RZEdAfeF5EfnmccfvVpGFp1NYrKzbZkGQTWLnb9pMSF8mgtASnQzFdVFCSi6pWAouA+0QkXkQmAhcDC31UfxK4QUSyRKQbcBewoHGniESISAwQDoSLSIyINM40MAUYBYx1f3YDN+Maqea4nAGpxEeF84E9GjMBlrvtADkDutmqk8YxwRyKfCsQC5QAzwK3qGq+iGSISIWIZACo6pvAQ8ASoND9ucfjPHcBR3ANV77GvX2X+9h9qlrU+AHqgQOqGhJDtKIiwjhtcA+W2pBkE0D7KqrZsreSHHu/xTgoaHOLqep+4BIf5dtxdeJ7lj0MPNzEeWYDs1t5zcw2hhlwk4el8c7aYjaVVDAkvcUxCsa0WW7hAcD6W4yzbPqXIJvsXlPjnXXFQbtmTV0DP31uFUtsGHSXkLttP1ERYYzqm+x0KKYLs+QSZH1TYjkxI4XX8oI3ecATK7bx8pe7mfXyGmrrG4J2XeOMz7cdILtfMtERtqSxcY4lFwdcOKYPa/ccYnNp4LuCSsqr+Ot7G8lIjWPngSMs/mJXwK9pnFNVW0/+7jLG2fstxmGWXBxwwejeiBCU1stDb26guq6eBd8/mdF9k3l0ySZrvXRi6/YcorZeGds/xelQTBdnycUBvZJjODkzlVdX7w7oqLEvdxzkxZU7uf70ExiYlsDtU4awff9hXl5lrZfOKm/HQQCy+1t/i3GWJReHXJTdh00lFWwo/nriAVWluq7eL+dvaFDueSWftMRofvwt17urZ4/oycg+STy6ZBN11nrplPJ2ltEzMZpeNlmlcZgtc+yQ80b1YvYr+byat5vhvZIor6rlxidz+XTrfgakxjE0PZHhvZMYN6AbJ2WkkBgT2abzv/zlLvJ2HOTPV2STEO36YxYRbp8yhJsXruR3b6wnNT6KorIq6hoayO6XwrgB3RiUlmAv3nVgeTsPkt0/BRH7MzTOsuTikB4J0Zw2qDuvrd7DTWcM4trHPyN/VxkzT8uk+FAVG4rKeXddMQ0KYQJZfZL49fkjOG1QjxbPXVVbzx/f2sDovsl858S+39g3NSudUX2TmL/cNf1acqwraT372Q4AusVFctUpGXx/YiY9E+23346k7EgtW0orudTrz9wYJ1hycdBFY/rwvy+t5sJHl1FcVs3ca8ZxTlb60f0V1XV8uf0gn2/bz6t5u5kx/zN+dd5wbjj9BESEzaUVzFmyidjIcO799kgiwl1POecv38qesioevnLsMa0QEeGZG8ezr6KGXkkxxEaFo6ps3VvJysIDvL++hH8s3cz8ZVu5bFxfbj5zEJk94oP6vZjjs2aXawLxbOvMNyHAkouDpo3sxW9e/oqSQ9XMu3bc0RcsGyVER3D6kB6cPqQHN545kP95Po8H/ruOvJ1lRIYJL3+5i8jwMKrrGjhcU8+frsjmwOEa5n6wmbNHpDNhkO8lcJJiIknyeMwmIgxMS2BgWgJX5PRn295K5i3bwosrd/Lvz3dw4Zg+3HrWIIb38p7Y+pvq6hvYUFxOeVUddfVKXUMDWX2SrAUUJF+6O/PH9LXkYpxnycVByXGRPDr9JNKTYlocOpoQHcHca07ibx9s5k9vbyA6IowfnDGQm84cyHOfbedPbxcQHRFGWJhwpLaeX50//LjjyuwRz4PfGc1Pzx7C/OVbeerjQl7J2823s/twz0VZdE+IPlr3QGUNi1btYsWmvXy2dT/lXssJxEeFc8c5Q7nutEwiw238SCDl7TjICT3iSY5rW/+cMYEgNoEi5OTkaG5urtNhtNr6okN0j48mLfHrf+T/9NYGHl2yCYBrJwzgvotH+e16Bw/XMH/5Vv6xdAvx0eHM/vZIJg1N41/LtvL4R1uprKnnhB7xTBjUnfEDu9MjIYqIsDDqG5R5H25myYZShvdK5LffGWUv9wXQ+AffY/zAVP7vqhOdDsV0ESKyUlVzfO2zlksH5Ovx1M+nDqVelVfzdvOTKUN8HHX8UuKi+PnUYXw7uw+/eHE1P3nuS6LCw6ipb+CC0b25fcoQhvXyPQnn+IGpvL22mHtfyeeKv3/M7VOG8ONvDSHcRqT5VVFZFUWHqqy/xYQMa7nQ8VouzVHVgA5DrW9QFn68jQ3F5Vw7IZMRvZvvh2lUUV3H3S+vYdGqXZx6Qir/d9VYeifHBizOruat/CJuXriSl245jXEDbDZkExzWculCAv1+Q3iYMHPiCW0+LiE6goe/O5aJg3sw6z9rOOfhDzk5sxsnZXRj3IBujB/Y3d6vaYfVOw8SESaM7NO6ZG9MoFlyMUF12bh+nJiRwrwPt7Cy8ABLNrhW5Tx3ZC/+76qxxETaTL7HI29HGcN7J9r3Z0KGJRcTdAPTEvj9ZWMA14t/z322nd+/uZ5r/vUp/7w2h27xUQ5H2LE0NCh5Ow9yUXYfp0Mx5qigjQ0VkVQRWSwilSJSKCLTm6l7h4gUiUiZiDwmItEe+24TkVwRqRaRBV7HjReRd0Rkv4iUisgLItI7gLdl2ik5NpKbJw1izvSTWL2rjMv+voId+w87HVaHsm1fJeVVdWT3s8kqTegI5osHc4AaIB24GpgrIiO9K4nINOBOYAqQCQwE7vWosht4AHjMxzW6AfPcxw0AyoHH/XUDJnDOH92bp244lb3l1Vz72GccPFzjdEgdxoYi1+SnWb0tuZjQEZTkIiLxwGXALFWtUNXlwCvADB/VrwPmq2q+qh4A7gdmNu5U1UWq+jKwz/tAVX1DVV9Q1UOqehh4FJjo/zsygXDKCak8NvNkdh04ws0LV1JTZzM3t0ZBcQUiMLhngtOhGHNUsFouQ4F6VS3wKMsDjmm5uMvyvOqli4jvuUyadyaQ72uHiNzkfryWW1paehynNoGQk5nKQ5eP4dOt+/nN4q8Cut5NZ1FQUk7/bnHERllnvgkdwerQTwDKvMrKAF9v3nnXbdxOxEdrpSkiMga4G7jY135VnYfrERo5OTn2L1gIueTEvmzZW8kj720kPjqCa8YPYFBavE0j34RNxRUMsVaLCTHBSi4VgPcA/CRcfSIt1W3c9lXXJxEZDLwB/ERVl7UhThMi7jh7CHsOHmHBim0sWLGNjNQ4pmalc+OZA0m3hbCOqq1vYMveCs4a3rPlysYEUbAeixUAESLiOS9JNr4fWeW793nWK1bVVrVaRGQA8C5wv6ouPM54jcNEhD9ekc3yX57F/ZeMYnDPBJ74eBuT/riEh95cT9mRWr7aWcbv31jP2Q8v5U9vbeiSj9AK91VSW68MTbeWiwktQWm5qGqliCwC7hORHwBjcT2uOs1H9SeBBSLyNLAHuAtY0LhTRCJwxR0OhItIDFCnqnUi0hd4H5ijqn8P5D2Z4OjXLY4Z4wcwY/wAtu87zJ/f2cDfPtjMvA+3UNegRIQJg3sm8OiSTVTX1fPr80d0qcdnBcUVAAxN9z23mzFOCeZLlLfiGj5cgqvv5BZVzReRDGAtkKWq21X1TRF5CFgCxAIvAfd4nOcur5+vwTVUeTbwA1xDl+8RkaN1VNV+resEMrrH8derTuTGMwbyQu4OsvokMTWrFylxkdz76lr+uWwrDQp3XdB1EkxBcTkiMCjN/oqb0GITV9K5Jq7sqlSV+15by+MfbePisX24fcqQLvEP7o+e/oI1u8tY+ouznA7FdEE2caXp9ESEuy/MIiE6gn8s3cJ/vtzNWcPSuPrUAYzul0zPxOhO2ZopKC63kWImJFlyMZ2GiPDzqcO4dkImT39ayFOfFPKDJ10t0qSYCIb3SuLsrJ5cMKYPfVM6/nT/tfUNbN1bydlZ6U6HYswxLLmYTictMZqfnj2UWyYPYmXhATYWV1BQXM7qnWU8+Pp6Hnx9PTkDunHpSf24KLs3iTEdc1ngbXsrqWuwkWImNFlyMZ1WdEQ4pw3qwWmDehwtK9xXyWur9/CfL3fx68Vfcf9ra7lgTG/OHdmLYb0S6ZsS22HWlWkcKTakp40UM6HHkovpUgZ0j+dHZw3m1smDyNtZxr8/384rX+7mxZU7AYiLCmdU32QuGduXC8b0Jjn261ZNbX0DkeHBnOu1eY0jxWxOMROKLLmYLklEGNs/hbH9U7j7wpGs3XOIguJyNhSVs3zTXn69+CvufTWfiYN7cOhILdv2HWZvRTXnjerFg98ZHRJrzmwsKScjNc4WCDMhyZKL6fJio8IZN6Db0bXnVZXVO8t46YudLN+4l7TEaKYM70lsVDhPf1rIF9sP8PCVY8nqncSiVbv49+fbKSmv5vKT+nHthEwyuscFJe6C4gp7JGZCliUXY7yICNn9U8jun3LMvsvH9eMnz63i6n99SlR4GDX1DWT3T2HCwAQWrNjG/I+2cuaQNE4b1J3s/imM7ptMfLT//zerqWtg295KptpIMROiLLkY0waj+ibz2o/PYM6STVTW1HFlTn9G9HbNrVpUVsUznxby8pe7WVrgWsYhzN0nMrpvCmP6JTOqbxLDeiWR0M6Es/XoSDFruZjQZMnFmDaKjQrnf6YNO6a8V3IMP5s6jJ9NHca+impW7yxj1Y6DrNlVxtKCEl76YufRuhmpcYzqm0TOgFROOSGVEb2TCG/DKLV1ew4BMMSGIZsQZcnFmADonhDNWcN7Hp0KX1XZU1bFuj2H3J9yvtxxkNe/KgIgNT6K2781mKvHD2hxRFrhvkoe+O9a+qbE2kgxE7IsuRgTBCJCn5RY+qTEMmXE1/0kuw8e4fNt+3k+dwezX13L059u5+6LsjhjSJrP85SWVzNj/mfUNSjPXX8K0RE2UsyEptAZtG9MF9QnJZaLx/blqRtOZd6McVTXNTBj/mfMmP8pX2w/8I265VW1zHz8M0rLq3l85snWajEhzWZFxmZFNqGjuq6eJ1cUMnfpZvZX1nDWsDT6p8axZlcZ6/aUU1PfwL+uy+GsYbbypHFec7MiW3LBkosJPZXVdTzx8TbXomj1SlbvJEb2TWLayF6MH9jd6fCMAWzKfWM6nPjoCG6dPJibzxyEQIeZ78yYRkHrcxGRVBFZLCKVIlIoItObqXuHiBSJSJmIPCYi0R77bhORXBGpFpEFPo6dIiLrReSwiCwRkQEBuiVjAi48TCyxmA4pmB36c4AaIB24GpgrIiO9K4nINOBOYAqQiWvZ4ns9quwGHsC1ZLL3sT2ARcAsIBXIBf7tz5swxhjTsqAkFxGJBy4DZqlqhaouB14BZviofh0wX1XzVfUAcD8ws3Gnqi5S1ZeBfT6OvRTIV9UXVLUKmA1ki8hwv96QMcaYZgWr5TIUqFfVAo+yPOCYlou7LM+rXrqItKYX8xvHqmolsLmJ6xhjjAmQYCWXBKDMq6wM8DUxknfdxu3WTKLU6uuIyE3uvpvc0tLSVpzaGGNMawUruVQASV5lSUB5K+o2bvuqe9zXUdV5qpqjqjlpab7fhjbGGHN8gpVcCoAIERniUZYN5Puom+/e51mvWFV99bE0e6y7r2dQE9cxxhgTIEFJLu6+j0XAfSISLyITgYuBhT6qPwncICJZItINuAtY0LhTRCJEJAYIB8JFJEZEGt/XWQyMEpHL3HXuBlar6vqA3ZwxxphjBO0NfRFJxTV8+BxcI73uVNVnRCQDWAtkqep2d92fAb8EYoGXgB+qarV732zgHq/T36uqs937zwYeBQYAnwIzVXVbC7GVAoXuH5P5Zr9N48+e5d5lPYC9rfsmfF6jpX1NxdTUdrDjay4mX3H5Kuvq32Fz8fmKy1eZfYf2HQY7vgGq6rtfQVXt4/EB5vn62bPcuwzIbc81WtrXVEytiCso8TUXk32H7Y/PvkP7DkM1vuY+NivysV5t4udXWyhrzzVa2tdUTE1tBzu+5mJqKh77Dpsvs+/QvkNf/22rQMfXJJu40g9EJFebmLwtFIR6fBD6MYZ6fBD6MYZ6fBD6MYZ6fJ6s5eIf85wOoAWhHh+EfoyhHh+EfoyhHh+EfoyhHt9R1nIxxhjjd9ZyMcYY43eWXIwxxvidJZcgEJHTReQD96dARP7idEy+iMhkEXnPvQ7Od5yOx5OIZIpIqcf3GJJz9ojI99zvTYUcEUkXkRUislRE3heR3k7H5E1EJojIx+4YnxWRSKdj8iQiySLymYhUiMgop+NpJCK/FZFlIvKiiMQ5HQ9YcgkKVV2uqpNVdTKwAnjZ4ZCO4Z7R4OfAeap6lqoudjomH5Y2fo+qGnL/gItIGHA5sMPpWJqwFzhdVSfhngnD4Xh8KQS+5Y5xC66ZPELJYeAC4EWnA2nkTnKDVPUM4F3geodDAiy5BJX7t7BTgGVOx+LDacAR4FX3iqG9nA7Ih4nu384eFJFQXJ5xOq5/dBqcDsQXVa1X1cbYEgnBOfdUdbeqHnH/WEeIfZeqWhuCv9icAbzh3n4DON3BWI6y5OKluWWU27JUcxPOAd7z+B88lGJMBwYDFwH/xLXQWijFt8cd35lAT1wLw4VMfCISDlyJn1Y+DdTfQxEZKyKfArcBX4RijO7jTwDOA14LxfgCoR3xduPraVnKcK3C67iIlqt0OY3LKE/DNbeZJ8+lmscC/xWRPFXNd/+m76upfLmqFrm3rwAeD8UYgYPAR6paIyLv4VpqOmTic3+HjfPLLQLG45p3LiTic5/reVVt8FOjKiDfoap+CZwqIlcCvwJ+GGoxikgS8AQwQ1VrQi2+dsQTkHiBA7jm+sL93/0BjLH12jJPTVf64PpDXuDxczyuP9yhHmULgd+38nyRwBogLBRjBLrjel4rwKnA4yEWX5LH9u+Aa0Msvj8AbwNv4vrt8ZEQ/DOO9tieBjwcgjFGAP/F1e/S7tj8HZ9H/QXAKH/F2J54gdHAM+7tm4AfByKutn6s5dJ6TS3VPKmVx58NvK/tfCTWguOOUVX3ichiYCmu59yB6BRsz3c4SVwzYh8GtgKz/B9eu76/XzZui2uKjtsDEB+07zs8SUT+ANQDVQSu47c9MX4P1y83d4vI3cBcVfXLo0Y/xYeIvI6r9TBMRP6hqgv8HJ+3ZuNV1a/cj8qWASXAtQGOp1UsubReW5ZqPoaqvsHXnW6B0t4Y5+BqfgfKccenqq9y/JP3tVa7vr9GGti5n9rzHX6Mq88q0NoT40J8r/PkT+39/+R8v0fUvBbjVdVfBTWiVrAO/dZry1LNTgn1GC2+9rMY2y/U4/PW0eIFLLm0RVuWanZKqMdo8bWfxdh+oR6ft44WL2DJ5RjSxDLK2ralmrt0jBafxRgKMYZ6fB093hY5PaIg1D643u9Qr89s975UXG/XVwLbgekWo8VnMYZmjKEeX0ePt6WPTblvjDHG7+yxmDHGGL+z5GKMMcbvLLkYY4zxO0suxhhj/M6SizHGGL+z5GKMMcbvLLkYY4zxO0suxjhMRM4QkQ1Ox2GMP1lyMV2aiGwTkbOdjEFVl6nqsECcW0Q+EJEqEakQkb0iskhEerfy2MkisjMQcZnOz5KLMQEmriWQnXSbqibgWiY6AfiTw/GYLsCSizE+iEiYiNwpIptFZJ+IPC8iqR77XxCRIhEpE5EPRWSkx74FIjJXRF4XkUrgLHcL6X9EZLX7mH+7Jyk8poXQXF33/v8VkT0isltEfiAiKiKDW7onVT2Ia36qsR7n+r6IrBORchHZIiI3u8vjca0/1Mfd6qkQkT4tfS/GNLLkYoxvtwOX4Frtrw+udco9F1J7AxgC9AS+AJ72On468FtcCzotd5ddCZwLnACMAWY2c32fdUXkXOBnuFY2HUzrV0JFRLoDlwKbPIpLgAtxrQ/yfeAvInKSumbiPQ/YraoJ7s9uWv5ejAEsuRjTlJuB36jqTlWtxjVj7eUiEgGgqo+parnHvmwRSfY4/j+q+pGqNqhqlbvsEVXdrar7ca2qOZamNVX3SuBxVc1X1cPAva24l0dEpAzYC/QAfty4Q1X/q6qb1WUp8DZwxvF+L8Y0suRijG8DgMUiclBEDgLrcK09ny4i4SLye/ejoUPANvcxPTyO3+HjnEUe24dx9X80pam6fbzO7es63m5X1WRcLaBuQL/GHSJynoh8IiL73fd5Pt+8D29Nfi+tiMN0IZZcjPFtB3CeqqZ4fGJUdReuR14X43o0lQxkuo8Rj+MDtZbFHjySA9C/tQeq6lfAA8AccYkGXsLVwZ+uqinA63x9H77uobnvxZijLLkYA5HuVf8aPxHA34HfisgAABFJE5GL3fUTgWpgHxAHPBjEWJ8Hvi8iI0QkDri7jcc/gauf6NtAFBANlAJ1InIeMNWjbjHQ3etxX3PfizFHWXIxxvXb+hGPz2zgr8ArwNsiUg58Apzqrv8kUAjsAta69wWFqr4BPAIswdUx/7F7V3Urj69xHz9LVctxddA/j6tjfjque26sux54FtjifgzWh+a/F2OOspUojenARGQEsAaIVtU6p+MxppG1XIzpYETkOyISJSLdgD8Ar1piMaHGkosxHc/NuPpJNuMaqXWLs+EYcyx7LGaMMcbvrOVijDHG7yy5GGOM8TtLLsYYY/zOkosxxhi/s+RijDHG7yy5GGOM8bv/D8t0JrYYNDTRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dset = list(zip(train_x,train_y))\n",
    "valid_dset = list(zip(valid_x,valid_y))\n",
    "\n",
    "dl = DataLoader(dset, batch_size=256)\n",
    "valid_dl = DataLoader(valid_dset, batch_size=256)\n",
    "dls = DataLoaders(dl, valid_dl)\n",
    "\n",
    "w1 = init_params((28*28,30))\n",
    "b1 = init_params(30)\n",
    "w2 = init_params((30,10))\n",
    "b2 = init_params(10)\n",
    "params = w1,b1,w2,b2\n",
    "\n",
    "learn = Learner(dls, simple_net, opt_func=SGD,\n",
    "                loss_func=mnist_loss, metrics=batch_accuracy)\n",
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>batch_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.065584</td>\n",
       "      <td>0.100056</td>\n",
       "      <td>0.509000</td>\n",
       "      <td>00:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.065126</td>\n",
       "      <td>0.097372</td>\n",
       "      <td>0.519900</td>\n",
       "      <td>00:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.065000</td>\n",
       "      <td>0.095899</td>\n",
       "      <td>0.526600</td>\n",
       "      <td>00:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.064967</td>\n",
       "      <td>0.094701</td>\n",
       "      <td>0.531600</td>\n",
       "      <td>00:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.065005</td>\n",
       "      <td>0.093518</td>\n",
       "      <td>0.538300</td>\n",
       "      <td>00:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.065129</td>\n",
       "      <td>0.092425</td>\n",
       "      <td>0.543700</td>\n",
       "      <td>00:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.065273</td>\n",
       "      <td>0.091485</td>\n",
       "      <td>0.547700</td>\n",
       "      <td>00:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.065440</td>\n",
       "      <td>0.090671</td>\n",
       "      <td>0.550500</td>\n",
       "      <td>00:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.065580</td>\n",
       "      <td>0.089871</td>\n",
       "      <td>0.554600</td>\n",
       "      <td>00:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.065689</td>\n",
       "      <td>0.089082</td>\n",
       "      <td>0.558200</td>\n",
       "      <td>00:01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit(10,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Truth: 0  Detected: 0 Filename: 1001.png\n",
      "Truth: 1  Detected: 1 Filename: 1004.png\n",
      "Truth: 2  Detected: 2 Filename: 1002.png\n",
      "Truth: 3  Detected: 3 Filename: 1020.png\n",
      "Truth: 4  Detected: 9 Filename: 1010.png\n",
      "Truth: 5  Detected: 3 Filename: 1003.png\n",
      "Truth: 6  Detected: 6 Filename: 100.png\n",
      "Truth: 7  Detected: 7 Filename: 0.png\n",
      "Truth: 8  Detected: 9 Filename: 1007.png\n",
      "Truth: 9  Detected: 9 Filename: 1000.png\n"
     ]
    }
   ],
   "source": [
    "for i in range (10):\n",
    "\n",
    "    f=pick_a_file(i);\n",
    "    targetTensors = tensor(Image.open(f)).float()/255\n",
    "    targetTensorsv=targetTensors.view(-1,28*28);\n",
    "    learn.model.eval()\n",
    "    res=learn.model(targetTensorsv)\n",
    "    head, tail = os.path.split(f)\n",
    "    #print(torch.argmax(res))\n",
    "    buf = \"%s\" % torch.argmax(res);\n",
    "    print(\"Truth:\", head[-1],' Detected:', buf[7], \"Filename:\", tail)      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### resnet-18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dlsEasy = ImageDataLoaders.from_folder(path, train=\"training\", valid=\"testing\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learnEasy = cnn_learner(dlsEasy, resnet18, pretrained=False,\n",
    "               loss_func=F.cross_entropy, metrics=accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learnEasy.fit_one_cycle(1, 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range (10):\n",
    "    f=pick_a_file(i);\n",
    "    targetTensors = tensor(Image.open(f)).float()/255\n",
    "    targetTensorsRGB=targetTensors.repeat(3, 1, 1)\n",
    "    learnEasy.model.eval()\n",
    "    res=learnEasy.model(targetTensorsRGB.view(1,3,28,28).cuda())\n",
    "    head, tail = os.path.split(f)\n",
    "    #print(torch.argmax(res))\n",
    "    buf = \"%s\" % torch.argmax(res);\n",
    "    print(\"Truth:\", head[-1],' Detected:', buf[7], \"Filename:\", tail)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Jargon Recap"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Questionnaire"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. How is a grayscale image represented on a computer? How about a color image?\n",
    "1. How are the files and folders in the `MNIST_SAMPLE` dataset structured? Why?\n",
    "1. Explain how the \"pixel similarity\" approach to classifying digits works.\n",
    "1. What is a list comprehension? Create one now that selects odd numbers from a list and doubles them.\n",
    "1. What is a \"rank-3 tensor\"?\n",
    "1. What is the difference between tensor rank and shape? How do you get the rank from the shape?\n",
    "1. What are RMSE and L1 norm?\n",
    "1. How can you apply a calculation on thousands of numbers at once, many thousands of times faster than a Python loop?\n",
    "1. Create a 3×3 tensor or array containing the numbers from 1 to 9. Double it. Select the bottom-right four numbers.\n",
    "1. What is broadcasting?\n",
    "1. Are metrics generally calculated using the training set, or the validation set? Why?\n",
    "1. What is SGD?\n",
    "1. Why does SGD use mini-batches?\n",
    "1. What are the seven steps in SGD for machine learning?\n",
    "1. How do we initialize the weights in a model?\n",
    "1. What is \"loss\"?\n",
    "1. Why can't we always use a high learning rate?\n",
    "1. What is a \"gradient\"?\n",
    "1. Do you need to know how to calculate gradients yourself?\n",
    "1. Why can't we use accuracy as a loss function?\n",
    "1. Draw the sigmoid function. What is special about its shape?\n",
    "1. What is the difference between a loss function and a metric?\n",
    "1. What is the function to calculate new weights using a learning rate?\n",
    "1. What does the `DataLoader` class do?\n",
    "1. Write pseudocode showing the basic steps taken in each epoch for SGD.\n",
    "1. Create a function that, if passed two arguments `[1,2,3,4]` and `'abcd'`, returns `[(1, 'a'), (2, 'b'), (3, 'c'), (4, 'd')]`. What is special about that output data structure?\n",
    "1. What does `view` do in PyTorch?\n",
    "1. What are the \"bias\" parameters in a neural network? Why do we need them?\n",
    "1. What does the `@` operator do in Python?\n",
    "1. What does the `backward` method do?\n",
    "1. Why do we have to zero the gradients?\n",
    "1. What information do we have to pass to `Learner`?\n",
    "1. Show Python or pseudocode for the basic steps of a training loop.\n",
    "1. What is \"ReLU\"? Draw a plot of it for values from `-2` to `+2`.\n",
    "1. What is an \"activation function\"?\n",
    "1. What's the difference between `F.relu` and `nn.ReLU`?\n",
    "1. The universal approximation theorem shows that any function can be approximated as closely as needed using just one nonlinearity. So why do we normally use more?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Further Research"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Create your own implementation of `Learner` from scratch, based on the training loop shown in this chapter.\n",
    "1. Complete all the steps in this chapter using the full MNIST datasets (that is, for all digits, not just 3s and 7s). This is a significant project and will take you quite a bit of time to complete! You'll need to do some of your own research to figure out how to overcome some obstacles you'll meet on the way."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "jupytext": {
   "split_at_heading": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}