Exp3A-Simple discriminator-Pure Tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Exp 3.A: a simple classifier with the barcode dataset\n",
    "### Pure reimplementation of exp3\n",
    "### Fuck you sugartensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "from __future__ import absolute_import\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import sugartensor as tf\n",
    "import os\n",
    "from IPython.display import display, Image\n",
    "import numpy as np\n",
    "from scipy import ndimage\n",
    "from six.moves import cPickle as pickle\n",
    "\n",
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   # see issue #152\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"3\"\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "batch_size = 1   # batch size\n",
    "image_size = 64\n",
    "pixel_depth = 255.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.46735139974\n",
      "1500\n",
      "1.0\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG0BJREFUeJzt3X+QVeWd5/H3ZxrBOPkBqMsSwDSJTazOFqKyhKyWk6BB\nxJQ4NUwGKjPByBbZWZwyNbMbIfnDiQm1urUTk9RmzGaUBLMZkcVY9hg2DiLW1lgj0oyAAiItkgK2\nFbTBJGPJbOt3/zjPNdfmXvr2/d3Hz6vqVp/znOec8z23i28fnnOe51FEYGZmo9/vtDoAMzOrDyd0\nM7OccEI3M8sJJ3Qzs5xwQjczywkndDOznHBCNzPLCSd0M7OccEI3M8uJMa0OwGy0O++886Kzs7PV\nYViO7dix49WIOH+4ek7oZjXq7Oykt7e31WFYjkn6ZSX13ORiZpYTTuhmZjnhhG5mlhNO6GZmOeGE\nbmaWE07oZmY54YRuZpYTTuhmZjnhhG65IqlD0jOSHknr0yVtk9Qn6QFJY1P5uLTel7Z3Fh1jdSrf\nL+ma1lyJ2ci5p6jlzS3APuCDaf1O4K6IWC/pB8By4O7080REXChpSar3R5K6gSXAJ4APA49JmhER\nb1UTTOeqn5fdduiO66o5pFlZvkO33JA0FbgOuCetC5gHbExV1gE3pOVFaZ20/apUfxGwPiJORcRL\nQB8wpzlXYFYbJ3TLk+8AXwXeTuvnAicjYjCtHwGmpOUpwGGAtP31VP+d8hL7vEPSCkm9knqPHz9e\n7+swq4oTuuWCpM8BxyJiRzPOFxE/jIjZETH7/POHHQTPrCnchm55cTlwvaSFwNlkbejfBcZLGpPu\nwqcCR1P9o8A04IikMcCHgNeKyguK9zFra75Dt1yIiNURMTUiOskeaj4eEV8AtgKLU7VlwMNpuSet\nk7Y/HhGRypekt2CmA13A0026DLOa+A7d8u5WYL2kbwHPAPem8nuBn0jqAwbI/ggQEXskbQD2AoPA\nymrfcDFrNid0y52IeAJ4Ii0fpMRbKhHxJvCHZfZfA6xpXIRmjeEmFzOznHBCNzPLCSd0M7OccEI3\nM8sJJ3Qzs5xwQjczywkndDOznHBCNzPLCSd0M7OccEI3M8sJJ3Qzs5xwQjczy4maErqkBWki3T5J\nq+oVlJmZjVzVCV1SB/B94FqgG1iaJtg1M7MWqGX43DlAXxqeFEnrySbY3Vtuh/POOy86OztrOKVZ\neYcOHeLVV19Vq+Mwa5VaEnqpyXQ/ObSSpBXACoALLriA3t7eGk5pVt7s2bNbHYJZSzX8oagn07Vm\nePPNN9m3bx+SdknaI+kbAJJ+LOklSTvTZ1Yql6Tvpec/uyVdWjiWpGWSDqTPsnLnNGs3tSR0T6Zr\nbWPcuHHMmDGDiLgYmAUskDQ3bf7PETErfXamsmvJ5gvtIvsf5N0AkiYCt5H9b3MOcJukCc28FrNq\n1ZLQtwNdkqZLGks2J2NPfcIyGxlJdHR0FFbPSp84wy6LgPsi8xQwXtJk4Bpgc0QMRMQJYDOwoIGh\nm9VN1Qk9IgaBm4FHgX3AhojYU6/AzEYqIpC0EzhGlpS3pU1rUrPKXZLGpbJSz4CmnKHcrO3V1IYe\nEZsiYkZEfCxNrGvWMpKIiFlkzX9zJP0bYDVwEfBvgYnArXU61wpJvZJ6jx8/Xo9DmtXMPUUtdyLi\nJLAVWBAR/alZ5RTwI7J2cSj/DKiiZ0N+2G/tyAndcuH48eMMDg4CIOl9wGeB51O7OJIE3AA8l3bp\nAb6Y3naZC7weEf1kTYjzJU1ID0PnpzKztlfLe+hmbaO/v58XXngBSbvJblQ2RMQjkh6XdD4gYCfw\nH9Ium4CFQB/wBvAlgIgYkPRNsof+ALdHxEAzr8WsWk7olgszZ86ku7ub3t7emcXlETGvVP2ICGBl\nmW1rgbX1j9KssdzkYmaWE07oZmY54YRuZpYTTuhmZjnhhG5mlhNO6GZmOeGEbmaWE07oZmY54YRu\nZpYTTuhmZjkxbEKXtFbSMUnPFZVNlLQ5TdG12TO6mJm1XiV36D/m9BlbVgFbIqIL2JLWzcyshYZN\n6BHxf4Cho80tAtal5XVkw5KamVkLVduGPimNHQ3wMjCpTvGYmVmVan4omoYhLTsZr6fqMjNrjmoT\n+itFM8FMJpuUtyRP1WVm1hzVJvQeYFlaXgY8XJ9wzMysWpW8tng/8I/AxyUdkbQcuAP4rKQDwNVp\n3axl3nzzTfbt24ekXZL2SPoGgKTpkrZJ6pP0gKSxqXxcWu9L2zsLx5K0OpXvl3RNa67IbOQqectl\naURMjoizImJqRNwbEa9FxFUR0RURV3vORWu1cePGMWPGDCLiYmAWsCBN/nwncFdEXAicAJanXZYD\nJ1L5XakekrqBJcAnyF7X/WtJHc29GrPquKeo5YIkOjreybtnpU8A84CNqbz4FdviV283AldJUipf\nHxGnIuIlskmk5zT+Csxq54RuuRERSNpJ9pB+M/AicDIiBlOVI8CUtDwFOJz2GwReB84tLi+xzzv8\n9pa1Iyd0yw1JRMQsYCrZXfVFjTqX396yduSEbrkTESeBrcCngPGSxqRNU4GjafkoMA0gbf8Q8Fpx\neYl9zNqaE7rlwvHjxxkczFpWJL0P+CywjyyxL07Vil+xLX71djHweOok1wMsSW/BTAe6gKebchFm\nNRozfBWz9tff388LL7yApN1kNyobIuIRSXuB9ZK+BTwD3Jt2uRf4iaQ+srGKlgBExB5JG4C9wCCw\nMiLeavb1mFXDCd1yYebMmXR3d9Pb2zuzuDwiDlLiLZWIeBP4w1LHiog1wJqGBGrWQG5yMTPLCSd0\nM7OccEI3M8sJJ3Qzs5xwQjczywkn9DYiiWw4ETOzkXNCNzPLCSd0M7OcqGSCi2mStkramyYOuCWV\nT5S0WdKB9HNC48M1M7NyKrlDHwT+IiK6gbnAyjQJwCpgS0R0AVvSupmZtUglMxb1R8Q/peVfkw14\nNIV3TxBQPHGAmZm1wIja0NO8i5cA24BJEdGfNr0MTKprZGZmNiIVJ3RJ7wceBL4SEb8q3paGHY0y\n+3lmFzOzJqgooUs6iyyZ/zQifpaKX5E0OW2fTDbt12k8s4uZWXNU8paLyMaO3hcR3y7aVDxBQPHE\nAWZm1gKVjId+OfAnwLNpAl6ArwF3ABskLQd+CXy+MSGamVklKnnL5R8iQhExMyJmpc+miHgtIq6K\niK6IuDoiBpoRsFkphw8fZv/+/ZToL/GXko5K2pk+Cwv7SFotqU/SfknXFJUvSGV9kvw6ro0anrHI\ncmHMmDFMmzaNvXv3dkv6ALBD0ua0+a6I+G/F9VNfiiXAJ4APA49JmpE2f59sTtIjwHZJPRGxtzlX\nYlY9J3TLhcmTJ3POOecAWX8JSYX+EuUsAtZHxCngpTS3aGGqur40dR2S1qe6TujW9jyWi+XOkP4S\nADdL2i1pbdEQFVOAw0W7HUll5crN2p4TuuVKif4SdwMfA2YB/cBf1ek87l9hbccJ3XIj69/27v4S\nEfFKRLwVEW8Df8Nvm1WOAtOKdp+aysqVDz2X+1dY23FCt1yICA4dOgRD+ksUOr8lvw88l5Z7gCWS\nxkmaDnQBTwPbgS5J0yWNJXtw2tOESzCrmR+KWi48+eSTDAwMAMwb0l9iqaRZZENTHAK+DBAReyRt\nIHvYOQisjIi3ACTdDDwKdABrI2JPM6/FrFpO6JYLV1xxBZdddhm9vb0zh2zaVG6fiFgDrClRvulM\n+5m1Kze5mJnlhBO6mVlOOKGbmeWEE7qZWU44oZuZ5YQTuplZTjihm5nlRCUzFp0t6WlJu9I4099I\n5dMlbUtjRj+QetWZmVmLVHKHfgqYFxEXkw1wtEDSXOBOsnGmLwROAMsbF6aZmQ2nkhmLIiJ+k1bP\nSp8A5gEbU/k64IaGRGhmZhWpqA1dUkcaH+MYsBl4ETgZEYOpiseMNjNrsYoSehp+dBbZUKJzgIsq\nPYHHjTYza44RveUSESeBrcCngPGSCoN7lRwzOu3jcaPNzJqgkrdczpc0Pi2/j2zy3H1kiX1xqrYM\neLhRQZqZ2fAqGT53MrBOUgfZH4ANEfGIpL3AeknfAp4B7m1gnGZmNoxhE3pE7CabcHdo+UF+O52X\nmZm1mHuKmpnlhBO65cLhw4fZv38/kvamHs23AEiaKGmzpAPp54RULknfSz2dd0u6tHAsSctS/QOS\nlrXqmsxGygndcmHMmDFMmzaNiOgG5gIrJXUDq4AtEdEFbEnrANeSTQzdBawA7obsDwBwG/BJsibF\n2wp/BMzanRO65cLkyZM555xzAIiIX5O9iTUFWETWkxne3aN5EXBf6gn9FNlruJOBa4DNETEQESfI\nOtItaN6VmFXPCd1yR1In2YP8bcCkiOhPm14GJqXlKcDhot0KvZ3LlZu1PSd0yxVJ7wceBL4SEb8q\n3hYRQTYOUT3O4x7Q1nac0C03snzNg8BPI+JnqfiV1JRC+nkslR8FphXtXujtXK586LncA9rajhO6\n5UJEcOjQIYB9EfHtok09ZD2Z4d09mnuAL6a3XeYCr6emmUeB+ZImpIeh81OZWdurpKeoWdt78skn\nGRgYAJiXRgYF+BpwB7BB0nLgl8Dn07ZNwEKgD3gD+BJARAxI+iawPdW7PSIGmnMVZrVxQrdcuOKK\nK7jsssvo7e2dWWLzVUMLUnv6ylLHioi1wNo6h2jWcG5yMTPLCSd0M7OccEI3M8sJJ3Qzs5xwQjcz\ny4mKE3qaKPoZSY+k9emStqXR6h6QNLZxYZqZ2XBGcod+C9mARwV3AndFxIXACWB5PQMzM7ORqSih\nS5oKXAfck9YFzAM2pirFo9iZmVkLVHqH/h3gq8Dbaf1c4GREDKb1siPSeRAjM7PmGDahS/occCwi\ndlRzAg9iZGbWHJV0/b8cuF7SQuBs4IPAd8kmBBiT7tJLjkhnZmbNM+wdekSsjoipEdEJLAEej4gv\nAFuBxala8Sh2ZmbWArW8h34r8OeS+sja1O+tT0hmZlaNEY22GBFPAE+k5YNkk+iamVkbcE9RM7Oc\ncEI3M8sJJ3TLhZtuuoldu3Yh6blCmaS/lHRU0s70WVi0bXUatmK/pGuKyheksj5Jq5p9HWa1cEK3\nXLjxxhvp6uoqtemuiJiVPpsAJHWTvbH1CWAB8NdprKIO4PvAtUA3sDTVNRsVPAWd5cKVV15JR0dH\npdUXAesj4hTwUnpTq/CAvy898EfS+lR3b73jNWsE36Fb3t0sabektZImpLIpwOGiOoWhK8qVm40K\nTuiWZ3cDHwNmAf3AX9XrwB6jyNqRE7rlVkS8EhFvRcTbwN/w22aVo8C0oqqFoSvKlZc6tscosrbj\nhG65JWly0ervA4U3YHqAJZLGSZoOdAFPA9uBrjR5y1iyB6c9zYzZrBZ+KGq5sHTpUp5//nmAj0s6\nAtwGfFrSLCCAQ8CXASJij6QNZA87B4GVEfEWgKSbgUeBDmBtROxp9rWYVcsJ3XLh/vvv58CBA/T2\n9p5VVFx2fKGIWAOsKVG+CdjUgBDNGs5NLmZmOeGEbmaWE07oZmY5UVEbuqRDwK+Bt4DBiJgtaSLw\nANBJ9sDp8xFxojFhmpnZcEZyh/6ZNB7G7LS+CtgSEV3AlrRuZmYtUkuTyyJgXVpeB9xQezhmZlat\nShN6AH8vaYekFalsUkT0p+WXgUl1j87MzCpW6XvoV0TEUUn/Ctgs6fnijRERkqLUjukPwAqACy64\noKZgzcysvIru0CPiaPp5DHiIbEyMVwpdq9PPY2X29ZgXZmZNMGxCl/S7kj5QWAbmk42J0QMsS9WW\nAQ83KkgzMxteJU0uk4CHJBXq/21E/ELSdmCDpOXAL4HPNy5MMzMbzrAJPc3ecnGJ8teAqxoRlJmZ\njZx7ipqZ5YQTuplZTjihm5nlhBO6mVlOOKGbmeWEE7rlwk033cSuXbuQVJg3FEkTJW2WdCD9nJDK\nJel7kvok7ZZ0adE+y1L9A5KWlTqXWbtyQrdcuPHGG+nq6hpaXG5E0GvJJobuIhuW4m7I/gCQzUX6\nSbLe0LcV/giYjQZO6JYLV155JR0dHUOLy40Iugi4LzJPAePT8BXXAJsjYiCN7b8ZWND46M3qwwnd\n8qzciKBTgMNF9Y6ksnLlZqOCE7q9J0REkA0DXReSVkjqldR7/Pjxeh3WrCZO6JZn5UYEPQpMK6o3\nNZWVKz+NRxG1duSEbnlWbkTQHuCL6W2XucDrqWnmUWC+pAnpYej8VGY2KlQ6wYVZW1u6dCnPP/88\nwMclHSF7W+UOSo8IuglYCPQBbwBfAoiIAUnfBLanerdHxEDzrsKsNk7olgv3338/Bw4coLe396wh\nm04bETS1p68sdZyIWAusbUCIZg3nJhczs5yoKKFLGi9po6TnJe2T9KlyvfDMzKw1Kr1D/y7wi4i4\niGyyi32U74VnZmYtUMmcoh8CrgTuBYiIf4mIk5TvhWdmZi1QyR36dOA48CNJz0i6J00WXa4XnpmZ\ntUAlCX0McClwd0RcAvwzQ5pXztQLzz3qzMyao5KEfgQ4EhHb0vpGsgRfrhfeu7hHnZlZcwyb0CPi\nZeCwpI+noquAvZTvhWdmZi1QaceiPwN+KmkscJCsZ93vULoXnpmZtUBFCT0idgKzS2w6rReemZm1\nhnuKmpnlhBO6mVlOOKGbmeVEU0db3LFjx6uS/hl4tZnnrbPzaHD8khp16IbH3mDDxf+RZgVi1o6a\nmtAj4nxJvRFR6gHrqDCa4x/NscPoj9+s0dzkYmaWE07oZmY50YqE/sMWnLOeRnP8ozl2qDJ+SYck\nPStpp6TeVFZyPP80z+j3JPVJ2i3p0npegFkjNT2hR8SoTiqjOf7RHDvUHP9nImJWURt8ufH8rwW6\n0mcFcHcN5zRrKje52HtVufH8FwH3ReYpYHxhEDqzdueEbu8FAfy9pB2SVqSycuP5TwEOF+17JJW9\ni4eFtnbU1IQuaYGk/al9sq2nrJM0TdJWSXsl7ZF0SyofNXOpSupIk5I8ktanS9qWvv8H0mBrbanO\n89heERGXkjWnrJR0ZfHGM43nX46HhbZ21LSELqkD+D7ZP6puYKmk7madvwqDwF9ERDcwlywRdDO6\n5lK9hWz+14I7gbsi4kLgBLC8JVFVpm7z2EbE0fTzGPAQMIfy4/kfBaYV7T41lZm1vWbeoc8B+iLi\nYET8C7CerL2yLUVEf0T8U1r+NVlCmcIomUtV0lTgOuCetC5gHtkEJdDesddtHltJvyvpA4VlYD7w\nHOXH8+8BvpjedpkLvF7UNGPW1prZU7RU2+Qnm3j+qknqBC4BtjF65lL9DvBV4ANp/VzgZEQMpvWS\nbcNtonge24uBHWT/26jmu58EPJSGUxgD/G1E/ELSdkqP578JWAj0AW+Qjf1vNio0tev/aCTp/cCD\nwFci4lfF46xEREgaUdtrM0j6HHAsInZI+nSr46lCYR7bP4uIbZK+S4l5bCv57iPiIFmTzdDy1ygx\nnn9qT19ZbeBmrdTMJpdR1zYp6SyyZP7TiPhZKq5oLtUWuxy4XtIhsqateWRt0uMlFf6It/P3X9M8\ntmbvVc1M6NuBrvSmxVhgCVl7ZVtKbc73Avsi4ttFm9p+LtWIWB0RUyOik+x7fjwivgBsBRanam0Z\nO3geW7NqNa3JJSIGJd0MPAp0AGsjYk+zzl+Fy4E/AZ6VtDOVfQ24g9E7l+qtwHpJ3wKeIT10bFOe\nx9ZshJo9fO4msodObS8i/gEoNzD5qJlLNSKeAJ5IywfJ3jZqe57H1mzk3FPUzCwnnNDNzHLCCd3M\nLCec0M3McsIJ3cwsJ5zQzcxywgndzCwnnNDNzHLCCd3MLCec0M3McsIJ3cwsJ5zQzcxywgndrITR\nNKG5WYFnLDIbomhC88+STbaxXVJPROxtbWRWb52rfl5226E7rmtiJPXhO3Sz042qCc3NCpzQzU5X\nakLzdp1Q2+wdbnIxq4KkFcCKtPobSfvLVD0PeLXkMe5sRGRnVDaWFmiXWEbL7+cjlRzACd3sdMNO\naB4RPwR+ONyBJPVGRKmZl5rOsbRvHFCfWNzkYna6UTWhuVmB79DNhhiFE5qbAU7oZiXVcULzYZtl\nmsixnK5d4oA6xKKIqEcgZmbWYm5DNzPLCSd0sxpJmihps6QD6eeEMvXekrQzfXqKyqdL2paGGXgg\nPYhtWCySZkn6R0l7JO2W9EdF234s6aWiOGeN8PxnHDJB0rh0jX3pmjuLtq1O5fslXTOyK68qlj+X\ntDd9B1skfaRoW8nfVYPiuFHS8aLz/fuibcvS7/KApGXDniwi/PHHnxo+wH8FVqXlVcCdZer9pkz5\nBmBJWv4B8KeNjAWYAXSl5Q8D/cD4tP5jYHGV5+4AXgQ+CowFdgHdQ+r8R+AHaXkJ8EBa7k71xwHT\n03E6avgeKonlM8A5aflPC7Gc6XfVoDhuBP57iX0nAgfTzwlpecKZzuc7dLPaLQLWpeV1wA2V7ihJ\nwDxgYzX7VxNLRLwQEQfS8v8FjgHn13DOgkqGTCiObyNwVfoOFgHrI+JURLwE9KXjNSyWiNgaEW+k\n1afI+hvUWy3DSFwDbI6IgYg4AWwGFpxpByd0s9pNioj+tPwyMKlMvbMl9Up6SlIh0Z4LnIyIwbRe\n6zADlcYCgKQ5ZHeOLxYVr0nNEHdJGjeCc1cyZMI7ddI1v072HdR7uIWRHm858L+L1kv9rhoZxx+k\n73yjpEKnthF/J35t0awCkh4D/nWJTV8vXomIkFTu1bGPRMRRSR8FHpf0LFlCa0UsSJoM/ARYFhFv\np+LVZH8IxpK9RncrcPtIYxxNJP0xMBv4vaLi035XEfFi6SPU7O+A+yPilKQvk/0PZl41B3JCN6tA\nRFxdbpukVyRNjoj+lCSPlTnG0fTzoKQngEuAB4HxksakO9bThhloRCySPgj8HPh6RDxVdOzC3f0p\nST8C/tOZYhli2CETiuockTQG+BDwWoX7jkRFx5N0Ndkfwt+LiFOF8jK/q2oSeiXDSLxWtHoP2XOQ\nwr6fHrLvE2c6mZtczGrXAxTeQFgGPDy0gqQJheYLSecBlwN7I3v6tRVYfKb96xzLWOAh4L6I2Dhk\n2+T0U2Tt78+N4NyVDJlQHN9i4PH0HfQAS9JbMNOBLuDpEZx7xLFIugT4H8D1EXGsqLzk76qBcUwu\nWr0e2JeWHwXmp3gmAPNTWXn1eJLrjz/v5Q9ZG/AW4ADwGDAxlc8G7knL/w54luwth2eB5UX7f5Qs\nefUB/wsY1+BY/hj4f8DOos+stO3xFN9zwP8E3j/C8y8EXiC7m/16KrudLGkCnJ2usS9d80eL9v16\n2m8/cG0dfi/DxfIY8ErRd9Az3O+qQXH8F2BPOt9W4KKifW9K31Uf8KXhzuWeomZmOeEmFzOznHBC\nNzPLCSd0M7OccEI3M8sJJ3Qzs5xwQjczywkndDOznHBCNzPLif8P4g/0ye8IYwMAAAAASUVORK5C\nYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fa1ce066518>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dir= \"barcode/train/\"\n",
    "fn = os.listdir(dir)\n",
    "total = len(fn)\n",
    "total\n",
    "#     change the range to switch between exp2 : [0, 1, 2, 3, 4, 5, 6]\n",
    "#     and exp2.1: [1, 3, 5, 7, 9]\n",
    "visible = [1, 2, 4, 7, 14]\n",
    "# put all the images into this blob of size total*size*size*1\n",
    "# REMEMBER to change shape of dataset\n",
    "training_dataset = np.ndarray(shape = (300*len(visible), image_size, image_size, 1), dtype = np.float32)\n",
    "counter = 0\n",
    "# REMEMBER to change shape of training label\n",
    "training_label = np.ndarray(shape = (300*len(visible)), dtype = np.float32)\n",
    "for file in fn:\n",
    "#     image_data = ndimage.imread(dir+file).astype(float)\n",
    "    image_data = (ndimage.imread(dir+file).astype(float) - 255/2) / pixel_depth\n",
    "    label = int(file.split(\"_\")[0])\n",
    "    if label in visible:\n",
    "        training_label[counter] = label\n",
    "        training_dataset[counter, :, :] = image_data[:,:,0].reshape(image_size, image_size, 1)\n",
    "        counter+=1\n",
    "#     else:\n",
    "#         print(file[0])\n",
    "# shuffle dataset\n",
    "permutation = np.random.permutation(counter)\n",
    "training_dataset = training_dataset[permutation,:,:,:]\n",
    "training_label = training_label[permutation]\n",
    "\n",
    "\n",
    "print(np.sum(training_dataset)/(300*len(visible)*image_size*image_size))\n",
    "print(len(training_label))\n",
    "%matplotlib inline\n",
    "\n",
    "# We'll show the image and its pixel value histogram side-by-side.\n",
    "_, (ax1, ax2) = plt.subplots(1, 2)\n",
    "\n",
    "# To interpret the values as a 28x28 image, we need to reshape\n",
    "# the numpy array, which is one dimensional.\n",
    "image = training_dataset[100]\n",
    "print(training_label[100])\n",
    "ax1.imshow(image.reshape(image_size, image_size), cmap=plt.cm.Greys);\n",
    "\n",
    "ax2.hist(image.reshape(image_size*image_size), bins=20, range=[-0.5, 0.5]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "train_data_node = tf.placeholder(tf.float32, shape=(batch_size, image_size, image_size, 1))\n",
    "\n",
    "train_labels_node = tf.placeholder(tf.float32, shape =(batch_size, 1))\n",
    "\n",
    "def print_variable():\n",
    "    for v in tf.global_variables():\n",
    "        print(v)\n",
    "# capping stddev at 0.1 is important to avoid exploding value\n",
    "def conv_relu(input, kernel_shape, bias_shape, stride = 2):\n",
    "    weights = tf.get_variable(\"weights\", kernel_shape, initializer=tf.truncated_normal_initializer(stddev=0.1))\n",
    "    biases = tf.get_variable(\"biases\", bias_shape, initializer=tf.constant_initializer(0.0))\n",
    "    conv = tf.nn.conv2d(input, weights, strides = [1, stride, stride, 1], padding = 'SAME')\n",
    "    return tf.nn.relu(conv+biases)\n",
    "def fully_connected(input, out_shape):\n",
    "    input_shape = input.get_shape().as_list()\n",
    "    batch = input_shape[0]\n",
    "    flattened_size = input_shape[1]\n",
    "    weights = tf.get_variable(\"weights\", [flattened_size, out_shape], \n",
    "                              initializer=tf.truncated_normal_initializer(stddev=0.1))\n",
    "    biases = tf.get_variable(\"biases\", [out_shape],\n",
    "                            initializer = tf.constant_initializer(0.1))\n",
    "    return tf.matmul(input, weights) + biases\n",
    "def network(input_images):\n",
    "    with tf.variable_scope(\"conv1\"):\n",
    "        # kernel is in shape height*width*in*out\n",
    "        conv_relu1 = conv_relu(input_images, [4, 4, 1, 32], [32])\n",
    "    with tf.variable_scope(\"conv2\"):\n",
    "        conv_relu2 = conv_relu(conv_relu1, [4, 4, 32, 64], [64])\n",
    "    with tf.variable_scope(\"conv3\"):\n",
    "        conv_relu3 = conv_relu(conv_relu2, [4, 4, 64, 128], [128])\n",
    "    with tf.variable_scope(\"fc1\"):\n",
    "        conv_relu3_shape = conv_relu3.get_shape().as_list()\n",
    "        reshape_conv3 = tf.reshape(conv_relu3,\n",
    "                                [conv_relu3_shape[0], conv_relu3_shape[1]*conv_relu3_shape[2]*conv_relu3_shape[3]])\n",
    "        fc1 = tf.nn.relu(fully_connected(reshape_conv3, 1024))\n",
    "    with tf.variable_scope(\"fc2\"):\n",
    "        fc2 = tf.nn.relu(fully_connected(fc1, 10))\n",
    "    with tf.variable_scope(\"kernel\"):\n",
    "        kernel = tf.concat([tf.pow(fc2, 2), fc2], 1)\n",
    "        result = fully_connected(kernel, 1)\n",
    "    return result\n",
    "    \n",
    "output = network(train_data_node)\n",
    "loss = tf.reduce_mean(tf.squared_difference(output, train_labels_node))\n",
    "\n",
    "\n",
    "fc1_weights = [v for v in tf.global_variables() if v.name == \"fc1/weights:0\"][0]\n",
    "fc1_biases = [v for v in tf.global_variables() if v.name == \"fc1/biases:0\"][0]\n",
    "fc2_weights = [v for v in tf.global_variables() if v.name == \"fc2/weights:0\"][0]\n",
    "fc2_biases = [v for v in tf.global_variables() if v.name == \"fc2/biases:0\"][0]\n",
    "regularizers = (tf.nn.l2_loss(fc1_weights) + tf.nn.l2_loss(fc1_biases) +\n",
    "                tf.nn.l2_loss(fc2_weights) + tf.nn.l2_loss(fc2_biases))\n",
    "\n",
    "loss += 5e-4 * regularizers\n",
    "\n",
    "train_size = len(training_label)\n",
    "\n",
    "batch = tf.Variable(0)\n",
    "# Decay once per epoch, using an exponential schedule starting at 0.01.\n",
    "learning_rate = tf.train.exponential_decay(\n",
    "  0.0001,                # Base learning rate.\n",
    "  batch * batch_size,  # Current index into the dataset.\n",
    "  train_size,          # Decay step. Decay after every epoch (seeing all example)\n",
    "  0.95,                # Decay rate.\n",
    "  staircase=True)\n",
    "# Use simple momentum for the optimization.\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate,\n",
    "                                       0.9).minimize(loss,\n",
    "                                                     global_step=batch)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.8)\n",
    "\n",
    "sess = tf.InteractiveSession(config=tf.ConfigProto(gpu_options=gpu_options))\n",
    "sess.as_default()\n",
    "\n",
    "tf.global_variables_initializer().run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "batch_data = training_dataset[:batch_size]\n",
    "batch_labels = training_label[:batch_size].reshape((batch_size, 1))\n",
    "\n",
    "\n",
    "_, l, lr, predictions = sess.run(\n",
    "  [optimizer, loss, learning_rate, output],\n",
    "  feed_dict={train_data_node: batch_data,\n",
    "             train_labels_node: batch_labels})\n",
    "\n",
    "print(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.Variable 'global_step:0' shape=() dtype=int32_ref>\n",
      "<tf.Variable 'conv1/weights:0' shape=(4, 4, 1, 32) dtype=float32_ref>\n",
      "<tf.Variable 'conv1/biases:0' shape=(32,) dtype=float32_ref>\n",
      "<tf.Variable 'conv2/weights:0' shape=(4, 4, 32, 64) dtype=float32_ref>\n",
      "<tf.Variable 'conv2/biases:0' shape=(64,) dtype=float32_ref>\n",
      "<tf.Variable 'conv3/weights:0' shape=(4, 4, 64, 128) dtype=float32_ref>\n",
      "<tf.Variable 'conv3/biases:0' shape=(128,) dtype=float32_ref>\n",
      "<tf.Variable 'fc1/weights:0' shape=(8192, 1024) dtype=float32_ref>\n",
      "<tf.Variable 'fc1/biases:0' shape=(1024,) dtype=float32_ref>\n",
      "<tf.Variable 'fc2/weights:0' shape=(1024, 10) dtype=float32_ref>\n",
      "<tf.Variable 'fc2/biases:0' shape=(10,) dtype=float32_ref>\n",
      "<tf.Variable 'kernel/weights:0' shape=(20, 1) dtype=float32_ref>\n",
      "<tf.Variable 'kernel/biases:0' shape=(1,) dtype=float32_ref>\n",
      "<tf.Variable 'Variable:0' shape=() dtype=int32_ref>\n",
      "<tf.Variable 'beta1_power:0' shape=() dtype=float32_ref>\n",
      "<tf.Variable 'beta2_power:0' shape=() dtype=float32_ref>\n",
      "<tf.Variable 'conv1/weights/Adam:0' shape=(4, 4, 1, 32) dtype=float32_ref>\n",
      "<tf.Variable 'conv1/weights/Adam_1:0' shape=(4, 4, 1, 32) dtype=float32_ref>\n",
      "<tf.Variable 'conv1/biases/Adam:0' shape=(32,) dtype=float32_ref>\n",
      "<tf.Variable 'conv1/biases/Adam_1:0' shape=(32,) dtype=float32_ref>\n",
      "<tf.Variable 'conv2/weights/Adam:0' shape=(4, 4, 32, 64) dtype=float32_ref>\n",
      "<tf.Variable 'conv2/weights/Adam_1:0' shape=(4, 4, 32, 64) dtype=float32_ref>\n",
      "<tf.Variable 'conv2/biases/Adam:0' shape=(64,) dtype=float32_ref>\n",
      "<tf.Variable 'conv2/biases/Adam_1:0' shape=(64,) dtype=float32_ref>\n",
      "<tf.Variable 'conv3/weights/Adam:0' shape=(4, 4, 64, 128) dtype=float32_ref>\n",
      "<tf.Variable 'conv3/weights/Adam_1:0' shape=(4, 4, 64, 128) dtype=float32_ref>\n",
      "<tf.Variable 'conv3/biases/Adam:0' shape=(128,) dtype=float32_ref>\n",
      "<tf.Variable 'conv3/biases/Adam_1:0' shape=(128,) dtype=float32_ref>\n",
      "<tf.Variable 'fc1/weights/Adam:0' shape=(8192, 1024) dtype=float32_ref>\n",
      "<tf.Variable 'fc1/weights/Adam_1:0' shape=(8192, 1024) dtype=float32_ref>\n",
      "<tf.Variable 'fc1/biases/Adam:0' shape=(1024,) dtype=float32_ref>\n",
      "<tf.Variable 'fc1/biases/Adam_1:0' shape=(1024,) dtype=float32_ref>\n",
      "<tf.Variable 'fc2/weights/Adam:0' shape=(1024, 10) dtype=float32_ref>\n",
      "<tf.Variable 'fc2/weights/Adam_1:0' shape=(1024, 10) dtype=float32_ref>\n",
      "<tf.Variable 'fc2/biases/Adam:0' shape=(10,) dtype=float32_ref>\n",
      "<tf.Variable 'fc2/biases/Adam_1:0' shape=(10,) dtype=float32_ref>\n",
      "<tf.Variable 'kernel/weights/Adam:0' shape=(20, 1) dtype=float32_ref>\n",
      "<tf.Variable 'kernel/weights/Adam_1:0' shape=(20, 1) dtype=float32_ref>\n",
      "<tf.Variable 'kernel/biases/Adam:0' shape=(1,) dtype=float32_ref>\n",
      "<tf.Variable 'kernel/biases/Adam_1:0' shape=(1,) dtype=float32_ref>\n"
     ]
    }
   ],
   "source": [
    "print_variable()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[[ -1.99053809e-01  -7.44020715e-02  -1.24661475e-01   8.01265612e-02\n",
      "     -4.37104963e-02   4.36762087e-02   5.99976815e-02   1.15336552e-01\n",
      "      7.14788272e-04   8.63619000e-02  -8.15904513e-02  -2.44693235e-02\n",
      "     -1.61531597e-01   8.67417306e-02   9.46746022e-02  -8.03697482e-02\n",
      "      4.85168472e-02   8.46753567e-02  -4.00047265e-02   2.78897192e-02\n",
      "      5.29607274e-02  -5.62932752e-02  -1.37443662e-01  -1.48446918e-01\n",
      "     -3.42981890e-02   6.07962161e-02   5.14490567e-02   3.37236300e-02\n",
      "      8.64395499e-02  -8.48618895e-02  -1.65788177e-02  -1.03640355e-01]]\n",
      "\n",
      "  [[  1.95826352e-01   1.14477508e-01   3.59015428e-02  -1.07955068e-01\n",
      "     -7.59413987e-02  -8.04913118e-02  -1.08434800e-02   6.75413236e-02\n",
      "      1.72568887e-01  -3.66455466e-02  -4.52249907e-02  -7.01508895e-02\n",
      "     -6.13531796e-04  -1.14761941e-01   1.02513120e-01  -4.59832810e-02\n",
      "      8.05846080e-02  -8.38233903e-02   5.87331019e-02   8.04345086e-02\n",
      "     -3.87987457e-02   8.29314590e-02   8.43048096e-02  -1.49392784e-01\n",
      "      1.71161648e-02   5.01734158e-03   2.17905827e-02   5.05506806e-02\n",
      "      6.55547678e-02  -8.99767596e-03  -1.29357681e-01   5.44660464e-02]]\n",
      "\n",
      "  [[  8.41955617e-02   4.92801890e-02   1.04170091e-01   1.38145965e-02\n",
      "     -1.58514544e-01  -4.81499694e-02   7.52081349e-02  -4.76399027e-02\n",
      "     -1.14467122e-01   7.06082284e-02   6.92334399e-02   1.10470569e-02\n",
      "      4.95399758e-02   5.00804447e-02  -1.81422666e-01   8.83801430e-02\n",
      "     -3.64780463e-02  -6.92514703e-02  -4.04731221e-02  -7.32856765e-02\n",
      "     -1.94169190e-02   1.65246964e-01   9.92272422e-02  -4.84037101e-02\n",
      "      1.14905657e-02  -4.38020043e-02   1.26347229e-01  -5.87045774e-02\n",
      "     -4.67399321e-02  -1.66178122e-02   2.18323693e-02   8.51521268e-02]]\n",
      "\n",
      "  [[ -1.97231956e-02   3.12249089e-04   1.22976163e-02   1.09681681e-01\n",
      "     -1.99396536e-02   7.10649863e-02  -8.46644193e-02   1.09864011e-01\n",
      "     -4.66308333e-02  -1.70021392e-02   1.74744204e-02  -3.02052964e-02\n",
      "      1.91643536e-01   7.24474490e-02   4.84735779e-02   1.22028030e-01\n",
      "     -2.07471512e-02  -4.03244048e-02  -1.15541480e-01   9.07766744e-02\n",
      "      6.18414395e-02  -3.53598557e-02  -5.84147535e-02  -3.55480850e-04\n",
      "     -9.70856324e-02   3.50112915e-02   1.04920007e-01   1.09816946e-01\n",
      "     -7.44085684e-02  -5.23613356e-02   1.16730236e-01   3.38613577e-02]]]\n",
      "\n",
      "\n",
      " [[[ -1.14043811e-02  -1.60417303e-01   3.09246257e-02   4.98834364e-02\n",
      "     -3.63366455e-02   2.84486394e-02  -7.76626691e-02  -1.05853342e-01\n",
      "      3.23782191e-02  -4.10734117e-02  -3.98453735e-02   4.50833254e-02\n",
      "      1.15176523e-02  -1.08216908e-02   1.51427507e-01   3.32380980e-02\n",
      "     -1.24381520e-01  -1.41473368e-01   9.78121832e-02   9.32273269e-03\n",
      "     -9.85674039e-02   2.92897969e-03   4.25491249e-03  -8.83926824e-02\n",
      "      4.53992467e-03  -2.84437556e-02   1.18547931e-01  -4.55218032e-02\n",
      "     -9.16488767e-02   1.17059436e-03   1.51077490e-02   1.91498518e-01]]\n",
      "\n",
      "  [[ -4.39378731e-02  -8.44824389e-02   7.95035735e-02  -5.23523316e-02\n",
      "      1.90494940e-01  -1.50740966e-01   2.43213028e-02   8.61107111e-02\n",
      "      9.74258259e-02  -6.51118811e-03  -1.43166393e-01   1.70389295e-01\n",
      "      1.98366828e-02   1.29398420e-01   2.22237650e-02   2.48954929e-02\n",
      "     -7.00815022e-02  -1.03566602e-01  -4.55553196e-02   3.82104330e-02\n",
      "      4.88981307e-02  -6.23233654e-02   7.47530088e-02   8.14726390e-03\n",
      "      8.42649043e-02  -1.46825135e-01   7.60475621e-02   6.06326154e-03\n",
      "     -5.55039458e-02  -2.32378989e-02   6.69161603e-02   1.18198926e-02]]\n",
      "\n",
      "  [[ -9.58735272e-02   6.55086115e-02   1.06322216e-02  -1.27229884e-01\n",
      "      1.51927546e-01   3.66111957e-02  -1.19364388e-01   8.60289782e-02\n",
      "      2.27567814e-02  -1.56985790e-01   1.89799547e-01  -7.85987154e-02\n",
      "      9.16883498e-02   1.35920227e-01  -8.40389729e-03  -4.70633991e-02\n",
      "     -6.34628311e-02  -1.31311789e-01   4.62360568e-02   1.66559681e-01\n",
      "     -7.09263533e-02   8.89498591e-02  -8.30850657e-03   3.57242785e-02\n",
      "     -4.19193767e-02  -1.58965677e-01   3.35811242e-03  -4.43850458e-02\n",
      "      3.17258798e-02   7.63753336e-03  -1.19649274e-02   6.63806722e-02]]\n",
      "\n",
      "  [[ -2.17174292e-02   7.89481029e-03  -3.13130207e-02  -8.46018568e-02\n",
      "      1.08493917e-01   1.19152643e-01   1.74640641e-02   7.29904398e-02\n",
      "      1.71077307e-02  -1.13283722e-02   8.17191377e-02  -6.26851618e-02\n",
      "      8.26892778e-02   7.41065666e-02   9.90097970e-02  -4.84825782e-02\n",
      "      4.33550142e-02  -1.74137771e-01   7.78175518e-02   1.06623650e-01\n",
      "      1.43866390e-01  -1.51829347e-01   6.84654713e-02   2.95902770e-02\n",
      "     -5.89413904e-02   4.85608429e-02  -2.99442001e-02  -1.38849065e-01\n",
      "     -1.55102141e-04   5.73872291e-02   8.92950296e-02   1.52901575e-01]]]\n",
      "\n",
      "\n",
      " [[[ -1.14841029e-01  -2.44586263e-03   3.71777499e-03   1.24555804e-01\n",
      "      7.37723336e-02  -3.22956517e-02  -3.95515598e-02  -9.70384628e-02\n",
      "     -2.01843656e-03   2.08218209e-02  -1.51365623e-01   8.76474474e-03\n",
      "      8.40872619e-03  -2.50900090e-02  -1.18693570e-02  -1.74003974e-01\n",
      "     -9.74315777e-02  -2.54296269e-02  -6.63011521e-02   1.10600367e-01\n",
      "     -4.62209992e-02  -2.50206217e-02  -8.51765741e-03  -1.22055665e-01\n",
      "      1.15643837e-01   1.65951610e-01   2.73228288e-02  -3.29951905e-02\n",
      "      1.64435133e-01  -1.12755662e-02   1.23345619e-02  -6.41965643e-02]]\n",
      "\n",
      "  [[ -1.12641498e-01   2.87602954e-02   6.38410822e-02  -4.91201803e-02\n",
      "      4.74230526e-03   7.55569153e-03  -1.09265946e-01   8.20739046e-02\n",
      "      1.66249916e-01   2.85283830e-02   1.25984371e-01   8.98218304e-02\n",
      "      1.14949219e-01  -7.79967532e-02  -4.95046079e-02   1.15760908e-01\n",
      "     -9.36372019e-03   1.70405749e-02  -8.68571028e-02   1.68892797e-02\n",
      "     -1.77021265e-01  -1.20281145e-01   6.07522577e-02   1.00674972e-01\n",
      "      4.33937386e-02   1.24964505e-01   1.08644180e-01   3.41590606e-02\n",
      "      2.06069518e-02  -1.58968806e-01   1.87297147e-02  -1.24650575e-01]]\n",
      "\n",
      "  [[ -1.70700878e-01   5.65023087e-02   8.70332941e-02  -1.11035876e-01\n",
      "     -5.76850846e-02  -9.32269730e-03  -6.97085336e-02   1.27443045e-01\n",
      "     -1.50930107e-01  -1.27754956e-01   8.69015306e-02  -7.36837685e-02\n",
      "     -7.41136372e-02   2.80684680e-02   1.66226327e-02   3.92274372e-02\n",
      "      1.28525600e-01  -5.47233038e-02  -1.08920284e-01  -3.66634950e-02\n",
      "      4.60766405e-02   1.44296184e-01   7.31314644e-02  -7.11937575e-03\n",
      "     -1.10296972e-01  -3.91699467e-03  -2.23937165e-02   1.21873980e-02\n",
      "      3.82668851e-03   3.88309024e-02  -6.16572425e-02  -1.63349379e-02]]\n",
      "\n",
      "  [[ -3.55886878e-03   3.19510847e-02  -9.66668129e-02   1.10695131e-01\n",
      "     -6.37417957e-02   1.18926592e-01  -2.53573321e-02  -5.17480150e-02\n",
      "     -1.06204171e-02  -1.59477085e-01  -7.39469901e-02   6.26432374e-02\n",
      "      1.33433864e-02   1.09669305e-01  -1.47004813e-01  -8.85182545e-02\n",
      "     -9.60810855e-02  -5.59681579e-02  -2.95553990e-02  -6.04743734e-02\n",
      "     -8.46096799e-02   2.88229045e-02  -4.54128496e-02  -5.48938364e-02\n",
      "      2.60929223e-02  -1.04229137e-01   1.47705287e-01  -3.41210887e-02\n",
      "      4.14311333e-04  -8.63657370e-02   1.27835184e-01   1.55427933e-01]]]\n",
      "\n",
      "\n",
      " [[[  1.99237578e-02   1.07699551e-01  -3.11143678e-02   1.77567095e-01\n",
      "      1.34465918e-01   3.01945191e-02  -6.33753315e-02   7.61247501e-02\n",
      "     -1.77835543e-02   1.17274292e-01   1.12663753e-01   7.25011155e-02\n",
      "      6.76556230e-02   3.55478488e-02  -4.28721122e-02   7.96294864e-03\n",
      "     -2.80830376e-02   7.22030848e-02   8.17646459e-02  -1.04293190e-02\n",
      "      5.69656417e-02  -4.72727232e-02   1.50123741e-02   6.02033548e-02\n",
      "      5.02091907e-02   7.62705132e-02   6.75383210e-03  -6.20572083e-02\n",
      "     -4.97027673e-02   1.55519322e-01  -2.75803525e-02   8.97940174e-02]]\n",
      "\n",
      "  [[  4.17074785e-02   4.46337536e-02  -6.03844374e-02  -5.22054248e-02\n",
      "     -2.09770538e-02  -5.60057610e-02  -1.86171338e-01   1.24601223e-01\n",
      "      8.60217586e-02   6.03032336e-02   1.10716142e-01  -8.68442804e-02\n",
      "     -1.40552640e-01   9.12587419e-02  -4.88560684e-02  -6.36664778e-02\n",
      "      1.63540483e-01  -5.28927594e-02  -3.31795625e-02  -1.68519970e-02\n",
      "     -1.07155234e-01   4.96089123e-02  -1.40032629e-02   9.19091795e-03\n",
      "      1.76769812e-02   1.01944655e-01   3.39295790e-02   1.57865241e-01\n",
      "     -1.17011145e-01   7.53224269e-02  -1.19228058e-01   6.12764470e-02]]\n",
      "\n",
      "  [[  4.85665165e-02  -7.60328993e-02  -1.14957308e-02   4.03047167e-02\n",
      "      4.05643843e-02   6.07161671e-02   1.83710530e-02  -3.19991373e-02\n",
      "     -7.07548335e-02  -2.58766096e-02   1.38675004e-01   8.32805037e-02\n",
      "     -6.56686798e-02   3.25761177e-03   5.78406565e-02   1.08468935e-01\n",
      "     -8.84192437e-02   5.10039590e-02  -9.73203126e-03  -3.80965672e-03\n",
      "     -8.12296942e-02   7.55806118e-02  -4.30883430e-02  -5.87994978e-03\n",
      "     -1.16126008e-01  -1.96772031e-02   6.71574567e-03   3.06112841e-02\n",
      "     -3.95990647e-02   2.95464657e-02   1.65581152e-01  -3.39941643e-02]]\n",
      "\n",
      "  [[ -1.22280143e-01  -8.72390568e-02  -6.98435828e-02  -1.09088905e-01\n",
      "      9.30181369e-02   2.94730477e-02  -1.07332863e-01   1.91269383e-01\n",
      "     -2.98230555e-02  -5.45656607e-02  -1.24182649e-01   6.23572692e-02\n",
      "     -2.54321918e-02  -1.14833795e-01  -1.68967977e-01  -6.47653192e-02\n",
      "     -1.92366928e-01  -4.84521650e-02   6.61567524e-02   3.96466218e-02\n",
      "     -1.73032284e-01   8.37245118e-03  -3.86268757e-02   1.17880620e-01\n",
      "     -7.33838156e-02   1.69147491e-01  -5.12477271e-02   1.82077959e-02\n",
      "     -7.19731301e-02  -1.49004549e-01  -4.92794253e-02  -1.29355788e-01]]]]\n"
     ]
    }
   ],
   "source": [
    "var_23 = [v for v in tf.global_variables() if v.name == \"conv1/weights:0\"][0]\n",
    "print(sess.run(var_23))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 0\n",
      "[[ 6.78777122]]\n",
      "Mini-batch loss: 23.99713 Learning rate: 0.00010\n",
      "Step 500\n",
      "[[ 2.75266409]]\n",
      "Mini-batch loss: 12.79630 Learning rate: 0.00010\n",
      "Step 1000\n",
      "[[ 10.70023441]]\n",
      "Mini-batch loss: 21.77671 Learning rate: 0.00010\n",
      "Step 1500\n",
      "[[ 3.86252069]]\n",
      "Mini-batch loss: 10.33881 Learning rate: 0.00009\n",
      "Step 2000\n",
      "[[ 2.15028882]]\n",
      "Mini-batch loss: 10.05876 Learning rate: 0.00009\n",
      "Step 2500\n",
      "[[ 6.88041496]]\n",
      "Mini-batch loss: 9.85802 Learning rate: 0.00009\n",
      "Step 3000\n",
      "[[ 15.6029501]]\n",
      "Mini-batch loss: 12.25310 Learning rate: 0.00009\n",
      "Step 3500\n",
      "[[ 3.73951483]]\n",
      "Mini-batch loss: 9.60291 Learning rate: 0.00009\n",
      "Step 4000\n",
      "[[ 12.46959877]]\n",
      "Mini-batch loss: 11.72317 Learning rate: 0.00009\n",
      "Step 4500\n",
      "[[ 2.11031699]]\n",
      "Mini-batch loss: 9.23307 Learning rate: 0.00009\n",
      "Step 5000\n",
      "[[ 1.86651957]]\n",
      "Mini-batch loss: 9.07859 Learning rate: 0.00009\n",
      "Step 5500\n",
      "[[ 0.85838169]]\n",
      "Mini-batch loss: 8.91223 Learning rate: 0.00009\n",
      "Step 6000\n",
      "[[ 1.47897351]]\n",
      "Mini-batch loss: 8.94855 Learning rate: 0.00008\n",
      "Step 6500\n",
      "[[ 1.82816637]]\n",
      "Mini-batch loss: 8.57254 Learning rate: 0.00008\n",
      "Step 7000\n",
      "[[ 7.14851856]]\n",
      "Mini-batch loss: 8.37776 Learning rate: 0.00008\n",
      "Step 7500\n",
      "[[ 6.66627073]]\n",
      "Mini-batch loss: 8.26491 Learning rate: 0.00008\n",
      "Step 8000\n",
      "[[ 2.91681075]]\n",
      "Mini-batch loss: 8.78365 Learning rate: 0.00008\n",
      "Step 8500\n",
      "[[ 3.88425231]]\n",
      "Mini-batch loss: 7.73092 Learning rate: 0.00008\n",
      "Step 9000\n",
      "[[ 16.10990334]]\n",
      "Mini-batch loss: 11.93335 Learning rate: 0.00007\n",
      "Step 9500\n",
      "[[ 1.21538842]]\n",
      "Mini-batch loss: 7.28831 Learning rate: 0.00007\n",
      "Step 10000\n",
      "[[ 14.00658512]]\n",
      "Mini-batch loss: 6.98749 Learning rate: 0.00007\n",
      "Step 10500\n",
      "[[ 7.60560703]]\n",
      "Mini-batch loss: 7.08805 Learning rate: 0.00007\n",
      "Step 11000\n",
      "[[ 0.98039627]]\n",
      "Mini-batch loss: 6.45307 Learning rate: 0.00007\n",
      "Step 11500\n",
      "[[ 13.04761887]]\n",
      "Mini-batch loss: 7.08494 Learning rate: 0.00007\n",
      "Step 12000\n",
      "[[ 1.5202527]]\n",
      "Mini-batch loss: 6.17498 Learning rate: 0.00007\n",
      "Step 12500\n",
      "[[ 9.5916214]]\n",
      "Mini-batch loss: 25.07327 Learning rate: 0.00007\n",
      "Step 13000\n",
      "[[ 6.04118967]]\n",
      "Mini-batch loss: 6.30347 Learning rate: 0.00007\n",
      "Step 13500\n",
      "[[ 4.30077982]]\n",
      "Mini-batch loss: 5.22651 Learning rate: 0.00006\n",
      "Step 14000\n",
      "[[ 3.83994102]]\n",
      "Mini-batch loss: 4.92530 Learning rate: 0.00006\n",
      "Step 14500\n",
      "[[ 0.79158795]]\n",
      "Mini-batch loss: 4.71303 Learning rate: 0.00006\n",
      "Step 15000\n",
      "[[ 7.04983854]]\n",
      "Mini-batch loss: 4.45252 Learning rate: 0.00006\n",
      "Step 15500\n",
      "[[ 3.96219015]]\n",
      "Mini-batch loss: 4.24812 Learning rate: 0.00006\n",
      "Step 16000\n",
      "[[ 12.94384384]]\n",
      "Mini-batch loss: 5.17205 Learning rate: 0.00006\n",
      "Step 16500\n",
      "[[ 4.59453201]]\n",
      "Mini-batch loss: 4.23306 Learning rate: 0.00006\n",
      "Step 17000\n",
      "[[ 11.31620026]]\n",
      "Mini-batch loss: 10.92720 Learning rate: 0.00006\n",
      "Step 17500\n",
      "[[ 6.77925014]]\n",
      "Mini-batch loss: 3.64060 Learning rate: 0.00006\n",
      "Step 18000\n",
      "[[ 14.00446033]]\n",
      "Mini-batch loss: 3.46865 Learning rate: 0.00005\n",
      "Step 18500\n",
      "[[ 6.88270378]]\n",
      "Mini-batch loss: 3.36537 Learning rate: 0.00005\n",
      "Step 19000\n",
      "[[ 3.98525333]]\n",
      "Mini-batch loss: 3.23337 Learning rate: 0.00005\n",
      "Step 19500\n",
      "[[ 2.04192209]]\n",
      "Mini-batch loss: 3.11603 Learning rate: 0.00005\n",
      "Step 20000\n",
      "[[ 7.20462513]]\n",
      "Mini-batch loss: 3.04094 Learning rate: 0.00005\n",
      "Step 20500\n",
      "[[ 13.26564598]]\n",
      "Mini-batch loss: 3.42245 Learning rate: 0.00005\n",
      "Step 21000\n",
      "[[ 4.09833574]]\n",
      "Mini-batch loss: 2.77855 Learning rate: 0.00005\n",
      "Step 21500\n",
      "[[ 3.765908]]\n",
      "Mini-batch loss: 2.71558 Learning rate: 0.00005\n",
      "Step 22000\n",
      "[[ 0.84805501]]\n",
      "Mini-batch loss: 2.57892 Learning rate: 0.00005\n",
      "Step 22500\n",
      "[[ 1.79126477]]\n",
      "Mini-batch loss: 2.49866 Learning rate: 0.00005\n",
      "Step 23000\n",
      "[[ 6.42819357]]\n",
      "Mini-batch loss: 2.69002 Learning rate: 0.00005\n",
      "Step 23500\n",
      "[[ 1.73526609]]\n",
      "Mini-batch loss: 2.36020 Learning rate: 0.00005\n",
      "Step 24000\n",
      "[[ 14.2049942]]\n",
      "Mini-batch loss: 2.27146 Learning rate: 0.00004\n",
      "Step 24500\n",
      "[[ 14.47634602]]\n",
      "Mini-batch loss: 2.40045 Learning rate: 0.00004\n",
      "Step 25000\n",
      "[[ 1.00763607]]\n",
      "Mini-batch loss: 2.12430 Learning rate: 0.00004\n",
      "Step 25500\n",
      "[[ 4.14561653]]\n",
      "Mini-batch loss: 2.10126 Learning rate: 0.00004\n",
      "Step 26000\n",
      "[[ 4.13052607]]\n",
      "Mini-batch loss: 2.05300 Learning rate: 0.00004\n",
      "Step 26500\n",
      "[[ 7.09110737]]\n",
      "Mini-batch loss: 1.99956 Learning rate: 0.00004\n",
      "Step 27000\n",
      "[[ 2.09536839]]\n",
      "Mini-batch loss: 1.95562 Learning rate: 0.00004\n",
      "Step 27500\n",
      "[[ 1.25006628]]\n",
      "Mini-batch loss: 1.96314 Learning rate: 0.00004\n",
      "Step 28000\n",
      "[[ 0.966829]]\n",
      "Mini-batch loss: 1.85570 Learning rate: 0.00004\n",
      "Step 28500\n",
      "[[ 14.42587185]]\n",
      "Mini-batch loss: 1.99063 Learning rate: 0.00004\n",
      "Step 29000\n",
      "[[ 4.06098652]]\n",
      "Mini-batch loss: 1.76659 Learning rate: 0.00004\n",
      "Step 29500\n",
      "[[ 14.06716633]]\n",
      "Mini-batch loss: 1.72326 Learning rate: 0.00004\n",
      "Step 30000\n",
      "[[ 7.15364647]]\n",
      "Mini-batch loss: 1.70045 Learning rate: 0.00004\n",
      "Step 30500\n",
      "[[ 4.10975027]]\n",
      "Mini-batch loss: 1.64668 Learning rate: 0.00004\n",
      "Step 31000\n",
      "[[ 7.13212585]]\n",
      "Mini-batch loss: 1.61463 Learning rate: 0.00004\n",
      "Step 31500\n",
      "[[ 1.10691106]]\n",
      "Mini-batch loss: 1.57531 Learning rate: 0.00003\n",
      "Step 32000\n",
      "[[ 4.10313511]]\n",
      "Mini-batch loss: 1.54213 Learning rate: 0.00003\n",
      "Step 32500\n",
      "[[ 3.8608799]]\n",
      "Mini-batch loss: 1.52116 Learning rate: 0.00003\n",
      "Step 33000\n",
      "[[ 7.01588488]]\n",
      "Mini-batch loss: 1.47562 Learning rate: 0.00003\n",
      "Step 33500\n",
      "[[ 3.73983622]]\n",
      "Mini-batch loss: 1.51756 Learning rate: 0.00003\n",
      "Step 34000\n",
      "[[ 3.90122104]]\n",
      "Mini-batch loss: 1.43692 Learning rate: 0.00003\n",
      "Step 34500\n",
      "[[ 4.0896554]]\n",
      "Mini-batch loss: 1.41394 Learning rate: 0.00003\n",
      "Step 35000\n",
      "[[ 1.01679182]]\n",
      "Mini-batch loss: 1.38419 Learning rate: 0.00003\n",
      "Step 35500\n",
      "[[ 1.97098064]]\n",
      "Mini-batch loss: 1.36384 Learning rate: 0.00003\n",
      "Step 36000\n",
      "[[ 14.32836342]]\n",
      "Mini-batch loss: 1.45001 Learning rate: 0.00003\n",
      "Step 36500\n",
      "[[ 3.73574328]]\n",
      "Mini-batch loss: 1.39033 Learning rate: 0.00003\n",
      "Step 37000\n",
      "[[ 14.57403183]]\n",
      "Mini-batch loss: 1.62872 Learning rate: 0.00003\n",
      "Step 37500\n",
      "[[ 2.03896499]]\n",
      "Mini-batch loss: 1.27994 Learning rate: 0.00003\n",
      "Step 38000\n",
      "[[ 13.18087769]]\n",
      "Mini-batch loss: 1.92798 Learning rate: 0.00003\n",
      "Step 38500\n",
      "[[ 14.12798405]]\n",
      "Mini-batch loss: 1.25300 Learning rate: 0.00003\n",
      "Step 39000\n",
      "[[ 2.09505296]]\n",
      "Mini-batch loss: 1.22642 Learning rate: 0.00003\n",
      "Step 39500\n",
      "[[ 6.3692894]]\n",
      "Mini-batch loss: 1.59548 Learning rate: 0.00003\n",
      "Step 40000\n",
      "[[ 7.11797142]]\n",
      "Mini-batch loss: 1.19363 Learning rate: 0.00003\n",
      "Step 40500\n",
      "[[ 14.05807972]]\n",
      "Mini-batch loss: 1.16715 Learning rate: 0.00003\n",
      "Step 41000\n",
      "[[ 6.41634846]]\n",
      "Mini-batch loss: 1.48791 Learning rate: 0.00003\n",
      "Step 41500\n",
      "[[ 7.19598341]]\n",
      "Mini-batch loss: 1.17151 Learning rate: 0.00003\n",
      "Step 42000\n",
      "[[ 1.08616138]]\n",
      "Mini-batch loss: 1.12842 Learning rate: 0.00002\n",
      "Step 42500\n",
      "[[ 13.59781361]]\n",
      "Mini-batch loss: 1.26983 Learning rate: 0.00002\n",
      "Step 43000\n",
      "[[ 7.33655787]]\n",
      "Mini-batch loss: 1.21006 Learning rate: 0.00002\n",
      "Step 43500\n",
      "[[ 4.04696226]]\n",
      "Mini-batch loss: 1.08797 Learning rate: 0.00002\n",
      "Step 44000\n",
      "[[ 6.95564032]]\n",
      "Mini-batch loss: 1.07588 Learning rate: 0.00002\n",
      "Step 44500\n",
      "[[ 0.97579509]]\n",
      "Mini-batch loss: 1.06350 Learning rate: 0.00002\n",
      "Step 45000\n",
      "[[ 1.12990606]]\n",
      "Mini-batch loss: 1.06827 Learning rate: 0.00002\n",
      "Step 45500\n",
      "[[ 1.86341393]]\n",
      "Mini-batch loss: 1.05769 Learning rate: 0.00002\n",
      "Step 46000\n",
      "[[ 7.2730341]]\n",
      "Mini-batch loss: 1.10181 Learning rate: 0.00002\n",
      "Step 46500\n",
      "[[ 2.04524398]]\n",
      "Mini-batch loss: 1.01692 Learning rate: 0.00002\n",
      "Step 47000\n",
      "[[ 1.89890897]]\n",
      "Mini-batch loss: 1.01194 Learning rate: 0.00002\n",
      "Step 47500\n",
      "[[ 0.96320862]]\n",
      "Mini-batch loss: 0.99046 Learning rate: 0.00002\n",
      "Step 48000\n",
      "[[ 3.93186665]]\n",
      "Mini-batch loss: 0.98058 Learning rate: 0.00002\n",
      "Step 48500\n",
      "[[ 13.65103531]]\n",
      "Mini-batch loss: 1.08394 Learning rate: 0.00002\n",
      "Step 49000\n",
      "[[ 4.02230787]]\n",
      "Mini-batch loss: 0.94957 Learning rate: 0.00002\n",
      "Step 49500\n",
      "[[ 14.22629642]]\n",
      "Mini-batch loss: 0.98671 Learning rate: 0.00002\n",
      "Step 50000\n",
      "[[ 1.00384092]]\n",
      "Mini-batch loss: 0.92151 Learning rate: 0.00002\n",
      "Step 50500\n",
      "[[ 7.16172981]]\n",
      "Mini-batch loss: 0.93462 Learning rate: 0.00002\n",
      "Step 51000\n",
      "[[ 1.08422029]]\n",
      "Mini-batch loss: 0.90246 Learning rate: 0.00002\n",
      "Step 51500\n",
      "[[ 13.98701572]]\n",
      "Mini-batch loss: 0.88223 Learning rate: 0.00002\n",
      "Step 52000\n",
      "[[ 0.90127879]]\n",
      "Mini-batch loss: 0.87988 Learning rate: 0.00002\n",
      "Step 52500\n",
      "[[ 6.95634747]]\n",
      "Mini-batch loss: 0.86038 Learning rate: 0.00002\n",
      "Step 53000\n",
      "[[ 13.98478985]]\n",
      "Mini-batch loss: 0.84692 Learning rate: 0.00002\n",
      "Step 53500\n",
      "[[ 0.99259591]]\n",
      "Mini-batch loss: 0.83692 Learning rate: 0.00002\n",
      "Step 54000\n",
      "[[ 4.02406025]]\n",
      "Mini-batch loss: 0.82873 Learning rate: 0.00002\n",
      "Step 54500\n",
      "[[ 1.93428922]]\n",
      "Mini-batch loss: 0.82407 Learning rate: 0.00002\n",
      "Step 55000\n",
      "[[ 14.66704941]]\n",
      "Mini-batch loss: 1.25851 Learning rate: 0.00002\n",
      "Step 55500\n",
      "[[ 1.89446878]]\n",
      "Mini-batch loss: 0.81914 Learning rate: 0.00001\n",
      "Step 56000\n",
      "[[ 6.96137619]]\n",
      "Mini-batch loss: 0.80395 Learning rate: 0.00001\n",
      "Step 56500\n",
      "[[ 2.07213974]]\n",
      "Mini-batch loss: 0.80311 Learning rate: 0.00001\n",
      "Step 57000\n",
      "[[ 0.88265765]]\n",
      "Mini-batch loss: 0.80685 Learning rate: 0.00001\n",
      "Step 57500\n",
      "[[ 1.01252496]]\n",
      "Mini-batch loss: 0.78806 Learning rate: 0.00001\n",
      "Step 58000\n",
      "[[ 1.00857496]]\n",
      "Mini-batch loss: 0.78318 Learning rate: 0.00001\n",
      "Step 58500\n",
      "[[ 3.88255143]]\n",
      "Mini-batch loss: 0.79153 Learning rate: 0.00001\n",
      "Step 59000\n",
      "[[ 2.01109695]]\n",
      "Mini-batch loss: 0.77206 Learning rate: 0.00001\n",
      "Step 59500\n",
      "[[ 1.99677694]]\n",
      "Mini-batch loss: 0.76647 Learning rate: 0.00001\n",
      "Step 60000\n",
      "[[ 4.0606041]]\n",
      "Mini-batch loss: 0.76405 Learning rate: 0.00001\n",
      "Step 60500\n",
      "[[ 3.8704195]]\n",
      "Mini-batch loss: 0.77065 Learning rate: 0.00001\n",
      "Step 61000\n",
      "[[ 1.97541463]]\n",
      "Mini-batch loss: 0.74831 Learning rate: 0.00001\n",
      "Step 61500\n",
      "[[ 1.00037658]]\n",
      "Mini-batch loss: 0.74098 Learning rate: 0.00001\n",
      "Step 62000\n",
      "[[ 3.93566775]]\n",
      "Mini-batch loss: 0.73795 Learning rate: 0.00001\n",
      "Step 62500\n",
      "[[ 14.07296562]]\n",
      "Mini-batch loss: 0.73239 Learning rate: 0.00001\n",
      "Step 63000\n",
      "[[ 1.94662499]]\n",
      "Mini-batch loss: 0.72268 Learning rate: 0.00001\n",
      "Step 63500\n",
      "[[ 3.86930919]]\n",
      "Mini-batch loss: 0.72927 Learning rate: 0.00001\n",
      "Step 64000\n",
      "[[ 4.0488286]]\n",
      "Mini-batch loss: 0.70745 Learning rate: 0.00001\n",
      "Step 64500\n",
      "[[ 14.0364542]]\n",
      "Mini-batch loss: 0.69888 Learning rate: 0.00001\n",
      "Step 65000\n",
      "[[ 14.09052849]]\n",
      "Mini-batch loss: 0.69785 Learning rate: 0.00001\n",
      "Step 65500\n",
      "[[ 1.97423136]]\n",
      "Mini-batch loss: 0.68309 Learning rate: 0.00001\n",
      "Step 66000\n",
      "[[ 13.87276459]]\n",
      "Mini-batch loss: 0.69121 Learning rate: 0.00001\n",
      "Step 66500\n",
      "[[ 13.9813261]]\n",
      "Mini-batch loss: 0.66764 Learning rate: 0.00001\n",
      "Step 67000\n",
      "[[ 4.01243782]]\n",
      "Mini-batch loss: 0.66054 Learning rate: 0.00001\n",
      "Step 67500\n",
      "[[ 3.98482966]]\n",
      "Mini-batch loss: 0.65385 Learning rate: 0.00001\n",
      "Step 68000\n",
      "[[ 3.97165298]]\n",
      "Mini-batch loss: 0.64744 Learning rate: 0.00001\n",
      "Step 68500\n",
      "[[ 2.09228992]]\n",
      "Mini-batch loss: 0.64944 Learning rate: 0.00001\n",
      "Step 69000\n",
      "[[ 1.95865607]]\n",
      "Mini-batch loss: 0.63745 Learning rate: 0.00001\n",
      "Step 69500\n",
      "[[ 6.91511679]]\n",
      "Mini-batch loss: 0.63765 Learning rate: 0.00001\n",
      "Step 70000\n",
      "[[ 3.98080468]]\n",
      "Mini-batch loss: 0.62670 Learning rate: 0.00001\n",
      "Step 70500\n",
      "[[ 6.94223166]]\n",
      "Mini-batch loss: 0.62614 Learning rate: 0.00001\n",
      "Step 71000\n",
      "[[ 1.02063358]]\n",
      "Mini-batch loss: 0.61956 Learning rate: 0.00001\n",
      "Step 71500\n",
      "[[ 1.04363513]]\n",
      "Mini-batch loss: 0.61810 Learning rate: 0.00001\n",
      "Step 72000\n",
      "[[ 1.05051136]]\n",
      "Mini-batch loss: 0.61594 Learning rate: 0.00001\n",
      "Step 72500\n",
      "[[ 0.96848595]]\n",
      "Mini-batch loss: 0.61129 Learning rate: 0.00001\n",
      "Step 73000\n",
      "[[ 13.99818897]]\n",
      "Mini-batch loss: 0.60759 Learning rate: 0.00001\n",
      "Step 73500\n",
      "[[ 7.06490517]]\n",
      "Mini-batch loss: 0.60905 Learning rate: 0.00001\n",
      "Step 74000\n",
      "[[ 3.9794023]]\n",
      "Mini-batch loss: 0.60224 Learning rate: 0.00001\n",
      "Step 74500\n",
      "[[ 1.93721986]]\n",
      "Mini-batch loss: 0.60297 Learning rate: 0.00001\n",
      "Step 75000\n",
      "[[ 13.95403957]]\n",
      "Mini-batch loss: 0.59825 Learning rate: 0.00001\n",
      "Step 75500\n",
      "[[ 3.96575308]]\n",
      "Mini-batch loss: 0.59415 Learning rate: 0.00001\n",
      "Step 76000\n",
      "[[ 13.9251852]]\n",
      "Mini-batch loss: 0.59559 Learning rate: 0.00001\n",
      "Step 76500\n",
      "[[ 1.89373732]]\n",
      "Mini-batch loss: 0.59817 Learning rate: 0.00001\n",
      "Step 77000\n",
      "[[ 13.95710468]]\n",
      "Mini-batch loss: 0.58533 Learning rate: 0.00001\n",
      "Step 77500\n",
      "[[ 3.97552109]]\n",
      "Mini-batch loss: 0.58087 Learning rate: 0.00001\n",
      "Step 78000\n",
      "[[ 13.99260998]]\n",
      "Mini-batch loss: 0.57699 Learning rate: 0.00001\n",
      "Step 78500\n",
      "[[ 0.96967244]]\n",
      "Mini-batch loss: 0.57427 Learning rate: 0.00001\n",
      "Step 79000\n",
      "[[ 1.02631605]]\n",
      "Mini-batch loss: 0.57064 Learning rate: 0.00001\n",
      "Step 79500\n",
      "[[ 4.02989864]]\n",
      "Mini-batch loss: 0.56736 Learning rate: 0.00001\n",
      "Step 80000\n",
      "[[ 0.94809717]]\n",
      "Mini-batch loss: 0.56545 Learning rate: 0.00001\n",
      "Step 80500\n",
      "[[ 7.04690695]]\n",
      "Mini-batch loss: 0.56147 Learning rate: 0.00001\n",
      "Step 81000\n",
      "[[ 1.9005307]]\n",
      "Mini-batch loss: 0.56566 Learning rate: 0.00001\n",
      "Step 81500\n",
      "[[ 3.89220381]]\n",
      "Mini-batch loss: 0.56368 Learning rate: 0.00001\n",
      "Step 82000\n",
      "[[ 14.02543068]]\n",
      "Mini-batch loss: 0.54930 Learning rate: 0.00001\n",
      "Step 82500\n",
      "[[ 2.02192163]]\n",
      "Mini-batch loss: 0.54580 Learning rate: 0.00001\n",
      "Step 83000\n",
      "[[ 1.91557562]]\n",
      "Mini-batch loss: 0.54893 Learning rate: 0.00001\n",
      "Step 83500\n",
      "[[ 6.98420382]]\n",
      "Mini-batch loss: 0.53892 Learning rate: 0.00001\n",
      "Step 84000\n",
      "[[ 3.99494195]]\n",
      "Mini-batch loss: 0.53567 Learning rate: 0.00001\n",
      "Step 84500\n",
      "[[ 1.90073919]]\n",
      "Mini-batch loss: 0.54233 Learning rate: 0.00001\n",
      "Step 85000\n",
      "[[ 3.94759393]]\n",
      "Mini-batch loss: 0.53251 Learning rate: 0.00001\n",
      "Step 85500\n",
      "[[ 3.95000339]]\n",
      "Mini-batch loss: 0.52968 Learning rate: 0.00001\n",
      "Step 86000\n",
      "[[ 1.00047684]]\n",
      "Mini-batch loss: 0.52451 Learning rate: 0.00001\n",
      "Step 86500\n",
      "[[ 3.99604297]]\n",
      "Mini-batch loss: 0.52230 Learning rate: 0.00001\n",
      "Step 87000\n",
      "[[ 13.95440674]]\n",
      "Mini-batch loss: 0.52219 Learning rate: 0.00001\n",
      "Step 87500\n",
      "[[ 4.01596737]]\n",
      "Mini-batch loss: 0.51814 Learning rate: 0.00001\n",
      "Step 88000\n",
      "[[ 0.95953602]]\n",
      "Mini-batch loss: 0.51758 Learning rate: 0.00001\n",
      "Step 88500\n",
      "[[ 1.01844954]]\n",
      "Mini-batch loss: 0.51428 Learning rate: 0.00000\n",
      "Step 89000\n",
      "[[ 7.12507105]]\n",
      "Mini-batch loss: 0.52754 Learning rate: 0.00000\n",
      "Step 89500\n",
      "[[ 14.02262783]]\n",
      "Mini-batch loss: 0.51051 Learning rate: 0.00000\n",
      "Step 90000\n",
      "[[ 6.99227524]]\n",
      "Mini-batch loss: 0.50808 Learning rate: 0.00000\n",
      "Step 90500\n",
      "[[ 7.14477062]]\n",
      "Mini-batch loss: 0.52694 Learning rate: 0.00000\n",
      "Step 91000\n",
      "[[ 4.01095772]]\n",
      "Mini-batch loss: 0.50417 Learning rate: 0.00000\n",
      "Step 91500\n",
      "[[ 3.95051074]]\n",
      "Mini-batch loss: 0.50447 Learning rate: 0.00000\n",
      "Step 92000\n",
      "[[ 14.20340347]]\n",
      "Mini-batch loss: 0.54131 Learning rate: 0.00000\n",
      "Step 92500\n",
      "[[ 0.92047763]]\n",
      "Mini-batch loss: 0.50427 Learning rate: 0.00000\n",
      "Step 93000\n",
      "[[ 0.99396282]]\n",
      "Mini-batch loss: 0.49588 Learning rate: 0.00000\n",
      "Step 93500\n",
      "[[ 7.06361914]]\n",
      "Mini-batch loss: 0.49774 Learning rate: 0.00000\n",
      "Step 94000\n",
      "[[ 1.93330407]]\n",
      "Mini-batch loss: 0.49608 Learning rate: 0.00000\n",
      "Step 94500\n",
      "[[ 13.9636116]]\n",
      "Mini-batch loss: 0.49080 Learning rate: 0.00000\n",
      "Step 95000\n",
      "[[ 14.13183403]]\n",
      "Mini-batch loss: 0.50466 Learning rate: 0.00000\n",
      "Step 95500\n",
      "[[ 1.89843976]]\n",
      "Mini-batch loss: 0.49549 Learning rate: 0.00000\n",
      "Step 96000\n",
      "[[ 2.05068684]]\n",
      "Mini-batch loss: 0.48557 Learning rate: 0.00000\n",
      "Step 96500\n",
      "[[ 7.02822447]]\n",
      "Mini-batch loss: 0.48160 Learning rate: 0.00000\n",
      "Step 97000\n",
      "[[ 7.02450895]]\n",
      "Mini-batch loss: 0.47931 Learning rate: 0.00000\n",
      "Step 97500\n",
      "[[ 1.88780117]]\n",
      "Mini-batch loss: 0.48917 Learning rate: 0.00000\n",
      "Step 98000\n",
      "[[ 3.98545575]]\n",
      "Mini-batch loss: 0.47465 Learning rate: 0.00000\n",
      "Step 98500\n",
      "[[ 0.99085349]]\n",
      "Mini-batch loss: 0.47252 Learning rate: 0.00000\n",
      "Step 99000\n",
      "[[ 1.02424216]]\n",
      "Mini-batch loss: 0.47100 Learning rate: 0.00000\n",
      "Step 99500\n",
      "[[ 1.93911731]]\n",
      "Mini-batch loss: 0.47211 Learning rate: 0.00000\n"
     ]
    }
   ],
   "source": [
    "for step in range(100000):\n",
    "    offset = (step * batch_size) % (train_size - batch_size)\n",
    "    batch_data = training_dataset[offset:batch_size+offset]\n",
    "    batch_labels = training_label[offset:batch_size+offset].reshape((batch_size, 1))\n",
    "    \n",
    "    _, l, lr, predictions = sess.run(\n",
    "      [optimizer, loss, learning_rate, output],\n",
    "      feed_dict={train_data_node: batch_data,\n",
    "             train_labels_node: batch_labels})\n",
    "    \n",
    "    if step % 500 ==0:\n",
    "        print('Step %d' %step)\n",
    "        print(predictions)\n",
    "        print('Mini-batch loss: %.5f Learning rate: %.5f' % (l, lr))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Validation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "dir= \"barcode/valid/\"\n",
    "fn = os.listdir(dir)\n",
    "total = len(fn)\n",
    "total\n",
    "#     change the range to switch between exp2 : [0, 1, 2, 3, 4, 5, 6]\n",
    "#     and exp2.1: [1, 3, 5, 7, 9]\n",
    "visible = [1, 2, 4, 7, 14]\n",
    "# put all the images into this blob of size total*size*size*1\n",
    "# REMEMBER to change shape of dataset\n",
    "valid_dataset = np.ndarray(shape = (3000*len(visible), image_size, image_size, 1), dtype = np.float32)\n",
    "counter = 0\n",
    "# REMEMBER to change shape of valid label\n",
    "valid_label = np.ndarray(shape = (3000*len(visible)), dtype = np.float32)\n",
    "for file in fn:\n",
    "#     image_data = ndimage.imread(dir+file).astype(float)\n",
    "    image_data = (ndimage.imread(dir+file).astype(float) - 255/2) / pixel_depth\n",
    "    label = int(file.split(\"_\")[0])\n",
    "    if label in visible:\n",
    "        valid_label[counter] = label\n",
    "        valid_dataset[counter, :, :] = image_data[:,:,0].reshape(image_size, image_size, 1)\n",
    "        counter+=1\n",
    "#     else:\n",
    "#         print(file[0])\n",
    "# shuffle dataset\n",
    "permutation = np.random.permutation(counter)\n",
    "valid_dataset = valid_dataset[permutation,:,:,:]\n",
    "valid_label = valid_label[permutation]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  4.   2.   4.   2.   4.   1.   4.   7.   1.   7.   2.   7.   1.   4.   7.\n",
      "   2.   1.   4.  14.   2.]\n",
      "[4.0, 2.0, 4.0, 2.0, 4.0, 1.0, 4.0, 7.0, 1.0, 7.0, 2.0, 7.0, 1.0, 4.0, 7.0, 2.0, 1.0, 4.0, 14.0, 2.0]\n",
      "accuracy: 0.944\n"
     ]
    }
   ],
   "source": [
    "offset = 0\n",
    "batch_data_all = valid_label\n",
    "results = []\n",
    "correct = 0\n",
    "for i in range(len(valid_label)):\n",
    "    batch_data = valid_dataset[offset+i:offset+i+1]\n",
    "    batch_labels = valid_label[offset+i:offset+i+1].reshape((batch_size, 1))\n",
    "    predictions = sess.run(output, feed_dict={train_data_node: batch_data})[0][0]\n",
    "    predictions = float(int(round(predictions)))\n",
    "    if predictions == batch_data_all[i]:\n",
    "        correct+=1\n",
    "    results.append(predictions)\n",
    "print(batch_data_all[100:120])\n",
    "print(results[100:120])\n",
    "print(\"accuracy: \"+str(correct/len(valid_label)))\n",
    "confusions = np.zeros([10, 10], np.float32)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}