Q-network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import gym\n",
    "import numpy as np\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-11-19 02:04:04,686] Making new env: Taxi-v2\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('Taxi-v2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "States: 500\tActions: 6\n"
     ]
    }
   ],
   "source": [
    "n_states = env.observation_space.n\n",
    "n_actions = env.action_space.n\n",
    "gamma = 0.99\n",
    "e = 0.1\n",
    "learning_rate = 1e-1\n",
    "episodes = 5000\n",
    "print(f'States: {n_states}\\tActions: {n_actions}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "X = tf.placeholder(shape=[1, n_states], dtype=tf.float32)\n",
    "y = tf.placeholder(shape=[1, n_actions],dtype=tf.float32)\n",
    "\n",
    "W = tf.Variable(tf.truncated_normal(shape=[n_states, n_actions], mean=0, stddev=0.5))\n",
    "b = tf.Variable(tf.zeros(shape=[n_actions]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Q_vals = tf.matmul(X, W) + b\n",
    "prediction = tf.argmax(Q_vals, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "loss = tf.reduce_sum(tf.squared_difference(y, Q_vals))\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
    "train = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "init = tf.global_variables_initializer()\n",
    "sess.run(init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 4,999\tRewards: -144\t%success: -11446.46%"
     ]
    }
   ],
   "source": [
    "reward_list = []\n",
    "episode_list = []\n",
    "for episode in range(episodes):\n",
    "    # reset agent's state\n",
    "    s = env.reset()\n",
    "    done = False\n",
    "    rewards, i = 0, 0\n",
    "    # The Q-Network\n",
    "    while i < 99:\n",
    "        i += 1\n",
    "        # Let the neural net predict an action 4u in ur current state\n",
    "        a, Q_old = sess.run([prediction, Q_vals], feed_dict={X: np.identity(n_states)[s:s+1]})\n",
    "        # take random actions every once in a while\n",
    "        if np.random.rand(1) < e:\n",
    "            a[0] = env.action_space.sample()\n",
    "        # take the action that transitions you to a new state (unlikely)\n",
    "        s1, r, done, _ = env.step(a[0])\n",
    "        # Get the Q values for the new state you are\n",
    "        Q_new = sess.run(Q_vals, feed_dict={X: np.identity(n_states)[s1:s1+1]})\n",
    "        # Update the previous state via Bellman's equation\n",
    "        target = np.copy(Q_old)\n",
    "        # Q[s, a] = r + ymax(Q[s'])\n",
    "        target[0, a[0]] = r + gamma*np.max(Q_new)  # new value\n",
    "        # Train the network using the predicted value and updated value\n",
    "        sess.run(train, feed_dict={X: np.identity(n_states)[s:s+1], y: target})\n",
    "        rewards += r\n",
    "        if done:\n",
    "            e = 1 / ((episode/50) + 10)  # reduce chance of random action as we train.\n",
    "            break\n",
    "    reward_list.append(rewards)\n",
    "    episode_list.append(i)\n",
    "    sys.stdout.write(f'\\rEpisode: {episode+1:,}\\tRewards: {rewards}\\t%success: {sum(reward_list)/episodes:.2%}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+\n",
      "|\u001b[34;1mR\u001b[0m: | : :\u001b[35mG\u001b[0m|\n",
      "| : : : : |\n",
      "| : : : :\u001b[43m \u001b[0m|\n",
      "| | : | : |\n",
      "|Y| : |B: |\n",
      "+---------+\n",
      "  (East)\n"
     ]
    }
   ],
   "source": [
    "env.render()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-90.59575653, -90.5747757 , -90.54410553, -90.56853485,\n",
       "        -98.0683136 , -99.59447479]], dtype=float32)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}