codes/rnn/train_rnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import tensorflow.contrib.rnn as rnn\n",
    "import tensorflow.contrib.layers as layers\n",
    "from text_loader import TextLoader "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_layers  = 3\n",
    "hidden_size = 512\n",
    "batch_size  = 200\n",
    "max_length  = 30\n",
    "learning_rate = 0.001\n",
    "\n",
    "loader = TextLoader(\"data/hamlet.txt\")\n",
    "vocab_size = len(loader.vocab)\n",
    "\n",
    "if not os.path.exists(\"checkpoints/\"):\n",
    "    os.makedirs(\"checkpoints/\")    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.int32, [None, max_length])\n",
    "y = tf.placeholder(tf.int32, [None, max_length]) # [N, seqlne]\n",
    "\n",
    "x_one_hot = tf.one_hot(X, vocab_size)\n",
    "y_one_hot = tf.one_hot(y, vocab_size)            # [N, seqlen, vocab_size]\n",
    "\n",
    "cells = [rnn.BasicLSTMCell(hidden_size) for _ in range(num_layers)]\n",
    "cells = rnn.MultiRNNCell(cells, state_is_tuple=True)\n",
    "\n",
    "initial_state = cells.zero_state(batch_size, tf.float32)\n",
    "outputs, _ = tf.nn.dynamic_rnn(cells, x_one_hot, \n",
    "    initial_state=initial_state, dtype=tf.float32)\n",
    "\n",
    "outputs = tf.reshape(outputs, [-1, hidden_size]) # [N x seqlen, hidden]\n",
    "logits = layers.linear(outputs, vocab_size)      # [N x seqlen, vocab_size]\n",
    "y_flat = tf.reshape(y_one_hot, [-1, vocab_size]) # [N x seqlen, vocab_size]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss_op = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y_flat) # [N x seqlen]\n",
    "loss_op = tf.reduce_mean(loss_op)\n",
    "\n",
    "opt = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss_op)\n",
    "\n",
    "y_softmax = tf.nn.softmax(logits)         # [N x seqlen, vocab_size]\n",
    "pred = tf.argmax(y_softmax, axis=1)       # [N x seqlen]\n",
    "pred = tf.reshape(pred, [batch_size, -1]) # [N, seqlen]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00000500 step, loss:1.8634\n",
      "nd theeeee tn thes toalte \n",
      "\tAh\n",
      "00001000 step, loss:1.3568\n",
      "htoreng \n",
      "sieer ng  sr t lut to\n",
      "00001500 step, loss:0.9023\n",
      "n  t af bl aonlateral hand\n",
      "\tTh\n",
      "00002000 step, loss:0.8368\n",
      "td ttterdants]\n",
      "\n",
      "KING CLAUDIUS\t\n",
      "00002500 step, loss:0.5624\n",
      "er  thich tor to drevent,\n",
      "\tI h\n",
      "00003000 step, loss:0.5573\n",
      "o  \tToeak to me:\n",
      "\n",
      "If there be \n",
      "00003500 step, loss:0.4503\n",
      "AEUDE\tOid you assay him?\n",
      "\tTo a\n",
      "00004000 step, loss:0.4424\n",
      "h she cueen the e, ho!\n",
      "\n",
      "HORATI\n",
      "00004500 step, loss:0.4331\n",
      "\t\t[Eetireng with HORATIO]\n",
      "\n",
      "LAE\n",
      "00005000 step, loss:0.3887\n",
      "l  ty dlessing season this in \n",
      "00005500 step, loss:0.4157\n",
      "ty sord,\n",
      "\n",
      "HORD POLONIUS\tAarewe\n",
      "00006000 step, loss:0.4259\n",
      " late is valenced since I saw \n",
      "00006500 step, loss:0.3739\n",
      "i ap another skull]\n",
      "\n",
      "HAMLET\tTh\n",
      "00007000 step, loss:0.3615\n",
      "td r  will.\n",
      "\n",
      "QUEEN GERTRUDE\tO \n",
      "00007500 step, loss:0.4061\n",
      "tnd tUILDENSTERN]\n",
      "\n",
      "KAMLET\tWhat\n",
      "00008000 step, loss:0.2965\n",
      " toes,\n",
      "\tI will be brief: your \n",
      "00008500 step, loss:0.3789\n",
      "h teave betimes?\n",
      "\n",
      "\t[Enter KING\n",
      "00009000 step, loss:0.3599\n",
      "has mast ne known; which, bein\n",
      "00009500 step, loss:0.3656\n",
      "ne tearen?\n",
      "\tBut in our circums\n",
      "00010000 step, loss:0.3399\n",
      "teood,is tame, it's humble,\n",
      "\tA\n",
      "00010500 step, loss:0.3423\n",
      " e  wne teceived.\n",
      "\n",
      "GUILDENSTER\n",
      "00011000 step, loss:0.3790\n",
      "esh tur valiant Hamlet--\n",
      "\tFor \n",
      "00011500 step, loss:0.3686\n",
      "eat aorm of the other,\n",
      "\tSubscr\n",
      "00012000 step, loss:0.3203\n",
      "eore rank, of midnight weeds c\n",
      "00012500 step, loss:0.3343\n",
      " tather'lost a father;\n",
      "\tThat f\n",
      "00013000 step, loss:0.3806\n",
      "\tAith gy disposition that this\n",
      "00013500 step, loss:0.3857\n",
      "teded\n",
      "\tThat from a shelf the p\n",
      "00014000 step, loss:0.3352\n",
      "eaasure  more into command\n",
      "\tTh\n",
      "00014500 step, loss:0.3340\n",
      "tosy,\n",
      "\n",
      "QUEEN GERTRUDE\tWhat sha\n",
      "00015000 step, loss:0.3015\n",
      "toty be too bold, my love is t\n",
      "00015500 step, loss:0.3755\n",
      "  aome, nay, speak.\n",
      "\n",
      "GUILDENST\n",
      "00016000 step, loss:0.3537\n",
      " t otract and brief chronicles\n",
      "00016500 step, loss:0.3836\n",
      "nl ne ohme danger: which for t\n",
      "00017000 step, loss:0.3515\n",
      "  aytrs, shapes of grief,\n",
      "\tTha\n",
      "00017500 step, loss:0.3175\n",
      "tis aast fit;\n",
      "\tBut, like the o\n",
      "00018000 step, loss:0.3740\n",
      "uth tedk the Lord Hamlet; ther\n",
      "00018500 step, loss:0.3665\n",
      " tf tiaven.\n",
      "\n",
      "LORD POLONIUS\tAy,\n",
      "00019000 step, loss:0.3289\n",
      "u \n",
      "\n",
      "HAMLET\tMou are welcome.\n",
      "\n",
      "G\n",
      "00019500 step, loss:0.3216\n",
      "itd oo a plurisy,\n",
      "\tDies in his\n",
      "00020000 step, loss:0.3404\n",
      "h ahiugstrumpet, Fortune! All \n",
      "00020500 step, loss:0.3333\n",
      "  \tTnd prologue to the omen co\n",
      "00021000 step, loss:0.3195\n",
      "et weve virtue\n",
      "\tUnder the moon\n",
      "00021500 step, loss:0.3273\n",
      "ethbk'd of you;\n",
      "\tAnd sure I am\n",
      "00022000 step, loss:0.3643\n",
      "That Irows to seed; things ran\n",
      "00022500 step, loss:0.3613\n",
      "tnd tomntry.en.--\n",
      "\tBut soft, b\n",
      "00023000 step, loss:0.3198\n",
      "e la!\n",
      "\n",
      "O heavens! is't possibl\n",
      "00023500 step, loss:0.3176\n",
      "  uls, though none else near.\n",
      "\n",
      "00024000 step, loss:0.3221\n",
      "toyntl O cursed spite,\n",
      "\tThat e\n",
      "00024500 step, loss:0.3164\n",
      " e coul, and sweet religion ma\n",
      "00025000 step, loss:0.3237\n",
      "tnsom?\n",
      "\n",
      "HAMLET\tI humbly thank \n",
      "00025500 step, loss:0.3250\n",
      "Gre wut his enemies.\n",
      "\n",
      "KING CLA\n",
      "00026000 step, loss:0.3691\n",
      " aousies, and long purples\n",
      "\tTh\n",
      "00026500 step, loss:0.3286\n",
      "e  ahet duty is,\n",
      "\tWhy day is d\n",
      "00027000 step, loss:0.3292\n",
      "TCGERTRUDE\t Pnd\n",
      "\tAttendants]\n",
      "\n",
      "\n",
      "00027500 step, loss:0.2868\n",
      "urd  changes\n",
      "\tAnd hath abateme\n",
      "00028000 step, loss:0.3126\n",
      "  tejicate carriages,\n",
      "\tand of \n",
      "00028500 step, loss:0.3010\n",
      "hr HAMLET] ROSENCRANTZ, GUILDE\n",
      "00029000 step, loss:0.3304\n",
      "e aousin, and our son.\n",
      "\n",
      "QUEEN \n",
      "00029500 step, loss:0.3280\n",
      "too .'\n",
      "\n",
      "LORD POLONIUS\tLook, wh\n",
      "00030000 step, loss:0.3174\n",
      "loro--\n",
      "\tHamlet comes back: wha\n"
     ]
    }
   ],
   "source": [
    "saver = tf.train.Saver()\n",
    "\n",
    "sess_config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))\n",
    "with tf.Session(config=sess_config) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    for step in range(30000):\n",
    "        batch_X, batch_y = loader.next_batch(batch_size, max_length)\n",
    "        loss, _ = sess.run([loss_op, opt], feed_dict={X: batch_X, y: batch_y})\n",
    "        \n",
    "        if (step+1) % 500 == 0:\n",
    "            print(\"{:08d} step, loss:{:.4f}\".format(step+1, loss))\n",
    "            \n",
    "            random = np.random.randint(0, batch_size)\n",
    "            results = sess.run(pred, feed_dict={X: batch_X})\n",
    "            words = [loader.words[word] for word in results[random]]\n",
    "            print(\"\".join(words))\n",
    "     \n",
    "        if (step+1) % 5000 == 0: \n",
    "            saver.save(sess, \"checkpoints/char-rnn_\"+str(step+1))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}