chapter_12_predicting_next_word.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('reviews.txt','r')\n",
    "raw_reviews = f.readlines()\n",
    "f.close\n",
    "\n",
    "f = open(\"labels.txt\",'r')\n",
    "raw_labels = f.readlines()\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = list(map(lambda x:set(x.split(\" \")),raw_reviews))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = set()\n",
    "\n",
    "for token in tokens:\n",
    "    for word in token:\n",
    "        if(len(word) > 0):\n",
    "            vocab.add(word)\n",
    "vocab = list(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2index = {}\n",
    "for i,word in enumerate(vocab):\n",
    "    word2index[word] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dataset = []\n",
    "\n",
    "for sent in tokens:\n",
    "    sentence = []\n",
    "    for word in sent:\n",
    "        if len(word) > 0:            \n",
    "            sentence.append(word2index[word])\n",
    "    input_dataset.append(sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25000"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(input_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_dataset = []\n",
    "for score in raw_labels:\n",
    "    if(score == \"positive\\n\"):\n",
    "        target_dataset.append(1)\n",
    "    else:\n",
    "        target_dataset.append(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights_0_1 = np.load('weights/c_11.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "norms = np.sum(weights_0_1 * weights_0_1,axis=1)\n",
    "norms.resize(norms.shape[0],1)\n",
    "normed_weights = weights_0_1 * norms\n",
    "\n",
    "def make_sent_vect(words):\n",
    "    indices = list(map(lambda x:word2index[x],filter(lambda x:x in word2index,words)))\n",
    "    return np.mean(normed_weights[indices],axis=0)\n",
    "\n",
    "reviews2vectors = list()\n",
    "for review in tokens: # tokenized reviews\n",
    "    reviews2vectors.append(make_sent_vect(review))\n",
    "reviews2vectors = np.array(reviews2vectors)\n",
    "\n",
    "def most_similar_reviews(review):\n",
    "    v = make_sent_vect(review)\n",
    "    scores = Counter()\n",
    "    for i,val in enumerate(reviews2vectors.dot(v)):\n",
    "        scores[i] = val\n",
    "    most_similar = list()\n",
    "    \n",
    "    for idx,score in scores.most_common(3):\n",
    "        most_similar.append(raw_reviews[idx][0:40])\n",
    "    return most_similar\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['at a panel discussion that i attended af',\n",
       " 'photography was too jumpy to follow . da',\n",
       " 'my short comment for this flick is go pi']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "most_similar_reviews(['awful'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sentence embedding using identity matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.array([1,2,3])\n",
    "b = np.array([0.1,0.2,0.3])\n",
    "c = np.array([-1,-0.5,0])\n",
    "d = np.array([0,0,0])\n",
    "\n",
    "identity = np.eye(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1. 0. 0.]\n",
      " [0. 1. 0.]\n",
      " [0. 0. 1.]]\n"
     ]
    }
   ],
   "source": [
    "print(identity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1. 2. 3.]\n",
      "[0.1 0.2 0.3]\n",
      "[-1.  -0.5  0. ]\n"
     ]
    }
   ],
   "source": [
    "print(a.dot(identity))\n",
    "print(b.dot(identity))\n",
    "print(c.dot(identity))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[13 15 17]\n",
      "[13. 15. 17.]\n"
     ]
    }
   ],
   "source": [
    "this = np.array([2,4,6])\n",
    "movie = np.array([10,10,10])\n",
    "rocks = np.array([1,1,1])\n",
    "\n",
    "print(this + movie + rocks)\n",
    "print((this.dot(identity) + movie).dot(identity) + rocks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# forward probagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(x_):\n",
    "    x = np.atleast_2d(x_)\n",
    "    temp = np.exp(x)\n",
    "    return temp / np.sum(temp, axis=1, keepdims=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_vects = {}\n",
    "word_vects['yankees'] = np.array([[0.,0.,0.]]) # word embeddings\n",
    "word_vects['bears'] = np.array([[0.,0.,0.]])\n",
    "word_vects['braves'] = np.array([[0.,0.,0.]])\n",
    "word_vects['red'] = np.array([[0.,0.,0.]])\n",
    "word_vects['sox'] = np.array([[0.,0.,0.]])\n",
    "word_vects['lose'] = np.array([[0.,0.,0.]])\n",
    "word_vects['defeat'] = np.array([[0.,0.,0.]])\n",
    "word_vects['beat'] = np.array([[0.,0.,0.]])\n",
    "word_vects['tie'] = np.array([[0.,0.,0.]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sentence embedding to output classification weights\n",
    "sent2output = np.random.rand(3,len(word_vects))\n",
    "identity = np.eye(3) # transition weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# “red sox defeat” -> “yankees”\n",
    "# forward probagation\n",
    "layer_0 = word_vects['red']\n",
    "layer_1 = layer_0.dot(identity) + word_vects['sox'] # create sentence embedding\n",
    "layer_2 = layer_1.dot(identity) + word_vects['defeat']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.11111111 0.11111111 0.11111111 0.11111111 0.11111111 0.11111111\n",
      "  0.11111111 0.11111111 0.11111111]]\n",
      "[[0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "pred = softmax(layer_2.dot(sent2output)) # predicts all over the vocab\n",
    "print(pred)\n",
    "print(word_vects['red'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# backprobagate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = np.array([1,0,0,0,0,0,0,0,0]) # one hot vector for yankess\n",
    "pred_delta = pred - y\n",
    "layer_2_delta = pred_delta.dot(sent2output.T)\n",
    "defeat_delta = layer_2_delta * 1\n",
    "layer_1_delta = layer_2_delta.dot(identity.T)\n",
    "sox_delta = layer_1_delta * 1\n",
    "layer_0_delta = layer_1_delta.dot(identity.T)\n",
    "alpha = 0.01\n",
    "\n",
    "word_vects['red'] -= layer_0_delta * alpha\n",
    "word_vects['sox'] -= sox_delta * alpha\n",
    "word_vects['defeat'] -= defeat_delta * alpha\n",
    "\n",
    "identity -= np.outer(layer_0,layer_1_delta) * alpha\n",
    "identity -= np.outer(layer_1, layer_2_delta) * alpha\n",
    "sent2output -= np.outer(layer_2,pred_delta) * alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# time for real shit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 4  5  6]\n",
      " [ 8 10 12]\n",
      " [12 15 18]]\n",
      "----------\n",
      "32\n",
      "[ 4 10 18]\n",
      "32\n"
     ]
    }
   ],
   "source": [
    "x = np.array([1,2,3])\n",
    "xx = np.array([4,5,6])\n",
    "\n",
    "print(np.outer(x,xx))\n",
    "\n",
    "print('----------')\n",
    "print(np.inner(x,xx))\n",
    "print(x * xx)\n",
    "print(14+18)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5 7 9]\n",
      "[[1 2 3]\n",
      " [4 5 6]]\n"
     ]
    }
   ],
   "source": [
    "x = np.array([[1,2,3],[4,5,6]])\n",
    "\n",
    "print(np.sum(x,axis=0))\n",
    "print(x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 6 15]\n",
      "[[1 2 3]\n",
      " [4 5 6]]\n"
     ]
    }
   ],
   "source": [
    "print(np.sum(x,axis=1))\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys,random,math\n",
    "from collections import Counter\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('data/tasksv11/en/qa1_single-supporting-fact_train.txt','r')\n",
    "raw = f.readlines()\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['mary', 'moved', 'to', 'the', 'bathroom.'], ['john', 'went', 'to', 'the', 'hallway.'], ['where', 'is', 'mary?', '\\tbathroom\\t1'], ['daniel', 'went', 'back', 'to', 'the', 'hallway.']]\n"
     ]
    }
   ],
   "source": [
    "tokens = []\n",
    "for line in raw[:1000]:\n",
    "    tokens.append(line.lower().replace(\"\\n\",'').split(' ')[1:])\n",
    "print(tokens[0:4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = set()\n",
    "for sent in tokens:\n",
    "    for word in sent:\n",
    "        vocab.add(word)\n",
    "\n",
    "vocab = list(vocab)\n",
    "\n",
    "word2index = {}\n",
    "for i,word in enumerate(vocab):\n",
    "    word2index[word]=i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def word2indices(sentence):\n",
    "    idx = list()\n",
    "    for word in sentence:\n",
    "        idx.append(word2index[word])\n",
    "    return idx\n",
    "\n",
    "def softmax(x):\n",
    "    e_x = np.exp(x - np.max(x))\n",
    "    return e_x / e_x.sum(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(1)\n",
    "embed_size = 10\n",
    "\n",
    "# word embeddings\n",
    "embed = (np.random.rand(len(vocab),embed_size) - 0.5) * 0.1\n",
    "\n",
    "# embedding -> embedding (initially the identity matrix)\n",
    "recurrent = np.eye(embed_size)\n",
    "\n",
    "# sentence embedding for empty sentence\n",
    "start = np.zeros(embed_size)\n",
    "\n",
    "# embedding -> output weights\n",
    "decoder = (np.random.rand(embed_size, len(vocab)) - 0.5) * 0.1\n",
    "\n",
    "# one hot lookups (for loss function)\n",
    "one_hot = np.eye(len(vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(sent):\n",
    "    \n",
    "    layers = list()\n",
    "    layer = {}\n",
    "    layer['hidden'] = start\n",
    "    layers.append(layer)\n",
    "    \n",
    "    loss = 0\n",
    "    preds = [] # forward probagation\n",
    "    for target_i in range(len(sent)):\n",
    "        layer = {}\n",
    "        # predict next term\n",
    "        layer['pred'] = softmax(layers[-1]['hidden'].dot(decoder))\n",
    "        loss += -np.log(layer['pred'][sent[target_i]])\n",
    "        #generate next hidden state\n",
    "        layer['hidden'] = layers[-1]['hidden'].dot(recurrent) +\\\n",
    "                            embed[sent[target_i]]\n",
    "        layers.append(layer)\n",
    "    return layers, loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Perplexity: 82.01779692170828\n",
      "Perplexity: 81.73470648410135\n",
      "Perplexity: 81.35553094905504\n",
      "Perplexity: 80.72991918489456\n",
      "Perplexity: 79.55243749150551\n",
      "Perplexity: 77.03505237345513\n",
      "Perplexity: 70.27199258597715\n",
      "Perplexity: 42.242267815761544\n",
      "Perplexity: 23.70721676631971\n",
      "Perplexity: 19.816426169789054\n",
      "Perplexity: 18.527723962281033\n",
      "Perplexity: 17.312146066720306\n",
      "Perplexity: 15.561221454971136\n",
      "Perplexity: 12.819843954001655\n",
      "Perplexity: 9.526145929743759\n",
      "Perplexity: 7.4141972341255595\n",
      "Perplexity: 6.373127080517985\n",
      "Perplexity: 5.659862351996081\n",
      "Perplexity: 5.180206882104858\n",
      "Perplexity: 4.895841973175592\n",
      "Perplexity: 4.703595379698135\n",
      "Perplexity: 4.573120558832889\n",
      "Perplexity: 4.492772790751694\n",
      "Perplexity: 4.4423439237315625\n",
      "Perplexity: 4.396430529486272\n",
      "Perplexity: 4.341330964755117\n",
      "Perplexity: 4.27714591602516\n",
      "Perplexity: 4.212008611316266\n",
      "Perplexity: 4.156077957531741\n",
      "Perplexity: 4.11730670251008\n"
     ]
    }
   ],
   "source": [
    "for iter in range(30000): # forward\n",
    "    alpha = 0.001\n",
    "    \n",
    "    sent = word2indices(tokens[iter%len(tokens)][1:])\n",
    "    layers, loss = predict(sent)\n",
    "    \n",
    "    for layer_idx in reversed(range(len(layers))): # backprobagate\n",
    "        layer = layers[layer_idx]\n",
    "        \n",
    "        target = sent[layer_idx-1]\n",
    "        \n",
    "        if(layer_idx > 0): # if not the first layer\n",
    "            layer['output_delta'] = layer['pred'] - one_hot[target]\n",
    "            new_hidden_delta = layer['output_delta'].dot(decoder.transpose())\n",
    "            \n",
    "            if(layer_idx == len(layers) -1): # if not the last layer of the list, 1 before last\n",
    "                layer['hidden_delta'] = new_hidden_delta\n",
    "            else:\n",
    "                layer['hidden_delta'] = new_hidden_delta +\\\n",
    "                layers[layer_idx +1]['hidden_delta'].dot(recurrent.transpose())\n",
    "        \n",
    "        else: # if the first layer\n",
    "            layer['hidden_delta'] = layers[layer_idx+1]['hidden_delta'].dot(recurrent.transpose())\n",
    "    \n",
    "    # update weights\n",
    "    \n",
    "    start -= layers[0]['hidden_delta'] * alpha / float(len(sent))\n",
    "    \n",
    "    for layer_idx, layer in enumerate(layers[1:]):\n",
    "        \n",
    "        decoder -= np.outer(layers[layer_idx]['hidden'],layer['output_delta']) * \\\n",
    "                           alpha  / float(len(sent)) # layer_2 , pred_delta\n",
    "        embed_idx = sent[layer_idx]\n",
    "        embed[embed_idx] -= layers[layer_idx]['hidden_delta'] * \\\n",
    "                            alpha / float(len(sent)) # layer_1 layer_2 delta\n",
    "        recurrent -= np.outer(layers[layer_idx] ['hidden'],\\\n",
    "                                layer['hidden_delta']) * alpha / float(len(sent))\n",
    "                                # layer_0, layer_1_delta\n",
    "    if(iter % 1000 == 0):\n",
    "        print(\"Perplexity: \"+str(np.exp(loss/len(sent))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_sentence(sent_index):\n",
    "    for i,each_layer in enumerate(l[1:-1]):\n",
    "        input = tokens[sent_index][i]\n",
    "        true = tokens[sent_index][i+1]\n",
    "        pred = vocab[each_layer['pred'].argmax()]\n",
    "        print(\"Prev Input: \"+input + (' ' * (12 - len(input)))+\\\n",
    "             \"True: \"+true+ (\" \"*(15-len(true))) + ' Pred: '+pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['mary', 'moved', 'to', 'the', 'kitchen.']\n",
      "Prev Input: mary        True: moved           Pred: is\n",
      "Prev Input: moved       True: to              Pred: to\n",
      "Prev Input: to          True: the             Pred: the\n",
      "Prev Input: the         True: kitchen.        Pred: bedroom.\n"
     ]
    }
   ],
   "source": [
    "sent_index = random.randrange(1,1000)\n",
    "\n",
    "l,_ = predict(word2indices(tokens[sent_index]))\n",
    "\n",
    "print(tokens[sent_index])\n",
    "\n",
    "predict_sentence(sent_index)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}