W2V_and_FastText_embeddings_both_dataset.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "W2V_and_FastText_embeddings_both_dataset.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "QF_-ZH_SYs6j",
        "iu3tX0TcEbID"
      ],
      "machine_shape": "hm",
      "background_execution": "on",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/larissapoghosyan/Capstone_Project/blob/main/W2V_and_FastText_embeddings_both_dataset.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GxVSwfQbvMHP"
      },
      "outputs": [],
      "source": [
        "from tqdm import tqdm\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import numpy\n",
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "import re\n",
        "import gensim\n",
        "from gensim.models import Word2Vec,  FastText, Doc2Vec, KeyedVectors\n",
        "from gensim import models\n",
        "import gensim.downloader as api\n",
        "import os, re, csv, math, codecs\n",
        "import h5py\n",
        "import nltk\n",
        "from nltk import ngrams, sent_tokenize, word_tokenize\n",
        "from nltk.stem import WordNetLemmatizer, LancasterStemmer, PorterStemmer\n",
        "from nltk.corpus import stopwords\n",
        "from sklearn import utils\n",
        "from sys import getsizeof\n",
        "nltk.download('punkt')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Reading the Dataset"
      ],
      "metadata": {
        "id": "G0sZFN5Rik1q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# dataset = pd.read_csv('/content/drive/MyDrive/Lariba/Intent/outofscope-intent-classification-dataset.csv')\n",
        "dataset = pd.read_csv('/content/IMDb_Reviews.csv',engine='python', error_bad_lines=False)\n",
        "dataset = dataset.dropna()\n",
        "print(\"Data Size: \", dataset.shape)\n",
        "dataset.iloc[:, 0] = dataset.iloc[:,0].apply(nltk.word_tokenize)"
      ],
      "metadata": {
        "id": "NqGog-VyAl47",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ed640c2b-23b1-4160-f311-3e341a9937f4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "Data Size:  (23700, 2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Get the w2v vectors"
      ],
      "metadata": {
        "id": "QF_-ZH_SYs6j"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "wv_model = api.load('word2vec-google-news-300')"
      ],
      "metadata": {
        "id": "79zybVphvNpb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "897fc159-0ba2-4a14-f110-2103c746b1e0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[==================================================] 100.0% 1662.8/1662.8MB downloaded\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# !wget -c \"https://s3.amazonaws.com/dl4j-distribution/GoogleNews-vectors-negative300.bin.gz\"\n",
        "# wv_model = gensim.models.KeyedVectors.load_word2vec_format('/content/GoogleNews-vectors-negative300.bin.gz', binary = True)"
      ],
      "metadata": {
        "id": "Se_pUVawBsKk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Get Fast Text vectors"
      ],
      "metadata": {
        "id": "GVQMEZ8KuZfo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Downloading fasttext word vectors\n",
        "!wget https://dl.fbaipublicfiles.com/fasttext/vectors-english/wiki-news-300d-1M.vec.zip\n",
        "!unzip wiki-news-300d-1M.vec.zip\n",
        "!ls"
      ],
      "metadata": {
        "id": "VIY8FxSmulED"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('loading word embeddings...')\n",
        "embeddings_index = {}\n",
        "f = codecs.open('/content/wiki-news-300d-1M.vec', encoding='utf-8')\n",
        "for line in tqdm(f):\n",
        "    values = line.rstrip().rsplit(' ')\n",
        "    word = values[0]\n",
        "    coefs = np.asarray(values[1:], dtype='float32')\n",
        "    embeddings_index[word] = coefs\n",
        "f.close()\n",
        "print('found %s word vectors' % len(embeddings_index))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8pW0XAQAulA7",
        "outputId": "c6681771-930b-4112-c70f-e1dd64817d8f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "loading word embeddings...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "999995it [01:08, 14679.71it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "found 999995 word vectors\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Extract FastText Embeddings"
      ],
      "metadata": {
        "id": "aW8aItU3vcWT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "new_dataset = []\n",
        "not_in_vocab = []\n",
        "dataset_vocab = []\n",
        "for id, sent in enumerate(dataset.iloc[:,0]):\n",
        "  new_sent = []\n",
        "  \n",
        "  for idx_, word in enumerate(sent):\n",
        "    if word in embeddings_index.keys():\n",
        "      new_sent.append(word)\n",
        "    else:\n",
        "      not_in_vocab.append(word)\n",
        "    dataset_vocab.append(word)\n",
        "  new_dataset.append(new_sent)\n",
        "\n",
        "print(len(new_dataset)) # 50000, ok\n",
        "new_dataset = np.array(new_dataset)\n"
      ],
      "metadata": {
        "id": "Ty-Ldyhyuk28"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_vectors = np.zeros((len(new_dataset), 300))\n",
        "for ix, data_point in enumerate(new_dataset):\n",
        "    for word in data_point :\n",
        "        data_vectors[ix, :]+=embeddings_index[word]\n",
        "    data_vectors[ix, :]/=len(data_point)"
      ],
      "metadata": {
        "id": "AR_ccnmJu0RV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "hf_ft = h5py.File('ft_vectors_intent.h5', 'w') # change intent to IMDb for IMDb datatset to avoid confusion\n",
        "vec_lis_hf = hf_ft.create_dataset('vec_lis_hf', data = data_vectors)\n",
        "hf_ft.close()"
      ],
      "metadata": {
        "id": "3RvgPa-au2Rq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Extract word2vec Embeddings"
      ],
      "metadata": {
        "id": "T24nZmGa8eHK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "new_dataset = []\n",
        "not_in_vocab = []\n",
        "dataset_vocab = []\n",
        "for id, sent in enumerate(dataset.iloc[:,0]):\n",
        "  new_sent = []\n",
        "  \n",
        "  for idx_, word in enumerate(sent):\n",
        "    if word in wv_model.vocab:\n",
        "      new_sent.append(word)\n",
        "    else:\n",
        "      not_in_vocab.append(word)\n",
        "    dataset_vocab.append(word)\n",
        "  #print(len(sent), len(new_sent))\n",
        "  new_dataset.append(new_sent)\n",
        "\n",
        "print(len(new_dataset)) # len in [50000, 23700] is ok\n",
        "new_dataset = np.array(new_dataset)\n"
      ],
      "metadata": {
        "id": "hsV0MgQBCYiT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_vectors = np.zeros((len(new_dataset), 300))\n",
        "for ix, data_point in enumerate(new_dataset):\n",
        "    for word in data_point :\n",
        "        data_vectors[ix, :]+=wv_model[word]\n",
        "    data_vectors[ix, :]/=len(data_point)\n",
        "data_vectors.shape"
      ],
      "metadata": {
        "id": "mT_DVEMJCq2n"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('\\n Number of words in the Dataset %s, \\n Number of words not in wv vocabulary %s, \\n Prc of words from Dataset not in the wv vpcabulary %s'\n",
        " % (len(dataset_vocab),\n",
        "    len(not_in_vocab),\n",
        "    (len(not_in_vocab)/len(dataset_vocab))*100)\n",
        " ) \n",
        "## (IMDB) 24% of words are not in the embeddings vector space\n",
        "## (INTENT) 10.98% of words are not in the embeddings vector space"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tZv5B06raX3w",
        "outputId": "0fd2c652-1cfc-4566-cac9-fa094946b401"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            " Number of words in the Dataset 202053, \n",
            " Number of words not in wv vocabulary 22186, \n",
            " Prc of words from Dataset not in the wv vpcabulary 10.98028735034867\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Save the embeddings in h5py files:"
      ],
      "metadata": {
        "id": "S5RArqP1ItGm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "hf_wv = h5py.File('wv_vectors_intent.h5', 'w') # change intent to IMDb for IMDb datatset to avoid confusion\n",
        "vec_lis_hf = hf_wv.create_dataset('vec_lis_hf', data = data_vectors)\n",
        "hf_wv.close()"
      ],
      "metadata": {
        "id": "HZFbAhkeAyTE"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}