Transformers_BERT_embeddings_both_datasets.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/larissapoghosyan/Capstone_Project/blob/main/Transformers_BERT_embeddings_both_datasets.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Import libraries"
      ],
      "metadata": {
        "id": "v56LQAyPHQM2"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iVIkEP8J1rkl"
      },
      "outputs": [],
      "source": [
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "so07TvX7B1-F"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import transformers\n",
        "from transformers import BertTokenizer, BertModel\n",
        "from transformers import RobertaTokenizer, RobertaModel\n",
        "from transformers import AlbertModel, AlbertTokenizer\n",
        "from transformers import DistilBertModel, DistilBertTokenizer\n",
        "from transformers import  AutoModel, AutoTokenizer\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "import math\n",
        "import h5py\n",
        "import os\n",
        "import copy\n",
        "from tqdm import tqdm\n",
        "import time\n",
        "\n",
        "import warnings\n",
        "\n",
        "torch.cuda.empty_cache()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6y-RjasXjd_E"
      },
      "outputs": [],
      "source": [
        "!/usr/local/cuda/bin/nvcc --version\n",
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bZ25bl9-aTYe",
        "outputId": "ca87d6cd-1c41-4ae9-f708-80a5373d6e4b"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "torch.cuda.is_available()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gUDtTcBH5BsF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dbc041ac-9ea4-4b4d-b7e9-60ca5f24ad66"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py:2882: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version.\n",
            "\n",
            "\n",
            "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
          ]
        }
      ],
      "source": [
        "data = pd.read_csv('/content/IMDb_Reviews.csv',engine='python', error_bad_lines=False)\n",
        "# data = pd.read_csv('/content/outofscope-intent-classification-dataset.csv')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data.shape"
      ],
      "metadata": {
        "id": "D4lPp5ZhGz4U",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "43a3a931-b8e4-4186-be44-96c33a9fda3c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(50000, 2)"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Label Encoder:\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "le = LabelEncoder()\n",
        "le_fitted = le.fit_transform(data.iloc[:,1])\n",
        "data.iloc[:,1] = le_fitted\n",
        "label_col = np.array(data.iloc[:,1]).reshape(len(data),1)\n",
        "label_col = label_col.astype(int)\n",
        "np.unique(label_col)"
      ],
      "metadata": {
        "id": "_WtjihO2EIp4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6d7a5bd5-0deb-48b9-c9e2-46c5c23d6928"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 1])"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Model Config"
      ],
      "metadata": {
        "id": "DkeLdcLh1pwT"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vE2364fr5Rkn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b3ebe9ad-c5cd-47e2-f517-d0e84d73677f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaModel: ['lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of the model checkpoint at albert-base-v2 were not used when initializing AlbertModel: ['predictions.LayerNorm.bias', 'predictions.bias', 'predictions.dense.bias', 'predictions.decoder.weight', 'predictions.LayerNorm.weight', 'predictions.dense.weight', 'predictions.decoder.bias']\n",
            "- This IS expected if you are initializing AlbertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing AlbertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_projector.bias', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_transform.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight']\n",
            "- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        }
      ],
      "source": [
        "\n",
        "## Initializing models for BERT and other variants of BERT, \n",
        "## Defining output files according to output type\n",
        "models_config = {\n",
        "    'BERT': {\n",
        "        'tokenizer' : AutoTokenizer.from_pretrained('bert-base-uncased'),\n",
        "        'model': AutoModel.from_pretrained('bert-base-uncased',\n",
        "                                        output_attentions = True,\n",
        "                                        output_hidden_states = True\n",
        "                                        ),\n",
        "        'output_type': ['pooler_output',\n",
        "                        'token_avg',\n",
        "                        'cls_last_hid',\n",
        "                        'n_lyrs_cat',\n",
        "                        'n_lyrs_cat_tkn_avg'],\n",
        "        'output_file': ['bert_imdb_pooler',\n",
        "                        'bert_imdb_token_avg',\n",
        "                        'bert_imdb_cls_last_hid',\n",
        "                        'bert_imdb_CLS_cat',\n",
        "                        'bert_imdb_tkn_cat']\n",
        "        },\n",
        "\n",
        "    'RoBERT': {\n",
        "        'tokenizer' : AutoTokenizer.from_pretrained('roberta-base'),\n",
        "        'model': AutoModel.from_pretrained('roberta-base',\n",
        "                                        output_attentions = True,\n",
        "                                        output_hidden_states = True\n",
        "                                        ),\n",
        "        'output_type': ['pooler_output',\n",
        "                        'token_avg',\n",
        "                        'cls_last_hid',\n",
        "                        'n_lyrs_cat',\n",
        "                        'n_lyrs_cat_tkn_avg'],\n",
        "        'output_file': ['robert_imdb_pooler',\n",
        "                        'robert_imdb_token_avg',\n",
        "                        'robert_imdb_cls_last_hid',\n",
        "                        'robert_imdb_CLS_cat',\n",
        "                        'robert_imdb_tkn_cat']\n",
        "        },\n",
        "\n",
        "    'AlBERT': {\n",
        "        'tokenizer' : AutoTokenizer.from_pretrained('albert-base-v2'),\n",
        "        'model': AutoModel.from_pretrained('albert-base-v2',\n",
        "                                        output_attentions = True,\n",
        "                                        output_hidden_states = True\n",
        "                                        ),\n",
        "        'output_type': ['pooler_output',\n",
        "                        'token_avg',\n",
        "                        'cls_last_hid',\n",
        "                        'n_lyrs_cat',\n",
        "                        'n_lyrs_cat_tkn_avg'],\n",
        "        'output_file': ['albert_imdb_pooler',\n",
        "                        'albert_imdb_token_avg',\n",
        "                        'albert_imdb_cls_last_hid',\n",
        "                        'albert_imdb_CLS_cat',\n",
        "                        'albert_imdb_tkn_cat']\n",
        "\n",
        "        },\n",
        "    'DistilBERT': {\n",
        "        'tokenizer' : AutoTokenizer.from_pretrained('distilbert-base-uncased'),\n",
        "        'model': AutoModel.from_pretrained('distilbert-base-uncased',\n",
        "                                        output_attentions = True,\n",
        "                                        output_hidden_states = True\n",
        "                                        ),\n",
        "        'output_type': ['pooler_output',\n",
        "                        'token_avg',\n",
        "                        'cls_last_hid',\n",
        "                        'n_lyrs_cat',\n",
        "                        'n_lyrs_cat_tkn_avg'],\n",
        "        'output_file': ['distilbert_imdb_pooler',\n",
        "                        'distilbert_imdb_token_avg',\n",
        "                        'distilbert_imdb_cls_last_hid',\n",
        "                        'distilbert_imdb_CLS_cat',\n",
        "                        'distilbert_imdb_tkn_cat']\n",
        "\n",
        "        },\n",
        "    'TinyBERT': {\n",
        "        'tokenizer' : AutoTokenizer.from_pretrained('sentence-transformers/paraphrase-TinyBERT-L6-v2'),\n",
        "        'model': AutoModel.from_pretrained('sentence-transformers/paraphrase-TinyBERT-L6-v2',\n",
        "                                        output_attentions = True,\n",
        "                                        output_hidden_states = True\n",
        "                                        ),\n",
        "        'output_type': ['pooler_output',\n",
        "                        'token_avg',\n",
        "                        'cls_last_hid',\n",
        "                        'n_lyrs_cat',\n",
        "                        'n_lyrs_cat_tkn_avg'],\n",
        "        'output_file': ['tinybert_imdb_pooler',\n",
        "                        'tinybert_imdb_token_avg',\n",
        "                        'tinybert_imdb_cls_last_hid',\n",
        "                        'tinybert_imdb_CLS_cat',\n",
        "                        'tinybert_imdb_tkn_cat']\n",
        "\n",
        "        },\n",
        "    'Sentence-BERT': {\n",
        "        'tokenizer' : AutoTokenizer.from_pretrained('sentence-transformers/bert-base-nli-mean-tokens'),\n",
        "        'model': AutoModel.from_pretrained('sentence-transformers/bert-base-nli-mean-tokens',\n",
        "                                        output_attentions = True,\n",
        "                                        output_hidden_states = True\n",
        "                                        ),\n",
        "        'output_type': ['pooler_output',\n",
        "                        'token_avg',\n",
        "                        'cls_last_hid',\n",
        "                        'n_lyrs_cat',\n",
        "                        'n_lyrs_cat_tkn_avg'],\n",
        "        'output_file': ['sentence_bert_imdb_pooler',\n",
        "                        'sentence_bert_imdb_token_avg',\n",
        "                        'sentence_bert_imdb_cls_last_hid',\n",
        "                        'sentence_bert_imdb_CSL_cat',\n",
        "                        'sentence_bert_imdb_tkn_cat']\n",
        "\n",
        "        }\n",
        "      }"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "models_config_intent = {\n",
        "    'BERT': {\n",
        "        'tokenizer' : AutoTokenizer.from_pretrained('bert-base-uncased'),\n",
        "        'model': AutoModel.from_pretrained('bert-base-uncased',\n",
        "                                        output_attentions = True,\n",
        "                                        output_hidden_states = True\n",
        "                                        ),\n",
        "        'output_type': ['pooler_output',\n",
        "                        'token_avg',\n",
        "                        'cls_last_hid',\n",
        "                        'n_lyrs_cat',\n",
        "                        'n_lyrs_cat_tkn_avg'],\n",
        "        'output_file': ['bert_intent_pooler',\n",
        "                        'bert_intent_token_avg',\n",
        "                        'bert_intent_cls_last_hid',\n",
        "                        'bert_intent_CLS_cat',\n",
        "                        'bert_intent_tkn_cat']\n",
        "        },\n",
        "\n",
        "    'RoBERT': {\n",
        "        'tokenizer' : AutoTokenizer.from_pretrained('roberta-base'),\n",
        "        'model': AutoModel.from_pretrained('roberta-base',\n",
        "                                        output_attentions = True,\n",
        "                                        output_hidden_states = True\n",
        "                                        ),\n",
        "        'output_type': ['pooler_output',\n",
        "                        'token_avg',\n",
        "                        'cls_last_hid',\n",
        "                        'n_lyrs_cat',\n",
        "                        'n_lyrs_cat_tkn_avg'],\n",
        "        'output_file': ['robert_intent_pooler',\n",
        "                        'robert_intent_token_avg',\n",
        "                        'robert_intent_cls_last_hid',\n",
        "                        'robert_intent_CLS_cat',\n",
        "                        'robert_intent_tkn_cat']\n",
        "        },\n",
        "\n",
        "    'AlBERT': {\n",
        "        'tokenizer' : AutoTokenizer.from_pretrained('albert-base-v2'),\n",
        "        'model': AutoModel.from_pretrained('albert-base-v2',\n",
        "                                        output_attentions = True,\n",
        "                                        output_hidden_states = True\n",
        "                                        ),\n",
        "        'output_type': ['pooler_output',\n",
        "                        'token_avg',\n",
        "                        'cls_last_hid',\n",
        "                        'n_lyrs_cat',\n",
        "                        'n_lyrs_cat_tkn_avg'],\n",
        "        'output_file': ['albert_intent_pooler',\n",
        "                        'albert_intent_token_avg',\n",
        "                        'albert_intent_cls_last_hid',\n",
        "                        'albert_intent_CLS_cat',\n",
        "                        'albert_intent_tkn_cat']\n",
        "\n",
        "        },\n",
        "    'DistilBERT': {\n",
        "        'tokenizer' : AutoTokenizer.from_pretrained('distilbert-base-uncased'),\n",
        "        'model': AutoModel.from_pretrained('distilbert-base-uncased',\n",
        "                                        output_attentions = True,\n",
        "                                        output_hidden_states = True\n",
        "                                        ),\n",
        "        'output_type': ['pooler_output',\n",
        "                        'token_avg',\n",
        "                        'cls_last_hid',\n",
        "                        'n_lyrs_cat',\n",
        "                        'n_lyrs_cat_tkn_avg'],\n",
        "        'output_file': ['distilbert_intent_pooler',\n",
        "                        'distilbert_intent_token_avg',\n",
        "                        'distilbert_intent_cls_last_hid',\n",
        "                        'distilbert_intent_CLS_cat',\n",
        "                        'distilbert_intent_tkn_cat']\n",
        "\n",
        "        },\n",
        "    'TinyBERT': {\n",
        "        'tokenizer' : AutoTokenizer.from_pretrained('sentence-transformers/paraphrase-TinyBERT-L6-v2'),\n",
        "        'model': AutoModel.from_pretrained('sentence-transformers/paraphrase-TinyBERT-L6-v2',\n",
        "                                        output_attentions = True,\n",
        "                                        output_hidden_states = True\n",
        "                                        ),\n",
        "        'output_type': ['pooler_output',\n",
        "                        'token_avg',\n",
        "                        'cls_last_hid',\n",
        "                        'n_lyrs_cat',\n",
        "                        'n_lyrs_cat_tkn_avg'],\n",
        "        'output_file': ['tinybert_intent_pooler',\n",
        "                        'tinybert_intent_token_avg',\n",
        "                        'tinybert_intent_cls_last_hid',\n",
        "                        'tinybert_intent_CLS_cat',\n",
        "                        'tinybert_intent_tkn_cat']\n",
        "\n",
        "        },\n",
        "    'Sentence-BERT': {\n",
        "        'tokenizer' : AutoTokenizer.from_pretrained('sentence-transformers/bert-base-nli-mean-tokens'),\n",
        "        'model': AutoModel.from_pretrained('sentence-transformers/bert-base-nli-mean-tokens',\n",
        "                                        output_attentions = True,\n",
        "                                        output_hidden_states = True\n",
        "                                        ),\n",
        "        'output_type': ['pooler_output',\n",
        "                        'token_avg',\n",
        "                        'cls_last_hid',\n",
        "                        'n_lyrs_cat',\n",
        "                        'n_lyrs_cat_tkn_avg'],\n",
        "        'output_file': ['sentence_bert_intent_pooler',\n",
        "                        'sentence_bert_intent_token_avg',\n",
        "                        'sentence_bert_intent_cls_last_hid',\n",
        "                        'sentence_bert_intent_CSL_cat',\n",
        "                        'sentence_bert_intent_tkn_cat']\n",
        "\n",
        "        }\n",
        "      }"
      ],
      "metadata": {
        "id": "8F86bl4cFiod"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!/usr/local/cuda/bin/nvcc --version\n",
        "!nvidia-smi"
      ],
      "metadata": {
        "id": "6fo-o6LRCDIH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "54f8946f-b923-4d2f-8920-cf0317c49a15"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "nvcc: NVIDIA (R) Cuda compiler driver\n",
            "Copyright (c) 2005-2020 NVIDIA Corporation\n",
            "Built on Mon_Oct_12_20:09:46_PDT_2020\n",
            "Cuda compilation tools, release 11.1, V11.1.105\n",
            "Build cuda_11.1.TC455_06.29190527_0\n",
            "Wed May 11 14:40:58 2022       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 460.32.03    Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   73C    P0    72W / 149W |   6238MiB / 11441MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Wrap into a Class"
      ],
      "metadata": {
        "id": "RZ9SkRCv1ucC"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7gZPc_0W0107"
      },
      "outputs": [],
      "source": [
        "class BertEmbeddings:\n",
        "\n",
        "  def __init__(self, model, tokenizer, batch_size, max_len):\n",
        "      self.model = model\n",
        "      self.tokenizer = tokenizer\n",
        "      self.batch_size = batch_size\n",
        "      self.max_len = max_len\n",
        "      self.embedding_size = 768\n",
        "      self.device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "      self.model.eval()\n",
        "      self.model = self.model.to(self.device)\n",
        "\n",
        "\n",
        "  def open_h5py_file(self, output_file, dataset, n_lyrs):\n",
        "     with h5py.File(output_file+'.h5', 'w') as hf_berts:\n",
        "       hf_berts.create_dataset(output_file,\n",
        "                               shape=(len(dataset), self.embedding_size*n_lyrs),\n",
        "                               chunks=True,\n",
        "                               dtype='float32'\n",
        "                               )\n",
        "  def derive_outputs(self, dataset, output_type, output_file, n_lyrs=1):\n",
        "    self.open_h5py_file(output_file, dataset, n_lyrs)\n",
        "    for offset in tqdm(range(0, len(dataset), self.batch_size)):\n",
        "        batch_text = dataset.iloc[:, 0][offset: offset + self.batch_size]\n",
        "        self.batched_encoding = self.tokenizer.batch_encode_plus(\n",
        "                batch_text.tolist(), \n",
        "                max_length=self.max_len,\n",
        "                padding='max_length',\n",
        "                return_tensors='pt',\n",
        "                truncation=True        \n",
        "                ).to(self.device)\n",
        "        with h5py.File(output_file+'.h5', 'a') as hf_berts:\n",
        "          with torch.no_grad():\n",
        "            input_ids_gpu = self.model(self.batched_encoding['input_ids'])\n",
        "            # Call the necessary method here\n",
        "            if output_type == 'token_avg':\n",
        "              avg_vects = self.last_hidden_state_token_avg(input_ids_gpu)\n",
        "              hf_berts[output_file][offset : offset + self.batch_size, :] = avg_vects.cpu()\n",
        "            elif output_type == 'n_lyrs_cat':\n",
        "              concat_CLS = self.n_lyrs_concat(n_lyrs, input_ids_gpu)\n",
        "              hf_berts[output_file][offset : offset + self.batch_size, :] = concat_CLS.cpu()\n",
        "            elif output_type == 'n_lyrs_cat_tkn_avg':\n",
        "              avg_cat_vects = self.n_lyrs_concat_tkn_avg(n_lyrs, input_ids_gpu)\n",
        "              hf_berts[output_file][offset : offset + self.batch_size, :] = avg_cat_vects.cpu()\n",
        "            else: \n",
        "              raise Exception('Not a valid Output type, check models_config')\n",
        "\n",
        "  def last_hidden_state_token_avg(self, input_ids):\n",
        "      attention_msk = self.batched_encoding['attention_mask'].unsqueeze(dim=-1)\n",
        "      # fill embeddings for CLS tokens with zeros\n",
        "      attention_msk[:, 0] = 0\n",
        "      valid_vects = (input_ids.last_hidden_state * attention_msk).sum(dim=1)\n",
        "      attention_sum = attention_msk.sum(dim=1)\n",
        "      avg_vects = valid_vects / attention_sum\n",
        "      return avg_vects\n",
        "  \n",
        "  def n_lyrs_concat(self, n_lyrs, input_ids):\n",
        "          lst_n_lyrs = tuple(torch.stack(input_ids.hidden_states, dim=0))[-n_lyrs:]\n",
        "          list_n_lyrs = [tensor[:,0,:] for tensor in lst_n_lyrs]\n",
        "          concat_CLS = torch.cat(list_n_lyrs, dim=1)\n",
        "          return concat_CLS\n",
        "\n",
        "  def n_lyrs_concat_tkn_avg(self, n_lyrs, input_ids):\n",
        "    valid_vects = []\n",
        "    avg_vects = []\n",
        "    attention_msk = self.batched_encoding['attention_mask'].unsqueeze(dim=-1)\n",
        "    # fill embeddings for CLS tokens with zeros\n",
        "    attention_msk[:, 0] = 0\n",
        "    attention_sum = attention_msk.sum(dim=1)\n",
        "    # get the last n layers from model\n",
        "    lst_n_lyrs = tuple(torch.stack(input_ids.hidden_states, dim=0))[-n_lyrs:]\n",
        "\n",
        "    for state in lst_n_lyrs:\n",
        "      valid_vect = (state * attention_msk).sum(dim=1)\n",
        "      token_avgs = valid_vect / attention_sum\n",
        "      avg_vects.append(token_avgs)\n",
        "      valid_vects.append(valid_vect)\n",
        "    \n",
        "   \n",
        "    concat_token_avg = torch.cat(avg_vects, dim=1)\n",
        "\n",
        "    return concat_token_avg\n",
        "    "
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### BERT Embedding objects -- feature extraction"
      ],
      "metadata": {
        "id": "NbxF_jkFZ7H0"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "change model_config to model_config_intent to get outputs on the Intent classification dataset"
      ],
      "metadata": {
        "id": "KmCMZzITH0nC"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dY-IJZAIFB7D"
      },
      "outputs": [],
      "source": [
        "# BERT\n",
        "\n",
        "## creating object of BertEmbeddings class\n",
        "bertembedding_object = BertEmbeddings(\n",
        "    model=models_config['BERT']['model'],\n",
        "    tokenizer=models_config['BERT']['tokenizer'],\n",
        "    batch_size=32,\n",
        "    max_len=256,\n",
        "    )\n",
        "\n",
        "## Token Average outputs from the last hidden layer\n",
        "bert_token_avg = bertembedding_object.derive_outputs(\n",
        "    dataset = data,\n",
        "    output_type=models_config['BERT']['output_type'][1],\n",
        "    output_file=models_config['BERT']['output_file'][1],\n",
        "    )\n",
        "\n",
        "## CLS concatenated outputs from last 4 layers\n",
        "bert_n_lyrs_cat = bertembedding_object.derive_outputs(\n",
        "    dataset = data,\n",
        "    output_type=models_config['BERT']['output_type'][3],\n",
        "    output_file=models_config['BERT']['output_file'][3],\n",
        "    n_lyrs = 4,\n",
        "    )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Tb4Zh8-nxN5y"
      },
      "outputs": [],
      "source": [
        "# RoBERT\n",
        "\n",
        "## Creating object of class BertEmbeddings\n",
        "robertembedding_object = BertEmbeddings(\n",
        "    model=models_config['RoBERT']['model'],\n",
        "    tokenizer=models_config['RoBERT']['tokenizer'],\n",
        "    batch_size=32,\n",
        "    max_len=256,\n",
        "    )\n",
        "\n",
        "## Token Average outputs from the last hidden layer\n",
        "robert_token_avg = robertembedding_object.derive_outputs(\n",
        "    dataset = data,\n",
        "    output_type=models_config['RoBERT']['output_type'][1],\n",
        "    output_file=models_config['RoBERT']['output_file'][1],\n",
        "    )\n",
        "\n",
        "\n",
        "## CLS concatenated outputs from last 4 layers\n",
        "robert_n_lyrs_cat = robertembedding_object.derive_outputs(\n",
        "    dataset = data,\n",
        "    output_type=models_config['RoBERT']['output_type'][3],\n",
        "    output_file=models_config['RoBERT']['output_file'][3],\n",
        "    n_lyrs = 4,\n",
        "    )"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Albert\n",
        "\n",
        "## Creating object of class BertEmbeddings\n",
        "albertembedding_object = BertEmbeddings(\n",
        "    model=models_config['AlBERT']['model'],\n",
        "    tokenizer=models_config['AlBERT']['tokenizer'],\n",
        "    batch_size=32,\n",
        "    max_len=256,\n",
        "    )\n",
        "\n",
        "## Token Average outputs from the last hidden layer\n",
        "robert_token_avg = albertembedding_object.derive_outputs(\n",
        "    dataset = data,\n",
        "    output_type=models_config['AlBERT']['output_type'][1],\n",
        "    output_file=models_config['AlBERT']['output_file'][1],\n",
        "    )\n",
        "\n",
        "## CLS concatenated outputs from last 4 layers\n",
        "robert_n_lyrs_cat = albertembedding_object.derive_outputs(\n",
        "    dataset = data,\n",
        "    output_type=models_config['AlBERT']['output_type'][3],\n",
        "    output_file=models_config['AlBERT']['output_file'][3],\n",
        "     n_lyrs = 4,\n",
        "     )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cmy5sa3BE-CG",
        "outputId": "6d4bb1e7-6205-4aaa-a454-20a2c3568cf7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1161/1161 [24:45<00:00,  1.28s/it]\n",
            "100%|██████████| 1161/1161 [24:57<00:00,  1.29s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# DistilBERT\n",
        "\n",
        "## Creating object of class BertEmbeddings\n",
        "distilbertembedding_object = BertEmbeddings(\n",
        "    model=models_config['DistilBERT']['model'],\n",
        "    tokenizer=models_config['DistilBERT']['tokenizer'],\n",
        "    batch_size=32,\n",
        "    max_len=256,\n",
        "    )\n",
        "\n",
        "## Token Average outputs from the last hidden layer\n",
        "distilbert_token_avg = distilbertembedding_object.derive_outputs(\n",
        "    dataset = data,\n",
        "    output_type=models_config['DistilBERT']['output_type'][1],\n",
        "    output_file=models_config['DistilBERT']['output_file'][1],\n",
        "    )\n",
        "\n",
        "## CLS concatenated outputs from last 4 layers\n",
        "distilbert_n_lyrs_cat = distilbertembedding_object.derive_outputs(\n",
        "    dataset = data,\n",
        "    output_type=models_config['DistilBERT']['output_type'][3],\n",
        "    output_file=models_config['DistilBERT']['output_file'][3],\n",
        "    n_lyrs = 4,\n",
        "    )\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Yv86oN0Zckmx",
        "outputId": "7da6f332-7178-40d7-8310-36855c6e73e9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1161/1161 [11:21<00:00,  1.70it/s]\n",
            "100%|██████████| 1161/1161 [11:23<00:00,  1.70it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# TinyBERT\n",
        "\n",
        "## Creating object of class BertEmbeddings\n",
        "tinybertembedding_object = BertEmbeddings(\n",
        "    model=models_config['TinyBERT']['model'],\n",
        "    tokenizer=models_config['TinyBERT']['tokenizer'],\n",
        "    batch_size=32,\n",
        "    max_len=256,\n",
        "    )\n",
        "\n",
        "## Token Average outputs from the last hidden layer\n",
        "tinybert_token_avg = tinybertembedding_object.derive_outputs(\n",
        "    dataset = data,\n",
        "    output_type=models_config['TinyBERT']['output_type'][1],\n",
        "    output_file=models_config['TinyBERT']['output_file'][1],\n",
        "    )\n",
        "\n",
        "## CLS concatenated outputs from last 4 layers\n",
        "tinybert_n_lyrs_cat = tinybertembedding_object.derive_outputs(\n",
        "    dataset = data,\n",
        "    output_type=models_config['TinyBERT']['output_type'][3],\n",
        "    output_file=models_config['TinyBERT']['output_file'][3],\n",
        "    n_lyrs = 4,\n",
        "    )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "77ppV0dq4IvH",
        "outputId": "99f698fb-29eb-47c1-fb62-699e5d5840c5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1563/1563 [06:50<00:00,  3.80it/s]\n",
            "100%|██████████| 1563/1563 [06:53<00:00,  3.78it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# SentenceBERT\n",
        "\n",
        "\n",
        "## Creating object of class BertEmbeddings\n",
        "sentencebertembedding_object = BertEmbeddings(\n",
        "    model=models_config['Sentence-BERT']['model'],\n",
        "    tokenizer=models_config['Sentence-BERT']['tokenizer'],\n",
        "    batch_size=32,\n",
        "    max_len=256,\n",
        "    )\n",
        "\n",
        "## Token Average outputs from the last hidden layer\n",
        "sentencebert_token_avg = sentencebertembedding_object.derive_outputs(\n",
        "    dataset = data,\n",
        "    output_type=models_config['Sentence-BERT']['output_type'][1],\n",
        "    output_file=models_config['Sentence-BERT']['output_file'][1],\n",
        "    )\n",
        "\n",
        "## CLS concatenated outputs from last 4 layers\n",
        "sentencebert_n_lyrs_cat = sentencebertembedding_object.derive_outputs(\n",
        "    dataset = data,\n",
        "    output_type=models_config['Sentence-BERT']['output_type'][3],\n",
        "    output_file=models_config['Sentence-BERT']['output_file'][3],\n",
        "    n_lyrs = 4,\n",
        "    )\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZkGgtlVYDDhS",
        "outputId": "0cb68ff9-04d0-4c07-8455-649f5bdf7d91"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1563/1563 [13:14<00:00,  1.97it/s]\n",
            "100%|██████████| 1563/1563 [13:21<00:00,  1.95it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Checking inference time of Robert (The Best Performing Model according to the final results)\n"
      ],
      "metadata": {
        "id": "zt5JtHCF4F4G"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# We sample 1000 data points randomly and then run the algorithm on those points\n",
        "data_1000_sample = data.sample(1000)\n",
        "data_1000_sample.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BzF1xDYu4NvA",
        "outputId": "90d03deb-f8ae-4cb8-ed05-3febecde75b3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1000, 2)"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "start_time = time.time()\n",
        "\n",
        "## Creating object of class BertEmbeddings\n",
        "robertembedding_object = BertEmbeddings(\n",
        "    model=models_config['RoBERT']['model'],\n",
        "    tokenizer=models_config['RoBERT']['tokenizer'],\n",
        "    batch_size=1,\n",
        "    max_len=256,\n",
        "    )\n",
        "\n",
        "## Token Average outputs from the last hidden layer\n",
        "robert_token_avg = robertembedding_object.derive_outputs(\n",
        "    dataset = data_1000_sample,\n",
        "    output_type=models_config['RoBERT']['output_type'][1],\n",
        "    output_file=models_config['RoBERT']['output_file'][1],\n",
        "    )\n",
        "\n",
        "\n",
        "end_time = time.time()\n",
        "print(f'\\n Inference ran for {round((end_time -  start_time))} seconds for 1000 datapoints')\n",
        "print(f' \\n For 1 datapoint inference ran for {round((end_time -  start_time)/1000, 2)} seconds')"
      ],
      "metadata": {
        "id": "3Pg9KrPp4xdN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "deee3a12-bee5-4a92-cc70-a71f832079a4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1000/1000 [00:46<00:00, 21.32it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            " Inference ran for 47 seconds for 1000 datapoints\n",
            " \n",
            " For 1 datapoint inference ran for 0.05 seconds\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "le = LabelEncoder()\n",
        "le_fitted = le.fit_transform(data_1000_sample.iloc[:,-1])\n",
        "data_1000_sample.iloc[:,-1] = le_fitted.astype('int')\n",
        "label_col = np.array(data_1000_sample['sentiment']).reshape(len(data_1000_sample),1)\n",
        "label_col.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oZtrTogLKWNW",
        "outputId": "c67f700b-2a04-4eb0-9d4b-5c78de73a723"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1000, 1)"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "hf_hidden = h5py.File('/content/robert_imdb_token_avg.h5', 'r')\n",
        "rb_lst_hidden = np.array(hf_hidden.get('robert_imdb_token_avg'))\n",
        "hf_hidden.close()\n",
        "print(rb_lst_hidden.shape)\n",
        "rb_lst_hidden = np.append(rb_lst_hidden, label_col, axis=1)\n",
        "rb_lst_hidden.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Je-E5GuuK2qn",
        "outputId": "4e104476-ba92-43aa-fb38-569b407fc1e8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(1000, 768)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1000, 769)"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# RoBERTa\n",
        "start_time = time.time()\n",
        "\n",
        "with warnings.catch_warnings():\n",
        "    warnings.simplefilter(\"ignore\")\n",
        "    clf = LogisticRegression(solver = \"lbfgs\", random_state = 0)\n",
        "    clf.fit(rb_lst_hidden, label_col)\n",
        "\n",
        "end_time = time.time()\n",
        "print(f'\\n Inference ran for {round((end_time -  start_time))} seconds for 1000 datapoints')\n",
        "print(f' \\n For 1 datapoint inference ran for {round((end_time -  start_time)/1000, 2)} seconds')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mkzDVo0-Bgzl",
        "outputId": "d8bf992c-a015-4050-c451-9e922992fa5c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            " Inference ran for 0 seconds for 1000 datapoints\n",
            " \n",
            " For 1 datapoint inference ran for 0.0 seconds\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Adding the Embedding extraction inference time with logistic regression inference time we get 0.05 seconds for 1 data point"
      ],
      "metadata": {
        "id": "nfv7TtOIMAnu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "NXN5TEPXV4Vz"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "DkeLdcLh1pwT",
        "RZ9SkRCv1ucC",
        "APQIQD9843oi"
      ],
      "name": "Transformers_BERT_embeddings_both_datasets.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}