baseline_classifiers_IMDb_Dataset.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "baseline_classifiers_IMDb_Dataset.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "7l_tLpkaEo6y",
        "UVFzDXwXb997",
        "aDejd4b_6ue0",
        "1WI0pIPqzGZD",
        "Z4qIFNNiFC9F",
        "JA_E9K8Lnfhk",
        "HUMrZQvPFJkn",
        "lYGxSUjMRfDS",
        "EtErM8Rz1JJc",
        "FzfO44bCpeFX",
        "jxpL7O9Mpslf",
        "D3Io7mXdp24T",
        "dUYomcKkqFtM",
        "KLlWzSVEqRZu",
        "CG9E00wsWxVt"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/larissapoghosyan/Capstone_Project/blob/main/baseline_classifiers_IMDb_Dataset.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Import libraries"
      ],
      "metadata": {
        "id": "7l_tLpkaEo6y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm\n",
        "import warnings\n",
        "import h5py\n",
        "import numpy\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import nltk\n",
        "from nltk import ngrams, sent_tokenize, word_tokenize\n",
        "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
        "from sklearn.model_selection import train_test_split, KFold, cross_val_score\n",
        "from sklearn.metrics import accuracy_score, f1_score, recall_score, classification_report\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "import matplotlib.pyplot as plt\n",
        "from gensim.models import Doc2Vec\n",
        "from gensim.models.doc2vec import TaggedDocument\n",
        "from sklearn import utils\n",
        "import multiprocessing\n",
        "import time"
      ],
      "metadata": {
        "id": "4VyR3nKj4Pph"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "np.random.seed(42)"
      ],
      "metadata": {
        "id": "yoxuF8B9oNua"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Label Encoding"
      ],
      "metadata": {
        "id": "UVFzDXwXb997"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = pd.read_csv('/content/IMDb_Reviews.csv',engine='python', error_bad_lines=False)"
      ],
      "metadata": {
        "id": "zsf2fF9YpRIH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c6e6952e-7eab-48e0-cc66-622dd2a41b9c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py:2882: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version.\n",
            "\n",
            "\n",
            "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "le = LabelEncoder()\n",
        "le_fitted = le.fit_transform(dataset.iloc[:,-1])\n",
        "dataset.iloc[:,-1] = le_fitted.astype('int')\n",
        "label_col = np.array(dataset['sentiment']).reshape(len(dataset),1)\n",
        "label_col.shape"
      ],
      "metadata": {
        "id": "h9MGb1qmvalQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b945ee15-4857-4875-bd88-158e4e9085de"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(50000, 1)"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset.shape"
      ],
      "metadata": {
        "id": "Mt6FOEItvjIm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7fdaf04f-59c5-4eeb-eaa9-41726ceb5015"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(50000, 2)"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(np.unique(le_fitted)), '\\n', np.unique(le_fitted))"
      ],
      "metadata": {
        "id": "Xp6XIQuUAAzh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3e18a517-7d90-4314-fe49-2504f4dde7e6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2 \n",
            " [0 1]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Splitting the Dataset"
      ],
      "metadata": {
        "id": "aDejd4b_6ue0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def Splitter(df):\n",
        "  np.random.seed(0)\n",
        "  msk = np.random.rand(len(df)) < 0.8\n",
        "  train = df[msk]\n",
        "  test = df[~msk]\n",
        "  return(train, test)"
      ],
      "metadata": {
        "id": "dl89705pdvoF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## BERT Embeddings"
      ],
      "metadata": {
        "id": "-fagLvYQ55a_"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### BERT"
      ],
      "metadata": {
        "id": "JqL_8sA3bdYN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# no CLS token avg\n",
        "hf_hidden = h5py.File('/content/bert_imdb_token_avg.h5', 'r')\n",
        "br_lst_hidden = np.array(hf_hidden.get('bert_imdb_token_avg'))\n",
        "hf_hidden.close()\n",
        "print(br_lst_hidden.shape)\n",
        "br_lst_hidden = np.append(br_lst_hidden, label_col, axis=1)\n",
        "print(br_lst_hidden.shape)"
      ],
      "metadata": {
        "id": "AgVJ8reMxRmB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "br_lst_hidden_df = pd.DataFrame(br_lst_hidden)\n",
        "br_lst_hidden_prc20 = np.array(br_lst_hidden_df.sample(int(br_lst_hidden_df.shape[0]*0.2),\n",
        "                                                       random_state=22))\n",
        "print(br_lst_hidden_prc20.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eI0gwsrmvrR4",
        "outputId": "91888713-3056-4c5c-f722-30551e8f5056"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 769)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 4 layers\n",
        "hf_hidden = h5py.File('/content/bert_imdb_CLS_cat.h5', 'r')\n",
        "bert_cat_hidden_4lyr_data = np.array(hf_hidden.get('bert_imdb_CLS_cat'))\n",
        "hf_hidden.close()\n",
        "print(bert_cat_hidden_4lyr_data.shape)\n",
        "bert_cat_hidden_4lyr = np.append(bert_cat_hidden_4lyr_data, label_col, axis=1)\n",
        "print(bert_cat_hidden_4lyr.shape)"
      ],
      "metadata": {
        "id": "I5xEYTm5RZCY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ef6f0f0b-804e-4af4-d687-61cefe26a0ee"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 3072)\n",
            "(50000, 3073)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "bert_cat_hidden_4lyr_df = pd.DataFrame(bert_cat_hidden_4lyr)\n",
        "bert_cat_hidden_4lyr_prc20 = np.array(bert_cat_hidden_4lyr_df.sample(int(bert_cat_hidden_4lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(bert_cat_hidden_4lyr_prc20.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lwby5nwoyJBz",
        "outputId": "f384d30a-0c21-4450-e74f-35340763360d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 3073)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 3 layers\n",
        "n = 768 * 3\n",
        "bert_cat_hidden_3lyr = np.append(bert_cat_hidden_4lyr_data[:,:n], label_col, axis=1)\n",
        "print(bert_cat_hidden_3lyr.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xPZyllispZDQ",
        "outputId": "ecabf967-87ee-43b0-fc2f-794129a141be"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 2305)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "bert_cat_hidden_3lyr_df = pd.DataFrame(bert_cat_hidden_3lyr)\n",
        "bert_cat_hidden_3lyr_prc20 = np.array(bert_cat_hidden_3lyr_df.sample(int(bert_cat_hidden_3lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(bert_cat_hidden_3lyr_prc20.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WOJkr4WYz8Fn",
        "outputId": "b511f168-a2b1-4afe-a929-7cf7f94414f9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 2305)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 2 layers\n",
        "n = 768 * 2\n",
        "bert_cat_hidden_2lyr = np.append(bert_cat_hidden_4lyr_data[:,:n], label_col, axis=1)\n",
        "print(bert_cat_hidden_2lyr.shape)"
      ],
      "metadata": {
        "id": "BSqWtLnhpY-V",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6442f541-ea30-487e-b200-ad1a54f302a8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 1537)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "bert_cat_hidden_2lyr_df = pd.DataFrame(bert_cat_hidden_2lyr)\n",
        "bert_cat_hidden_2lyr_prc20 = np.array(bert_cat_hidden_2lyr_df.sample(int(bert_cat_hidden_2lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(bert_cat_hidden_2lyr_prc20.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EkumZ3Ww0pQI",
        "outputId": "24bb484d-41db-493a-a04f-42e17a10fbf3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 1537)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 1 layer\n",
        "n = 768 * 1\n",
        "bert_cat_hidden_1lyr = np.append(bert_cat_hidden_4lyr_data[:,:n], label_col, axis=1)\n",
        "print(bert_cat_hidden_1lyr.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8QG8E4EMzwII",
        "outputId": "2950524b-afed-445d-8e09-861d3897186a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 769)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "bert_cat_hidden_1lyr_df = pd.DataFrame(bert_cat_hidden_1lyr)\n",
        "bert_cat_hidden_1lyr_prc20 = np.array(bert_cat_hidden_1lyr_df.sample(int(bert_cat_hidden_1lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(bert_cat_hidden_1lyr_prc20.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZUU-dxVV0taa",
        "outputId": "291b7f01-f166-477c-8d8f-9adef09638b5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 769)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Robert"
      ],
      "metadata": {
        "id": "5PszOTds00oe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# no CLS token avg\n",
        "hf_hidden = h5py.File('/content/robert_imdb_token_avg.h5', 'r')\n",
        "rb_lst_hidden = np.array(hf_hidden.get('robert_imdb_token_avg'))\n",
        "hf_hidden.close()\n",
        "print(rb_lst_hidden.shape)\n",
        "rb_lst_hidden = np.append(rb_lst_hidden, label_col, axis=1)\n",
        "rb_lst_hidden.shape"
      ],
      "metadata": {
        "id": "PLn-Yo4oxlzS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fd800d1e-f309-4966-d58a-8061fb464fc9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 768)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(50000, 769)"
            ]
          },
          "metadata": {},
          "execution_count": 101
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "rb_lst_hidden_df = pd.DataFrame(rb_lst_hidden)\n",
        "rb_lst_hidden_prc20 = np.array(rb_lst_hidden_df.sample(int(rb_lst_hidden_df.shape[0]*0.2),\n",
        "                                                       random_state=22))\n",
        "print(rb_lst_hidden_prc20.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V6nW3i4e7xIm",
        "outputId": "9ebc45e9-885b-4cf0-cc54-39feccf1713f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 769)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 4 layers\n",
        "hf_hidden = h5py.File('/content/robert_imdb_CLS_cat.h5', 'r')\n",
        "rb_cat_hidden_4lyr_data = np.array(hf_hidden.get('robert_imdb_CLS_cat'))\n",
        "hf_hidden.close()\n",
        "print(rb_cat_hidden_4lyr_data.shape)\n",
        "rb_cat_hidden_4lyr = np.append(rb_cat_hidden_4lyr_data, label_col, axis=1)\n",
        "print(rb_cat_hidden_4lyr.shape)"
      ],
      "metadata": {
        "id": "8zVFgMMnRebT",
        "outputId": "74109f76-858c-4f93-9cba-b241cb329ad4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 3072)\n",
            "(50000, 3073)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "rb_cat_hidden_4lyr_df = pd.DataFrame(rb_cat_hidden_4lyr)\n",
        "rb_cat_hidden_4lyr_prc20 = np.array(rb_cat_hidden_4lyr_df.sample(int(rb_cat_hidden_4lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(rb_cat_hidden_4lyr_prc20.shape)"
      ],
      "metadata": {
        "id": "ObbyjSBWHPz0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b5a54824-9078-403c-d244-6ff20670e0a0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 3073)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 3 layers\n",
        "n = 768 * 3\n",
        "rb_cat_hidden_3lyr = np.append(rb_cat_hidden_4lyr_data[:,:n], label_col, axis=1)\n",
        "print(rb_cat_hidden_3lyr.shape)"
      ],
      "metadata": {
        "id": "MrFpq-eMpdmP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4fec799d-09a4-4082-a27f-7b98a88ad96c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 2304)\n",
            "(50000, 2305)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "rb_cat_hidden_3lyr_df = pd.DataFrame(rb_cat_hidden_3lyr)\n",
        "rb_cat_hidden_3lyr_prc20 = np.array(rb_cat_hidden_3lyr_df.sample(int(rb_cat_hidden_3lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(rb_cat_hidden_3lyr_prc20.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z6dKz59Sqw7h",
        "outputId": "53f47826-b15b-470c-ef22-2d794a310bcb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 2305)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 2 layers\n",
        "n = 768 * 2\n",
        "rb_cat_hidden_2lyr = np.append(rb_cat_hidden_4lyr_data[:,:n], label_col, axis=1)\n",
        "print(rb_cat_hidden_2lyr.shape)"
      ],
      "metadata": {
        "id": "zDOKxL24pdiU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "50b57a57-fd8f-40d5-994d-002a958fe3d5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 1537)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "rb_cat_hidden_2lyr_df = pd.DataFrame(rb_cat_hidden_2lyr)\n",
        "rb_cat_hidden_2lyr_prc20 = np.array(rb_cat_hidden_2lyr_df.sample(int(rb_cat_hidden_2lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(rb_cat_hidden_2lyr_prc20.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xjZnwNP_q_87",
        "outputId": "280820d8-3e7e-4a3f-892c-e95ddcec241e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 1537)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 1 layer\n",
        "n = 768 * 1\n",
        "rb_cat_hidden_1lyr = np.append(rb_cat_hidden_4lyr_data[:,:n], label_col, axis=1)\n",
        "print(rb_cat_hidden_1lyr.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PyZpgOnIrGb-",
        "outputId": "a9c3f18c-5e99-4cae-8531-6660f5e230b5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 769)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "rb_cat_hidden_1lyr_df = pd.DataFrame(rb_cat_hidden_1lyr)\n",
        "rb_cat_hidden_1lyr_prc20 = np.array(rb_cat_hidden_1lyr_df.sample(int(rb_cat_hidden_1lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(rb_cat_hidden_1lyr_prc20.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qPQ_xBrWrVtG",
        "outputId": "91f8fbe8-4862-4193-c4b6-753f90f5ac23"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 769)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### AlBERT"
      ],
      "metadata": {
        "id": "Km7OM_71wv0i"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# no CLS token avg\n",
        "hf_hidden = h5py.File('/content/albert_imdb_token_avg.h5', 'r')\n",
        "ab_lst_hidden = np.array(hf_hidden.get('albert_imdb_token_avg'))\n",
        "hf_hidden.close()\n",
        "print(ab_lst_hidden.shape)\n",
        "ab_lst_hidden = np.append(ab_lst_hidden, label_col, axis=1)\n",
        "ab_lst_hidden.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0esP46ydwzC_",
        "outputId": "d9a8e02c-a44e-4d9b-e572-8a4394882dfc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 768)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(50000, 769)"
            ]
          },
          "metadata": {},
          "execution_count": 103
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "ab_lst_hidden_df = pd.DataFrame(ab_lst_hidden)\n",
        "ab_lst_hidden_prc20 = np.array(ab_lst_hidden_df.sample(int(ab_lst_hidden_df.shape[0]*0.2),\n",
        "                                                       random_state=22))\n",
        "print(ab_lst_hidden_prc20.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tD9RQyph8DzO",
        "outputId": "30e16769-fac4-4bb3-c566-4f1f54c3c113"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 769)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 4 layers\n",
        "hf_hidden = h5py.File('/content/albert_imdb_CLS_cat.h5', 'r')\n",
        "ab_cat_hidden_4lyr_data = np.array(hf_hidden.get('albert_imdb_CLS_cat'))\n",
        "hf_hidden.close()\n",
        "print(ab_cat_hidden_4lyr_data.shape)\n",
        "ab_cat_hidden_4lyr = np.append(ab_cat_hidden_4lyr_data, label_col, axis=1)\n",
        "print(ab_cat_hidden_4lyr.shape)"
      ],
      "metadata": {
        "id": "7Re18zIgpq8Y",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d6bb0fbe-1bf8-4295-d451-71f72041dfcd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 3072)\n",
            "(50000, 3073)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "ab_cat_hidden_4lyr_df = pd.DataFrame(ab_cat_hidden_4lyr)\n",
        "ab_cat_hidden_4lyr_prc20 = np.array(ab_cat_hidden_4lyr_df.sample(int(ab_cat_hidden_4lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(ab_cat_hidden_4lyr_prc20.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iBW-gqUbwqpd",
        "outputId": "59ef3e95-5b37-4e6b-b963-c366cae5e5fe"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 3073)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 3 layers\n",
        "n = 768 * 3\n",
        "ab_cat_hidden_3lyr = np.append(ab_cat_hidden_4lyr_data[:,:n], label_col, axis=1)\n",
        "print(ab_cat_hidden_3lyr.shape)"
      ],
      "metadata": {
        "id": "XbW33cHARk0x",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3e7b44d4-ac68-444b-cc0c-575b4ef8d2ec"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 2305)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "ab_cat_hidden_3lyr_df = pd.DataFrame(ab_cat_hidden_3lyr)\n",
        "ab_cat_hidden_3lyr_prc20 = np.array(ab_cat_hidden_3lyr_df.sample(int(ab_cat_hidden_3lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(ab_cat_hidden_3lyr_prc20.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F13eQwbjwoiF",
        "outputId": "517efd46-04c2-41b2-9d04-f69b95f8ee07"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 2305)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 2 layers\n",
        "n = 768 * 2\n",
        "ab_cat_hidden_2lyr = np.append(ab_cat_hidden_4lyr_data[:,:n], label_col, axis=1)\n",
        "print(ab_cat_hidden_2lyr.shape)"
      ],
      "metadata": {
        "id": "qVe_21L5ptwT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f544adb8-5df1-400c-f497-6d9c3e022269"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 1537)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "ab_cat_hidden_2lyr_df = pd.DataFrame(ab_cat_hidden_2lyr)\n",
        "ab_cat_hidden_2lyr_prc20 = np.array(ab_cat_hidden_2lyr_df.sample(int(ab_cat_hidden_2lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(ab_cat_hidden_2lyr_prc20.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jNkrm13swnRV",
        "outputId": "ed27e118-9344-4afc-c6de-0eb5e8b30d23"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 1537)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "n = 768 * 1\n",
        "ab_cat_hidden_1lyr = np.append(ab_cat_hidden_4lyr_data[:,:n], label_col, axis=1)\n",
        "print(ab_cat_hidden_1lyr.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WbSoV0jrwlPZ",
        "outputId": "034e1cdd-c1dc-4f99-9c82-9aa951eac142"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 769)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "ab_cat_hidden_1lyr_df = pd.DataFrame(ab_cat_hidden_1lyr)\n",
        "ab_cat_hidden_1lyr_prc20 = np.array(ab_cat_hidden_1lyr_df.sample(int(ab_cat_hidden_1lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(ab_cat_hidden_1lyr_prc20.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P4Bs7m3EwnuH",
        "outputId": "5732803e-b95b-4e91-f08c-2fffe08d0950"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 769)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### DistilBERT"
      ],
      "metadata": {
        "id": "6JvCUZGBbkxN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# no CLS token avg\n",
        "hf_hidden = h5py.File('/content/distilbert_imdb_token_avg.h5', 'r')\n",
        "db_lst_hidden = np.array(hf_hidden.get('distilbert_imdb_token_avg'))\n",
        "hf_hidden.close()\n",
        "print(db_lst_hidden.shape)\n",
        "db_lst_hidden = np.append(db_lst_hidden, label_col, axis=1)\n",
        "db_lst_hidden.shape"
      ],
      "metadata": {
        "id": "jkQ-3XydbnM1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5e131836-83e4-4371-ec10-e1e2afb32a0b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 768)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(50000, 769)"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "db_lst_hidden_df = pd.DataFrame(db_lst_hidden)\n",
        "db_lst_hidden_prc20 = np.array(db_lst_hidden_df.sample(int(db_lst_hidden_df.shape[0]*0.2),\n",
        "                                                       random_state=22))\n",
        "print(db_lst_hidden_prc20.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5QIL6uBc8Gmm",
        "outputId": "642aab7e-22b1-48a5-de67-320793362c94"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 769)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 4 layers\n",
        "hf_hidden = h5py.File('/content/distilbert_imdb_CLS_cat.h5', 'r')\n",
        "db_cat_hidden_4lyr_data = np.array(hf_hidden.get('distilbert_imdb_CLS_cat'))\n",
        "hf_hidden.close()\n",
        "print(db_cat_hidden_4lyr_data.shape)\n",
        "db_cat_hidden_4lyr = np.append(db_cat_hidden_4lyr_data, label_col, axis=1)\n",
        "print(db_cat_hidden_4lyr.shape)"
      ],
      "metadata": {
        "id": "Rm8OWzvypysS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "db_cat_hidden_4lyr_df = pd.DataFrame(db_cat_hidden_4lyr)\n",
        "db_cat_hidden_4lyr_prc20 = np.array(db_cat_hidden_4lyr_df.sample(int(db_cat_hidden_4lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(db_cat_hidden_4lyr_prc20.shape)"
      ],
      "metadata": {
        "id": "hnyPDntPHAqj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 3 layers\n",
        "n = 768 * 3\n",
        "db_cat_hidden_3lyr = np.append(db_cat_hidden_4lyr_data[:,:n], label_col, axis=1)\n",
        "print(db_cat_hidden_3lyr.shape)"
      ],
      "metadata": {
        "id": "Ppkjj4vwRpyj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "db_cat_hidden_3lyr_df = pd.DataFrame(db_cat_hidden_3lyr)\n",
        "db_cat_hidden_3lyr_prc20 = np.array(db_cat_hidden_3lyr_df.sample(int(db_cat_hidden_3lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(db_cat_hidden_3lyr_prc20.shape)"
      ],
      "metadata": {
        "id": "Kj_YTWNCHFvs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 2 layers\n",
        "n = 768 * 2\n",
        "db_cat_hidden_2lyr = np.append(db_cat_hidden_4lyr_data[:,:n], label_col, axis=1)\n",
        "print(db_cat_hidden_2lyr.shape)"
      ],
      "metadata": {
        "id": "KCGoeHmip0zU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "db_cat_hidden_2lyr_df = pd.DataFrame(db_cat_hidden_2lyr)\n",
        "db_cat_hidden_2lyr_prc20 = np.array(db_cat_hidden_2lyr_df.sample(int(db_cat_hidden_2lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(db_cat_hidden_2lyr_prc20.shape)"
      ],
      "metadata": {
        "id": "6gR2S2gLHKEy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 1 layers\n",
        "n = 768 * 1\n",
        "db_cat_hidden_1lyr = np.append(db_cat_hidden_4lyr_data[:,:n], label_col, axis=1)\n",
        "print(db_cat_hidden_1lyr.shape)"
      ],
      "metadata": {
        "id": "viRZQI9JG6FP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "db_cat_hidden_1lyr_df = pd.DataFrame(db_cat_hidden_1lyr)\n",
        "db_cat_hidden_1lyr_prc20 = np.array(db_cat_hidden_1lyr_df.sample(int(db_cat_hidden_1lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(db_cat_hidden_1lyr_prc20.shape)"
      ],
      "metadata": {
        "id": "AJbpvyjXHMOA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### TinyBERT"
      ],
      "metadata": {
        "id": "dEjk8NktBi9Y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# no CLS token avg\n",
        "hf_hidden = h5py.File('/content/tinybert_imdb_token_avg.h5', 'r')\n",
        "tb_lst_hidden = np.array(hf_hidden.get('tinybert_imdb_token_avg'))\n",
        "hf_hidden.close()\n",
        "print(tb_lst_hidden.shape)\n",
        "tb_lst_hidden = np.append(tb_lst_hidden, label_col, axis=1)\n",
        "tb_lst_hidden.shape"
      ],
      "metadata": {
        "id": "WfB4R_zPBinl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tb_lst_hidden_df = pd.DataFrame(tb_lst_hidden)\n",
        "tb_lst_hidden_prc20 = np.array(tb_lst_hidden_df.sample(int(tb_lst_hidden_df.shape[0]*0.2),\n",
        "                                                       random_state=22))\n",
        "print(tb_lst_hidden_prc20.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aRjOOTHs8Iyk",
        "outputId": "0a6b5801-abf6-4adb-9bef-b8b9411724ea"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 769)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 4 layers\n",
        "hf_hidden = h5py.File('/content/tinybert_imdb_CLS_cat.h5', 'r')\n",
        "tb_cat_hidden_4lyr_data = np.array(hf_hidden.get('tinybert_imdb_CLS_cat'))\n",
        "hf_hidden.close()\n",
        "print(tb_cat_hidden_4lyr_data.shape)\n",
        "tb_cat_hidden_4lyr = np.append(tb_cat_hidden_4lyr_data, label_col, axis=1)\n",
        "print(tb_cat_hidden_4lyr.shape)"
      ],
      "metadata": {
        "id": "eew35ai7p4Hl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "tb_cat_hidden_4lyr_df = pd.DataFrame(tb_cat_hidden_4lyr)\n",
        "tb_cat_hidden_4lyr_prc20 = np.array(tb_cat_hidden_4lyr_df.sample(int(tb_cat_hidden_4lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(tb_cat_hidden_4lyr_prc20.shape)"
      ],
      "metadata": {
        "id": "VdfjLvvdpk21"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 3 layers\n",
        "n = 768 * 3\n",
        "tb_cat_hidden_3lyr = np.append(tb_cat_hidden_4lyr_data[:,:n], label_col, axis=1)\n",
        "print(tb_cat_hidden_3lyr.shape)"
      ],
      "metadata": {
        "id": "DpDC_rAGRsKQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "tb_cat_hidden_3lyr_df = pd.DataFrame(tb_cat_hidden_3lyr)\n",
        "tb_cat_hidden_3lyr_prc20 = np.array(tb_cat_hidden_3lyr_df.sample(int(tb_cat_hidden_3lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(tb_cat_hidden_3lyr_prc20.shape)"
      ],
      "metadata": {
        "id": "2m7mUTzhqjku"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 2 layers\n",
        "n = 768 * 2\n",
        "tb_cat_hidden_2lyr = np.append(tb_cat_hidden_4lyr_data[:,:n], label_col, axis=1)\n",
        "print(tb_cat_hidden_2lyr.shape)"
      ],
      "metadata": {
        "id": "uK3m6LQzp6d-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "tb_cat_hidden_2lyr_df = pd.DataFrame(tb_cat_hidden_2lyr)\n",
        "tb_cat_hidden_2lyr_prc20 = np.array(tb_cat_hidden_2lyr_df.sample(int(tb_cat_hidden_2lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(tb_cat_hidden_2lyr_prc20.shape)"
      ],
      "metadata": {
        "id": "OjRAI1ycqsYB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 1 layers\n",
        "n = 768 * 1\n",
        "tb_cat_hidden_1lyr = np.append(tb_cat_hidden_4lyr_data[:,:n], label_col, axis=1)\n",
        "print(tb_cat_hidden_1lyr.shape)"
      ],
      "metadata": {
        "id": "zVw5OCYOqvFV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "tb_cat_hidden_1lyr_df = pd.DataFrame(tb_cat_hidden_1lyr)\n",
        "tb_cat_hidden_1lyr_prc20 = np.array(tb_cat_hidden_1lyr_df.sample(int(tb_cat_hidden_1lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(tb_cat_hidden_1lyr_prc20.shape)"
      ],
      "metadata": {
        "id": "fsFzKGGhqu75"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Sentence-BERT"
      ],
      "metadata": {
        "id": "gmdSdmiGN1GT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# no CLS token avg\n",
        "hf_hidden = h5py.File('/content/sentence_bert_imdb_token_avg.h5', 'r')\n",
        "sb_lst_hidden = np.array(hf_hidden.get('sentence_bert_imdb_token_avg'))\n",
        "hf_hidden.close()\n",
        "print(sb_lst_hidden.shape)\n",
        "sb_lst_hidden = np.append(sb_lst_hidden, label_col, axis=1)\n",
        "sb_lst_hidden.shape"
      ],
      "metadata": {
        "id": "0lXJ87m5N0vM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bd110a01-70a4-4569-b54b-4c56f51bd482"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 768)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(50000, 769)"
            ]
          },
          "metadata": {},
          "execution_count": 109
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "sb_lst_hidden_df = pd.DataFrame(sb_lst_hidden)\n",
        "sb_lst_hidden_prc20 = np.array(sb_lst_hidden_df.sample(int(sb_lst_hidden_df.shape[0]*0.2),\n",
        "                                                       random_state=22))\n",
        "print(sb_lst_hidden_prc20.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "prdPkzzv8N9U",
        "outputId": "859a711f-98ad-48c5-9859-0ee52472470a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 769)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 4 layers\n",
        "hf_hidden = h5py.File('/content/sentence_bert_imdb_CLS_cat.h5', 'r')\n",
        "sb_cat_hidden_4lyr_data = np.array(hf_hidden.get('sentence_bert_imdb_CLS_cat'))\n",
        "hf_hidden.close()\n",
        "print(sb_cat_hidden_4lyr_data.shape)\n",
        "sb_cat_hidden_4lyr = np.append(sb_cat_hidden_4lyr_data, label_col, axis=1)\n",
        "print(sb_cat_hidden_4lyr.shape)"
      ],
      "metadata": {
        "id": "xXjr9Ff2p-A0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "sb_cat_hidden_4lyr_df = pd.DataFrame(sb_cat_hidden_4lyr)\n",
        "sb_cat_hidden_4lyr_prc20 = np.array(sb_cat_hidden_4lyr_df.sample(int(sb_cat_hidden_4lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(sb_cat_hidden_4lyr_prc20.shape)"
      ],
      "metadata": {
        "id": "pvfLE0xEr7Rc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 3 layers\n",
        "n = 768 * 3\n",
        "sb_cat_hidden_3lyr = np.append(sb_cat_hidden_4lyr_data[:,:n], label_col, axis=1)\n",
        "print(sb_cat_hidden_3lyr.shape)"
      ],
      "metadata": {
        "id": "PbO0TlUkN3Vo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "sb_cat_hidden_3lyr_df = pd.DataFrame(sb_cat_hidden_3lyr)\n",
        "sb_cat_hidden_3lyr_prc20 = np.array(sb_cat_hidden_3lyr_df.sample(int(sb_cat_hidden_3lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(sb_cat_hidden_3lyr_prc20.shape)"
      ],
      "metadata": {
        "id": "mu6yJ7EDrauO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 2 layers\n",
        "n = 768 * 2\n",
        "sb_cat_hidden_2lyr = np.append(sb_cat_hidden_4lyr_data[:,:n], label_col, axis=1)\n",
        "print(sb_cat_hidden_2lyr.shape)"
      ],
      "metadata": {
        "id": "5ruRQ8Mnp-aw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "sb_cat_hidden_2lyr_df = pd.DataFrame(sb_cat_hidden_2lyr)\n",
        "sb_cat_hidden_2lyr_prc20 = np.array(sb_cat_hidden_2lyr_df.sample(int(sb_cat_hidden_2lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(sb_cat_hidden_2lyr_prc20.shape)"
      ],
      "metadata": {
        "id": "7xZThlq4rnS8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# CLS Concatenated from last 1 layers\n",
        "n = 768 * 1\n",
        "sb_cat_hidden_1lyr = np.append(sb_cat_hidden_4lyr_data[:,:n], label_col, axis=1)\n",
        "print(sb_cat_hidden_1lyr.shape)"
      ],
      "metadata": {
        "id": "Dfu_pKFSrnEI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 20% sampled\n",
        "sb_cat_hidden_1lyr_df = pd.DataFrame(sb_cat_hidden_1lyr)\n",
        "sb_cat_hidden_1lyr_prc20 = np.array(sb_cat_hidden_1lyr_df.sample(int(sb_cat_hidden_1lyr_df.shape[0]*0.2),\n",
        "                                                                     random_state=22))\n",
        "print(sb_cat_hidden_1lyr_prc20.shape)"
      ],
      "metadata": {
        "id": "m8glp2sMrmzF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Splitting"
      ],
      "metadata": {
        "id": "ngiIoPtkb2V-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "training, testing = (Splitter(db_lst_hidden_prc20)[0], Splitter(db_lst_hidden_prc20)[1])\n",
        "train_hidden, train_lab_hidden = (training[:,:training.shape[1]-1], training[:,-1])\n",
        "test_hidden, test_lab_hidden = (testing[:,:testing.shape[1]-1], testing[:,-1])\n",
        "train_hidden = train_hidden.astype('float64')\n",
        "test_hidden = test_hidden.astype('float64')\n",
        "train_lab_hidden = train_lab_hidden.astype('int')\n",
        "test_lab_hidden = test_lab_hidden.astype('int')"
      ],
      "metadata": {
        "id": "_5mJrVE87cb5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Shape of the Train data is %s, Shape of Train labels is %s ' % (train_hidden.shape, train_lab_hidden.shape))\n",
        "print('Shape of the Test data is  %s,  Shape of Test labels is %s' % (test_hidden.shape, test_lab_hidden.shape))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pynkngVv7cUt",
        "outputId": "656aa2fd-35b9-4a22-d9ee-2bb766bea4cc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape of the Train data is (8028, 768), Shape of Train labels is (8028,) \n",
            "Shape of the Test data is  (1972, 768),  Shape of Test labels is (1972,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Word2vec embeddings"
      ],
      "metadata": {
        "id": "1WI0pIPqzGZD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "hf_wv_ = h5py.File('/content/wv_vectors_imdb.h5', 'r')\n",
        "# hf_wv_.keys()\n",
        "embedding_wv = hf_wv_.get('vec_lis_hf')\n",
        "embedding_wv = np.array(embedding_wv)\n",
        "print(embedding_wv.shape)\n",
        "hf_wv_.close()"
      ],
      "metadata": {
        "id": "pnNlTMM2DBh6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3190cbef-d0ce-4cb9-d9b0-b174ff7986bc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 300)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embedding_wv_full = np.append(embedding_wv, label_col, axis=1)\n",
        "print(embedding_wv_full.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rDTFRaz-lc5z",
        "outputId": "bbee5335-8b25-4ae2-ee70-885609312cfd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 301)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embedding_wv_df = pd.DataFrame(embedding_wv_full)\n",
        "embedding_wv_sample = np.array(embedding_wv_df.sample(int(embedding_wv_df.shape[0]*0.2),\n",
        "                                                       random_state=22))\n",
        "print(embedding_wv_sample.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dPErnr0S85nA",
        "outputId": "b87ff5e3-89d0-4090-b1fa-61af751b6df7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 301)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Fast TExt Embeddings"
      ],
      "metadata": {
        "id": "Z4qIFNNiFC9F"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "hf_ft_ = h5py.File('/content/ft_vectors_imdb.h5', 'r')\n",
        "embedding_ft = hf_ft_.get('vec_lis_hf')\n",
        "embedding_ft = np.array(embedding_ft)\n",
        "print(embedding_ft.shape)\n",
        "hf_ft_.close()"
      ],
      "metadata": {
        "id": "R3QIQCSMFYXt",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f9bd1bc8-8436-46e3-d5c9-9436075d11c8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 300)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embedding_ft_full = np.append(embedding_ft, label_col, axis=1)\n",
        "print(embedding_ft_full.shape)"
      ],
      "metadata": {
        "id": "8KOhglu5NiXr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "762e2f77-0e57-478a-d430-4e0359152ef1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 301)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "embedding_ft_df = pd.DataFrame(embedding_ft_full)\n",
        "embedding_ft_sample = np.array(embedding_ft_df.sample(int(embedding_ft_df.shape[0]*0.2),\n",
        "                                                       random_state=22))\n",
        "print(embedding_ft_sample.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D88xmGxp_dzi",
        "outputId": "8675ebb9-7876-498e-b48b-79dc781ccba1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(10000, 301)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## ELMo Embeddings"
      ],
      "metadata": {
        "id": "YC8eocmIt0N-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset_prc20 = dataset.sample(int(dataset.shape[0]*0.2),\n",
        "                            random_state=22)\n",
        "label_col_imdb = np.array(dataset_prc20.iloc[:,-1]).reshape(dataset_prc20.shape[0],1)"
      ],
      "metadata": {
        "id": "b_3Jt68Lu2yS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "hf_elmo = h5py.File('/content/drive/MyDrive/Lariba/IMDb/elmo_imdb_clf.h5', 'r')\n",
        "hf_elmo_data = np.array(hf_elmo.get('elmo_imdb_clf'))\n",
        "hf_elmo.close()\n",
        "hf_elmo = np.append(hf_elmo_data, label_col_imdb, axis=1)"
      ],
      "metadata": {
        "id": "e-BsDJ_Wt7Hr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Classification on Bert, Word2Vec, Fast Text and ELMo"
      ],
      "metadata": {
        "id": "o6rJ9LWJmo70"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Bert** Full datasets"
      ],
      "metadata": {
        "id": "xam9VoprnNTp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with warnings.catch_warnings():\n",
        "    warnings.simplefilter(\"ignore\")\n",
        "    clf = LogisticRegression(solver = \"lbfgs\", random_state = 0)\n",
        "    clf.fit(train_hidden, train_lab_hidden)"
      ],
      "metadata": {
        "id": "IATw5p9j2qnC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "BERT"
      ],
      "metadata": {
        "id": "ORioxBo2Xw1_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score no CLS token avg BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score no CLS token avg BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7WNs6uK52sJV",
        "outputId": "bc29f3e6-fe7b-42ac-c76c-29e7d516b458"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score no CLS token avg BERT 0.8771316571371501\n",
            "Test score no CLS token avg BERT 0.8761684591416222\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 4 layers BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 4 layers BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pfImeUo5w2Lv",
        "outputId": "d8c04fb0-8508-48ca-a036-208eb7478ced"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score CLS concatenated last 4 layers BERT 0.8441487103942473\n",
            "Test score CLS concatenated last 4 layers BERT 0.8419941702683686\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 3 layers BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 3 layers BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "ICttoJopf70M",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5e2a7f92-bc70-4bbe-aee6-3794613ef7df"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score CLS concatenated last 3 layers BERT 0.8420014481536041\n",
            "Test score CLS concatenated last 3 layers BERT 0.8393808422957081\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 2 layers BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 2 layers BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "hWNaVO4ArKiX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "96f5a85a-9add-4847-b599-7b8c7fbf5276"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score CLS concatenated last 2 layers BERT 0.8135876757134654\n",
            "Test score CLS concatenated last 2 layers BERT 0.8126444868831039\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 1 layer BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 1 layer BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pavMVOq45eUN",
        "outputId": "7d9543e8-94bb-45e7-a0b4-0cf4d40cd3d1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score CLS concatenated last 1 layer BERT 0.7828019275423834\n",
            "Test score CLS concatenated last 1 layer BERT 0.7783696853955171\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "RoBERT"
      ],
      "metadata": {
        "id": "SN1m678Ox8Ss"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score no CLS token avg RoBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score no CLS token avg RoBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "uGqiM5UryChB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 4 layers RoBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 4 layers RoBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "1ZDo-yoQrd_v",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8602872f-bf99-4fce-9d1c-bbf8978771f4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score CLS concatenated last 4 layers RoBERT 0.8537864223115528\n",
            "Test score CLS concatenated last 4 layers RoBERT 0.8517438938586792\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 3 layers RoBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 3 layers RoBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Exfwe0I8yCQo",
        "outputId": "23daf7ba-a59d-4883-d884-a3c8c7b750d5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score CLS concatenated last 3 layers RoBERT 0.824773413897281\n",
            "Test score CLS concatenated last 3 layers RoBERT 0.8231983113880792\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 2 layers RoBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 2 layers RoBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "l01epxSgf81V",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2a3110fd-2e9c-4285-b210-f2387a366b11"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score CLS concatenated last 2 layers RoBERT 0.813862325534943\n",
            "Test score CLS concatenated last 2 layers RoBERT 0.8182731932857573\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 1 layer RoBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 1 layer RoBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sQ-cWdQltGTb",
        "outputId": "7717b101-d38c-4a4c-d4d9-8f27004c2264"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score CLS concatenated last 1 layer RoBERT 0.8166587600808969\n",
            "Test score CLS concatenated last 1 layer RoBERT 0.8220926726304151\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "AlBERT"
      ],
      "metadata": {
        "id": "7FSkl_CLXdCp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score no CLS token avg AlBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score no CLS token avg AlBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4SCZiDYZX9Nj",
        "outputId": "2394e695-2216-4be2-c8c1-fb931600fe2d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score no CLS token avg AlBERT 0.677511173254101\n",
            "Test score no CLS token avg AlBERT 0.6797668107347472\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 4 layers AlBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 4 layers AlBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "9iBAn9qBf-PA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c48144b1-3fc6-46ce-cc44-d8fe31857693"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score CLS concatenated last 4 layers AlBERT 0.6793588175076777\n",
            "Test score CLS concatenated last 4 layers AlBERT 0.6802693738064127\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 3 layers AlBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 3 layers AlBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "SEHjG2M1rgaT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f2e7294a-a8f0-4a62-985b-7024f0cc9998"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score CLS concatenated last 3 layers AlBERT 0.6819305385633317\n",
            "Test score CLS concatenated last 3 layers AlBERT 0.6787616845914162\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 2 layers AlBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 2 layers AlBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "_DecfEqxrgTs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "864857ac-7b72-4466-f201-83803bf5ac07"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score CLS concatenated last 2 layers AlBERT 0.6789842950238446\n",
            "Test score CLS concatenated last 2 layers AlBERT 0.6772539953764197\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 1 layer AlBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 1 layer AlBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "As-bSZsr6KNA",
        "outputId": "cecc7603-8f21-405e-f97b-3296cd660d43"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score CLS concatenated last 1 layer AlBERT 0.6684726973109286\n",
            "Test score CLS concatenated last 1 layer AlBERT 0.6677052970147753\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "DistilBERT"
      ],
      "metadata": {
        "id": "nCpB0P9GXj1q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score no CLS token avg DistilBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score no CLS token vag DistilBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mqlIHiHQX_NB",
        "outputId": "8049dc09-7cd7-42f2-bdbe-8b7ace3459be"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score no CLS token avg DistilBERT 0.8690419714863549\n",
            "Test score no CLS token vag DistilBERT 0.8662177103226455\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 4 layers DistilBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 4 layers DistilBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "6I3BaWfTrrvD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 3 layers DistilBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 3 layers DistilBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "XqYkI-1ZrrnH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 2 layers AlBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 2 layers AlBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "0MA50uTMf_Hg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 1 layer AlBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 1 layer AlBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "60X7ViV9sTgq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "TinyBERT"
      ],
      "metadata": {
        "id": "mcRp5yD3XloE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score no CLS TinyBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score no CLS TinyBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Wj548gvOYBwB",
        "outputId": "3ad5127a-a5cb-4a35-ce19-22b4085d53ed"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score no CLS TinyBERT 0.8327881950513096\n",
            "Test score no CLS TinyBERT 0.8316413709920595\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 4 layers TinyBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 4 layers TinyBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "y5Va7Ucnrzse"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 3 layers TinyBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 3 layers TinyBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KmcRqf62YBsS",
        "outputId": "d9c2e8c2-2a8b-40d8-ae47-02bcc18bf476"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score only CLS TinyBERT 0.8147611794961425\n",
            "Test score only CLS TinyBERT 0.811840385968439\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 2 layers TinyBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 2 layers TinyBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "kr2-huKEgAPW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 1 layer TinyBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 1 layer TinyBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "gDmRnSDcsPGy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Sentence-BERT"
      ],
      "metadata": {
        "id": "DYoLgn5SRJCh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score no CLS TinyBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score no CLS TinyBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "xlEozVTMRIfn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "172081fc-9964-449d-8702-fb6267cea73a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score no CLS TinyBERT 0.8810017228034256\n",
            "Test score no CLS TinyBERT 0.8798874258719469\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 4 layers Sentence-BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 4 layers Sentence-BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "33PR8QagRIZM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 3 layers Sentence-BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 3 layers Sentence-BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "HC90CMn9gBKK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 2 layers Sentence-BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 2 layers Sentence-BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "sb3SEDY3r7Vc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 1 layer Sentence-BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 1 layer Sentence-BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "FtdjTpJcsWtj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **BERT** 20% of data Sampled"
      ],
      "metadata": {
        "id": "gdtynNnKwd6r"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with warnings.catch_warnings():\n",
        "    warnings.simplefilter(\"ignore\")\n",
        "    clf = LogisticRegression(solver = \"lbfgs\", random_state = 0)\n",
        "    clf.fit(train_hidden, train_lab_hidden)"
      ],
      "metadata": {
        "id": "Gpoe10QYwn8J"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "BERT"
      ],
      "metadata": {
        "id": "JiIHdDdXwqwl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score no CLS token avg BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score no CLS token avg BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "ZbXWgZv2wqGk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 4 layers BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 4 layers BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "zuHvbBVMwrld"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 3 layers BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 3 layers BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "A9k_XuVZwuaQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 2 layers BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 2 layers BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "p0ssNHlgwu8T"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 1 layer BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 1 layer BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "uvAEWT9t4-x1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "RoBERT"
      ],
      "metadata": {
        "id": "iFkiI1rKwzYZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score no CLS token avg RoBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score no CLS token avg RoBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "9mAku1Snw1_B"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 4 layers RoBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 4 layers RoBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "X14zonkxw3ti"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 3 layers RoBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 3 layers RoBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "hrr4usP6w60S"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 2 layers RoBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 2 layers RoBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "KadnMQZ-w8LT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 1 layer RoBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 1 layer RoBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "jdQFkfRxuEp2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "ALBERT"
      ],
      "metadata": {
        "id": "iAUzzybGEocz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score no CLS token avg ALBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score no CLS token avg ALBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "XSyGcAZ-EoIp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 4 layers ALBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 4 layers ALBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "j6eySh79uYnc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 3 layers ALBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 3 layers ALBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "IVPNeyWwuYkF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 2 layers ALBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 2 layers ALBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "P7D2V_HRuYgK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 1 layer ALBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 1 layer ALBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "wNat80aguYUB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "DistilBERT"
      ],
      "metadata": {
        "id": "rx_pBcrcE_L8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score no CLS token avg DistilBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score no CLS token avg DistilBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "8y0h6YceFBZM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 4 layers DistilBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 4 layers DistilBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "-VHytR40umCn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 3 layers DistilBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 3 layers DistilBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "z6vyuE4Wusyn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 2 layers DistilBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 2 layers DistilBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "ioYPF49DusHd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 1 layer DistilBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 1 layer DistilBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "PUlAkjRYusDs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "TinyBERT"
      ],
      "metadata": {
        "id": "ol56Jh_XFG7V"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score no CLS token avg TinyBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score no CLS token avg TinyBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "tk6RlVzDFSHE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 4 layers TinyBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 4 layers TinyBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "mzI-nVXcu0At"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 3 layers TinyBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 3 layers TinyBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "G-jXtrZ4uz8S"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 2 layers TinyBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 2 layers TinyBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "y1zjVMKzuz3E"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 1 layer TinyBERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 1 layer TinyBERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "YsgmMLLJuzyB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Sentence-BERT"
      ],
      "metadata": {
        "id": "uJd8HnwoFY3B"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score no CLS token avg Sentence-BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score no CLS token avg Sentence-BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "sRYlVKoZFcB8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 4 layers Sentence-BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 4 layers Sentence-BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "gJTX3FA4vCUo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 3 layers Sentence-BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 3 layers Sentence-BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "ahdfnRmkvFbW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 2 layers Sentence-BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 2 layers Sentence-BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "2d7E2M4jvGvX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score CLS concatenated last 1 layer Sentence-BERT %s' % clf.score(train_hidden, train_lab_hidden))\n",
        "print('Test score CLS concatenated last 1 layer Sentence-BERT %s' % clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "d7TKQ0Y4vGqr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Word2Vec"
      ],
      "metadata": {
        "id": "JA_E9K8Lnfhk"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Splitting the Data"
      ],
      "metadata": {
        "id": "7ixmXqfPsjcb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "## Input embedding_wv_full to get estimates for the full dataset\n",
        "training, testing = (Splitter(embedding_wv_sample)[0], Splitter(embedding_wv_sample)[1])\n",
        "train_dat_wv, train_lab_wv = (training[:,:training.shape[1]-1], training[:,-1])\n",
        "test_dat_wv, test_lab_wv = (testing[:,:testing.shape[1]-1], testing[:,-1])\n",
        "train_dat_wv = train_dat_wv.astype('float64')\n",
        "test_dat_wv = test_dat_wv.astype('float64')\n",
        "train_lab_wv = train_lab_wv.astype('int')\n",
        "test_lab_wv = test_lab_wv.astype('int')"
      ],
      "metadata": {
        "id": "nmK0uMF1_C8m"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Shape of the Train data is %s, Shape of Train labels is %s ' % (train_dat_wv.shape, train_lab_wv.shape))\n",
        "print('Shape of the Test data is  %s,  Shape of Test labels is %s' % (test_dat_wv.shape, test_lab_wv.shape))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ULuTjoFG_E-S",
        "outputId": "a771556f-e263-40df-ab86-44a96dc1ea4d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape of the Train data is (8028, 300), Shape of Train labels is (8028,) \n",
            "Shape of the Test data is  (1972, 300),  Shape of Test labels is (1972,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Classifier"
      ],
      "metadata": {
        "id": "4nQx-DKtsmLh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with warnings.catch_warnings():\n",
        "    warnings.simplefilter(\"ignore\")\n",
        "    clf = LogisticRegression(solver = \"lbfgs\",\n",
        "                             random_state = 0)\n",
        "    clf.fit(train_dat_wv, train_lab_wv)"
      ],
      "metadata": {
        "id": "8hrb6SXengwB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score on full data %s' % clf.score(train_dat_wv, train_lab_wv))\n",
        "print('Test score on full data %s' % clf.score(test_dat_wv, test_lab_wv))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "undSdH0HoeWv",
        "outputId": "39a1ac0c-6108-4f72-862b-ff28b48e4bbc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score on full data 0.8355754857997011\n",
            "Test score on full data 0.8331643002028397\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score on sample data %s' % clf.score(train_dat_wv, train_lab_wv))\n",
        "print('Test score on sample data %s' % clf.score(test_dat_wv, test_lab_wv))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EE4rAZJF9k8z",
        "outputId": "7cb0536e-b37e-48fc-a13e-cde182628fe6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score on sample data 0.8355754857997011\n",
            "Test score on sample data 0.8331643002028397\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Fast Text"
      ],
      "metadata": {
        "id": "HUMrZQvPFJkn"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Splitting the Data"
      ],
      "metadata": {
        "id": "IKcVRo9AsuB4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "## Input embedding_ft_full to get estimates for the full dataset\n",
        "training, testing = (Splitter(embedding_ft_sample)[0], Splitter(embedding_ft_sample)[1])\n",
        "train_dat_ft, train_lab_ft = (training[:,:training.shape[1]-1], training[:,-1])\n",
        "test_dat_ft, test_lab_ft = (testing[:,:testing.shape[1]-1], testing[:,-1])\n",
        "train_dat_ft = train_dat_ft.astype('float64')\n",
        "test_dat_ft = test_dat_ft.astype('float64')\n",
        "train_lab_ft = train_lab_ft.astype('int')\n",
        "test_lab_ft = test_lab_ft.astype('int')"
      ],
      "metadata": {
        "id": "iwYdShG5_oR7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Shape of the Train data is %s, Shape of Train labels is %s ' % (train_dat_ft.shape, train_lab_ft.shape))\n",
        "print('Shape of the Test data is  %s,  Shape of Test labels is %s' % (test_dat_ft.shape, test_lab_ft.shape))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9Rv-P_y9_oyX",
        "outputId": "0f1b3a20-db6a-42c7-9554-69d7aee3cffa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape of the Train data is (8028, 300), Shape of Train labels is (8028,) \n",
            "Shape of the Test data is  (1972, 300),  Shape of Test labels is (1972,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Classifier"
      ],
      "metadata": {
        "id": "q0HGdradsu1w"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with warnings.catch_warnings():\n",
        "    warnings.simplefilter(\"ignore\")\n",
        "    clf = LogisticRegression(solver = \"lbfgs\",\n",
        "                             random_state = 0)\n",
        "    clf.fit(train_dat_ft, train_lab_ft)"
      ],
      "metadata": {
        "id": "aMqRl-gSFOnn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score on full data %s' % clf.score(train_dat_ft, train_lab_ft))\n",
        "print('Test score on full data %s' % clf.score(test_dat_ft, test_lab_ft))"
      ],
      "metadata": {
        "id": "7f1EIlseFOlB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aed41515-bee4-476a-f357-f71ade363bc4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score on full data 0.8143996013951171\n",
            "Test score on full data 0.8113590263691683\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train score on sample data %s' % clf.score(train_dat_ft, train_lab_ft))\n",
        "print('Test score on sample data %s' % clf.score(test_dat_ft, test_lab_ft))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Nr4fmNAn_27n",
        "outputId": "689cb269-11a3-4813-dd95-203539536e77"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train score on sample data 0.8143996013951171\n",
            "Test score on sample data 0.8113590263691683\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### ELMo"
      ],
      "metadata": {
        "id": "hq0DaotEuKxb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "## Input hf_elmo is the sample dataset already\n",
        "training, testing = (Splitter(hf_elmo)[0], Splitter(hf_elmo)[1])\n",
        "train_elmo, train_lab_elmo = (training[:,:training.shape[1]-1], training[:,-1])\n",
        "test_elmo, test_lab_elmo = (testing[:,:testing.shape[1]-1], testing[:,-1])\n",
        "train_elmo = train_elmo.astype('float64')\n",
        "test_elmo = test_elmo.astype('float64')\n",
        "train_lab_elmo = train_lab_elmo.astype('int')\n",
        "test_lab_elmo = test_lab_elmo.astype('int')"
      ],
      "metadata": {
        "id": "LEZ8myiPt7Dj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with warnings.catch_warnings():\n",
        "    warnings.simplefilter(\"ignore\")\n",
        "    clf = LogisticRegression(solver = \"lbfgs\", random_state = 0)\n",
        "    clf.fit(train_hidden, train_lab_hidden)"
      ],
      "metadata": {
        "id": "NYcuE-DuuOHv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# imdb 20% \n",
        "# (the embedding matrix is already a 20% sample no need to slice)\n",
        "\n",
        "print(clf.score(train_hidden, train_lab_hidden))\n",
        "print(clf.score(test_hidden, test_lab_hidden))"
      ],
      "metadata": {
        "id": "RUN_kMH_uOqE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Splitting the data into training and validation sets"
      ],
      "metadata": {
        "id": "EtErM8Rz1JJc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# randomly sample 20% of the data\n",
        "\n",
        "dataframe_intent = pd.DataFrame(dataset)\n",
        "dataset_prc20 = np.array(dataframe_intent.sample(int(dataframe_intent.shape[0]*0.2),\n",
        "                                                       random_state=22))\n",
        "print(dataset_prc20.shape)"
      ],
      "metadata": {
        "id": "NhPO-Cr5NoH9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = np.array(dataset) # change to dataset_prc20 for the sampled dataset\n",
        "training, testing = (Splitter(data)[0], Splitter(data)[1])\n",
        "train_dat, train_lab = (training[:,:training.shape[1]-1], training[:,-1])\n",
        "test_dat, test_lab = (testing[:,:testing.shape[1]-1], testing[:,-1])\n",
        "train_lab = train_lab.astype('int')\n",
        "test_lab = test_lab.astype('int')"
      ],
      "metadata": {
        "id": "2oqArn49cGhq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train sentences shape is %s, Train labels shape is %s' % (train_dat.shape, train_lab.shape))\n",
        "print('Test sentences shape is %s, Test labels shape is %s' % (test_dat.shape, test_lab.shape))"
      ],
      "metadata": {
        "id": "rVzF0Ut5txDy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "## Baseline classifier using Count Vectorizer on Full Data"
      ],
      "metadata": {
        "id": "FzfO44bCpeFX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#nltk.download('punkt')\n",
        "vectorizer_count= CountVectorizer()\n",
        "cv_train_tokens = vectorizer_count.fit_transform(train_dat[:,0])\n",
        "cv_test_tokens = vectorizer_count.transform(test_dat[:,0])"
      ],
      "metadata": {
        "id": "BEbY9iNlwi1v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(cv_train_tokens.shape, len(train_lab))\n",
        "print(cv_test_tokens.shape, len(test_lab))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eMopKI3SxsSC",
        "outputId": "c3bacca7-c196-4fd9-8349-e71a080a0f61"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(40051, 92899) 40051\n",
            "(9949, 92899) 9949\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(train_dat.shape, test_dat.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IMVP0QqAdvHK",
        "outputId": "9f19cdf2-e90f-405b-9a6e-9672122124a0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(40051, 1) (9949, 1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Logistic Regression Classifier"
      ],
      "metadata": {
        "id": "jxpL7O9Mpslf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with warnings.catch_warnings():\n",
        "    warnings.simplefilter(\"ignore\")\n",
        "    clf_lr_count = LogisticRegression(solver = \"lbfgs\",\n",
        "                                      random_state=0)\n",
        "    clf_fit_count = clf_lr_count.fit(cv_train_tokens,train_lab)\n",
        "    y_pred_count = clf_lr_count.predict(cv_test_tokens)\n",
        "    y_true_count = test_lab"
      ],
      "metadata": {
        "id": "4rOLpyNtnz9y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(accuracy_score(y_true_count, y_pred_count))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YY8h_9Hnhg1L",
        "outputId": "cf5d0673-cf7a-4bad-dc27-cf73b014d796"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.8926525278922505\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Baseline classifier using Count Vectorizer on 20% of the Data"
      ],
      "metadata": {
        "id": "ZxDcqDXQBYU_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vectorizer_count= CountVectorizer()\n",
        "cv_train_tokens_sample = vectorizer_count.fit_transform(train_dat[:,0])\n",
        "cv_test_tokens_sample = vectorizer_count.transform(test_dat[:,0])"
      ],
      "metadata": {
        "id": "Wv2Y-GQVBnZy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with warnings.catch_warnings():\n",
        "    warnings.simplefilter(\"ignore\")\n",
        "    clf_lr_count = LogisticRegression(solver = \"lbfgs\",\n",
        "                                      random_state=0)\n",
        "    clf_fit_count = clf_lr_count.fit(cv_train_tokens_sample,train_lab)\n",
        "    y_pred_count = clf_lr_count.predict(cv_test_tokens_sample)\n",
        "    y_true_count = test_lab"
      ],
      "metadata": {
        "id": "lohUBS_kBnQ6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(accuracy_score(y_true_count, y_pred_count))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w0pcbPFKBnNY",
        "outputId": "c98a4574-7c9a-4cf5-d27b-6d6bc22f25eb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.8732251521298174\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Baseline Classifier using TFIDF on full Data"
      ],
      "metadata": {
        "id": "D3Io7mXdp24T"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "nltk.download('punkt')\n",
        "vectorizer_tfidf = TfidfVectorizer(tokenizer=nltk.word_tokenize)\n",
        "tf_train_tokens = vectorizer_tfidf.fit_transform(train_dat[:,0])\n",
        "tf_test_tokens = vectorizer_tfidf.transform(test_dat[:,0])"
      ],
      "metadata": {
        "id": "mZuXfIHepyLi",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2f34cae7-48ac-4770-9bc5-ab2efd0d4eda"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(tf_train_tokens.shape, len(train_lab))\n",
        "print(tf_test_tokens.shape, len(test_lab))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NgwF49F1p6xn",
        "outputId": "42a869db-a47e-4554-e292-69c56aa3d67e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(40051, 149233) 40051\n",
            "(9949, 149233) 9949\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Logistic Regression Classifier"
      ],
      "metadata": {
        "id": "dUYomcKkqFtM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with warnings.catch_warnings():\n",
        "    warnings.simplefilter(\"ignore\")\n",
        "    clf_lr_tfidf = LogisticRegression(solver = \"lbfgs\",\n",
        "                                      random_state=0)\n",
        "    clf_fit_tfidf = clf_lr_tfidf.fit(tf_train_tokens,train_lab)\n",
        "    y_pred_tfidf = clf_lr_tfidf.predict(tf_test_tokens)\n",
        "    y_true_tfidf = test_lab"
      ],
      "metadata": {
        "id": "brY5LDIUqB3E"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(accuracy_score(y_true_tfidf, y_pred_tfidf))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eCvKEdldiJz2",
        "outputId": "edee5cd5-fe45-4da3-e992-56786608d7fd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.9027037893255604\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Baseline Classifier using TFIDF on 20% of the Data"
      ],
      "metadata": {
        "id": "mQJHMMbOCBUE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "nltk.download('punkt')\n",
        "vectorizer_tfidf = TfidfVectorizer(tokenizer=nltk.word_tokenize)\n",
        "tf_train_tokens_sample = vectorizer_tfidf.fit_transform(train_dat[:,0])\n",
        "tf_test_tokens_sample = vectorizer_tfidf.transform(test_dat[:,0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I_1GtWGvDEg5",
        "outputId": "72ed05e7-c79f-4efb-e1e5-1273df768531"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(tf_train_tokens.shape, len(train_lab))\n",
        "print(tf_test_tokens.shape, len(test_lab))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-wKsZnaWDERF",
        "outputId": "894ff817-3273-4ddb-acc4-c3641f5a405b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(40051, 2017959) 8028\n",
            "(9949, 2017959) 1972\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with warnings.catch_warnings():\n",
        "    warnings.simplefilter(\"ignore\")\n",
        "    clf_lr_tfidf = LogisticRegression(solver = \"lbfgs\",\n",
        "                                      random_state=0)\n",
        "    clf_fit_tfidf = clf_lr_tfidf.fit(tf_train_tokens_sample,train_lab)\n",
        "    y_pred_tfidf = clf_lr_tfidf.predict(tf_test_tokens_sample)\n",
        "    y_true_tfidf = test_lab"
      ],
      "metadata": {
        "id": "UMsULrkZDELQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(accuracy_score(y_true_tfidf, y_pred_tfidf))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LBvQPBqVDI7c",
        "outputId": "197b6773-1a4a-4c72-e558-7128c15e2409"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.8737322515212982\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##### Checking inference time of Robert (The Best Performing Model according to the final results)\n"
      ],
      "metadata": {
        "id": "LgQB8OYtOIY_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# We sample 1000 data points randomly and then run the algorithm on those points\n",
        "data_1000_sample = np.array(dataset.sample(1000))\n",
        "data_1000_sample.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y66F7tnpOH4p",
        "outputId": "ce740c27-c353-4783-e407-ca950c26370a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(100, 2)"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data = data_1000_sample\n",
        "training, testing = (Splitter(data)[0], Splitter(data)[1])\n",
        "train_dat, train_lab = (training[:,:training.shape[1]-1], training[:,-1])\n",
        "test_dat, test_lab = (testing[:,:testing.shape[1]-1], testing[:,-1])\n",
        "train_lab = train_lab.astype('int')\n",
        "test_lab = test_lab.astype('int')"
      ],
      "metadata": {
        "id": "PUkjWWHs4S0Z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "start_time = time.time()\n",
        "\n",
        "nltk.download('punkt')\n",
        "vectorizer_tfidf = TfidfVectorizer(tokenizer=nltk.word_tokenize)\n",
        "tf_train_tokens_sample = vectorizer_tfidf.fit_transform(train_dat[:,0])\n",
        "tf_test_tokens_sample = vectorizer_tfidf.transform(test_dat[:,0])\n",
        "\n",
        "with warnings.catch_warnings():\n",
        "    warnings.simplefilter(\"ignore\")\n",
        "    clf_lr_tfidf = LogisticRegression(solver = \"lbfgs\",\n",
        "                                      random_state=0)\n",
        "    clf_fit_tfidf = clf_lr_tfidf.fit(tf_train_tokens_sample,train_lab)\n",
        "    y_pred_tfidf = clf_lr_tfidf.predict(tf_test_tokens_sample)\n",
        "    y_true_tfidf = test_lab\n",
        "\n",
        "end_time = time.time()\n",
        "print(f'\\n Inference ran for {round((end_time -  start_time))} seconds for 1000 datapoints')\n",
        "print(f' \\n For 1 datapoint inference ran for {round((end_time -  start_time)/1000, 2)} seconds')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ogzERv6AOapd",
        "outputId": "a483ed51-67c0-4232-f2bc-f4c26683f5ef"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "\n",
            " Inference ran for 13 seconds for 1000 datapoints\n",
            " \n",
            " For 1 datapoint inference ran for 0.01 seconds\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## bigram of TF-IDF and CountVectorizer on full Dataset"
      ],
      "metadata": {
        "id": "5lZsyM0aDOqb"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Initialize Vectorizer"
      ],
      "metadata": {
        "id": "lkBIXFnWtTSr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vectorizer_count= CountVectorizer(ngram_range=(2,2))\n",
        "cv_train_tokens = vectorizer_count.fit_transform(train_dat[:,0])\n",
        "cv_test_tokens = vectorizer_count.transform(test_dat[:,0])"
      ],
      "metadata": {
        "id": "oH4FPteQDOTA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Classifier"
      ],
      "metadata": {
        "id": "pTNTzHoitWMr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with warnings.catch_warnings():\n",
        "    warnings.simplefilter(\"ignore\")\n",
        "    clf_lr_count = LogisticRegression(solver = \"lbfgs\",\n",
        "                                      random_state=0)\n",
        "    clf_fit_count = clf_lr_count.fit(cv_train_tokens,train_lab)\n",
        "    y_pred_count = clf_lr_count.predict(cv_test_tokens)\n",
        "    y_true_count = test_lab"
      ],
      "metadata": {
        "id": "kIC4qktuZ0tk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('accuracy socre for CountVecotrizer is:' ,accuracy_score(y_true_count, y_pred_count))\n",
        "count_bgram_acc = accuracy_score(y_true_count, y_pred_count)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VtjdCETFZ-WW",
        "outputId": "66cff84f-c16e-4fa2-e0d2-4734147e09e9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "accuracy socre for CountVecotrizer is: 0.8978791838375716\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Initialize Vectorizer"
      ],
      "metadata": {
        "id": "cJO3rlmktXud"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vectorizer_tfidf = TfidfVectorizer(use_idf=True, ngram_range=(2,2))\n",
        "tf_train_tokens = vectorizer_tfidf.fit_transform(train_dat[:,0])\n",
        "tf_test_tokens = vectorizer_tfidf.transform(test_dat[:,0])"
      ],
      "metadata": {
        "id": "WkSOI6DiaFsu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Classifier"
      ],
      "metadata": {
        "id": "lXSLUCoutYQl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with warnings.catch_warnings():\n",
        "    warnings.simplefilter(\"ignore\")\n",
        "    clf_lr_tfidf = LogisticRegression(solver = \"lbfgs\",\n",
        "                                      multi_class='multinomial',\n",
        "                                      random_state=0)\n",
        "    clf_fit_tfidf = clf_lr_tfidf.fit(tf_train_tokens,train_lab)\n",
        "    y_pred_tfidf = clf_lr_tfidf.predict(tf_test_tokens)\n",
        "    y_true_tfidf = test_lab"
      ],
      "metadata": {
        "id": "EljA1IEqaFd0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('accuracy socre is for TFIDF:' ,accuracy_score(y_true_tfidf, y_pred_tfidf))\n",
        "tfidf_bgram_acc = accuracy_score(y_true_tfidf, y_pred_tfidf)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mIzLtU6-aB0j",
        "outputId": "e5b69668-9c72-4836-d576-30e933de5210"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "accuracy socre is for TFIDF: 0.8956679063222435\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## bigram of TF-IDF and CountVectorizer on 20% of Data"
      ],
      "metadata": {
        "id": "tnUiXADQDrKd"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Initialize Vectorizer"
      ],
      "metadata": {
        "id": "A0LM0PzqtIpj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vectorizer_count= CountVectorizer(ngram_range=(2,2))\n",
        "cv_train_tokens_sample = vectorizer_count.fit_transform(train_dat[:,0])\n",
        "cv_test_tokens_sample = vectorizer_count.transform(test_dat[:,0])"
      ],
      "metadata": {
        "id": "VWS4hW2aV9U3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Classifier"
      ],
      "metadata": {
        "id": "kLiGMkt3tLR2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with warnings.catch_warnings():\n",
        "    warnings.simplefilter(\"ignore\")\n",
        "    clf_lr_count = LogisticRegression(solver = \"lbfgs\",\n",
        "                                      random_state=0)\n",
        "    clf_fit_count = clf_lr_count.fit(cv_train_tokens_sample,train_lab)\n",
        "    y_pred_count = clf_lr_count.predict(cv_test_tokens_sample)\n",
        "    y_true_count = test_lab"
      ],
      "metadata": {
        "id": "dL-yToujD8Zo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('accuracy socre for CountVecotrizer on 20% of data is:' ,accuracy_score(y_true_count,\n",
        "                                                                              y_pred_count))\n",
        "count_bgram_acc = accuracy_score(y_true_count,\n",
        "                                 y_pred_count)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E7ERFvP0D8Wr",
        "outputId": "787f506d-faa3-4087-f040-7e59cf938d38"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "accuracy socre for CountVecotrizer on 20% of data is: 0.8483772819472617\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Initialize Vectorizer"
      ],
      "metadata": {
        "id": "Y1xn5cKltN3i"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vectorizer_tfidf = TfidfVectorizer(use_idf=True, ngram_range=(2,2))\n",
        "tf_train_tokens_sample = vectorizer_tfidf.fit_transform(train_dat[:,0])\n",
        "tf_test_tokens_sample = vectorizer_tfidf.transform(test_dat[:,0])"
      ],
      "metadata": {
        "id": "fyA20NcnD8TY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Classifier"
      ],
      "metadata": {
        "id": "CHjcBD_UtQl-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with warnings.catch_warnings():\n",
        "    warnings.simplefilter(\"ignore\")\n",
        "    clf_lr_tfidf = LogisticRegression(solver = \"lbfgs\",\n",
        "                                      multi_class='multinomial',\n",
        "                                      random_state=0)\n",
        "    clf_fit_tfidf = clf_lr_tfidf.fit(tf_train_tokens_sample,train_lab)\n",
        "    y_pred_tfidf = clf_lr_tfidf.predict(tf_test_tokens_sample)\n",
        "    y_true_tfidf = test_lab"
      ],
      "metadata": {
        "id": "R1ujibq4D8Qc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('accuracy socre is for TFIDF on 20% of data is:' ,accuracy_score(y_true_tfidf, y_pred_tfidf))\n",
        "tfidf_bgram_acc = accuracy_score(y_true_tfidf, y_pred_tfidf)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rTcVzTNHD8M9",
        "outputId": "dcf3ace9-6b9e-4787-ecad-d31c48dfa919"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "accuracy socre is for TFIDF on 20% of data is: 0.8635902636916836\n"
          ]
        }
      ]
    }
  ]
}