Ch5/02_NERTraining.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P7pplVJgcnAQ"
      },
      "source": [
        "This notebook illustrates how to train a NER model using the well known CONLL dataset, and sklearn_crfsuite library."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_EImA3uAeUoS"
      },
      "source": [
        "### Importing Necessary Libraries"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Uncomment below lines to download libraries required for this notebook\n",
        "!pip install sklearn_crfsuite==0.3.6"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gb8AFLD_psqI",
        "outputId": "567114e8-153a-48ec-c4b8-7511b4d76d74"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: sklearn_crfsuite==0.3.6 in /usr/local/lib/python3.10/dist-packages (0.3.6)\n",
            "Requirement already satisfied: python-crfsuite>=0.8.3 in /usr/local/lib/python3.10/dist-packages (from sklearn_crfsuite==0.3.6) (0.9.9)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from sklearn_crfsuite==0.3.6) (1.16.0)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/dist-packages (from sklearn_crfsuite==0.3.6) (0.9.0)\n",
            "Requirement already satisfied: tqdm>=2.0 in /usr/local/lib/python3.10/dist-packages (from sklearn_crfsuite==0.3.6) (4.66.1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "260mW70zcnAR",
        "outputId": "65cb5ca2-eb4b-4577-c94f-dc3613c5c3ef"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
            "[nltk_data]       date!\n"
          ]
        }
      ],
      "source": [
        "#Make the necessary imports\n",
        "from nltk.tag import pos_tag\n",
        "from sklearn_crfsuite import CRF, metrics\n",
        "from sklearn.metrics import make_scorer,confusion_matrix,f1_score,classification_report\n",
        "from pprint import pprint\n",
        "from sklearn.pipeline import Pipeline\n",
        "import string\n",
        "import warnings\n",
        "import nltk\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "warnings.filterwarnings('ignore')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "23NhWgM4eT9l"
      },
      "source": [
        "### Loading The Data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "3f-Pvr5tcnAg"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "Load the training/testing data.\n",
        "input: conll format data, but with only 2 tab separated colums - words and NEtags.\n",
        "output: A list where each item is 2 lists.  sentence as a list of tokens, NER tags as a list for each token.\n",
        "\"\"\"\n",
        "def load__data_conll(file_path):\n",
        "    myoutput,words,tags = [],[],[]\n",
        "    fh = open(file_path)\n",
        "    for line in fh:\n",
        "        line = line.strip()\n",
        "        if \"\\t\" not in line:\n",
        "            #Sentence ended.\n",
        "            myoutput.append([words,tags])\n",
        "            words,tags = [],[]\n",
        "        else:\n",
        "            word, tag = line.split(\"\\t\")\n",
        "            words.append(word)\n",
        "            tags.append(tag)\n",
        "    fh.close()\n",
        "    return myoutput"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "EZNEu753cnAl"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "Get features for all words in the sentence\n",
        "Features:\n",
        "- word context: a window of 2 words on either side of the current word, and current word.\n",
        "- POS context: a window of 2 POS tags on either side of the current word, and current tag.\n",
        "input: sentence as a list of tokens.\n",
        "output: list of dictionaries. each dict represents features for that word.\n",
        "\"\"\"\n",
        "def sent2feats(sentence):\n",
        "    feats = []\n",
        "    sen_tags = pos_tag(sentence) #This format is specific to this POS tagger!\n",
        "    for i in range(0,len(sentence)):\n",
        "        word = sentence[i]\n",
        "        wordfeats = {}\n",
        "       #word features: word, prev 2 words, next 2 words in the sentence.\n",
        "        wordfeats['word'] = word\n",
        "        if i == 0:\n",
        "            wordfeats[\"prevWord\"] = wordfeats[\"prevSecondWord\"] = \"<S>\"\n",
        "        elif i==1:\n",
        "            wordfeats[\"prevWord\"] = sentence[0]\n",
        "            wordfeats[\"prevSecondWord\"] = \"</S>\"\n",
        "        else:\n",
        "            wordfeats[\"prevWord\"] = sentence[i-1]\n",
        "            wordfeats[\"prevSecondWord\"] = sentence[i-2]\n",
        "        #next two words as features\n",
        "        if i == len(sentence)-2:\n",
        "            wordfeats[\"nextWord\"] = sentence[i+1]\n",
        "            wordfeats[\"nextNextWord\"] = \"</S>\"\n",
        "        elif i==len(sentence)-1:\n",
        "            wordfeats[\"nextWord\"] = \"</S>\"\n",
        "            wordfeats[\"nextNextWord\"] = \"</S>\"\n",
        "        else:\n",
        "            wordfeats[\"nextWord\"] = sentence[i+1]\n",
        "            wordfeats[\"nextNextWord\"] = sentence[i+2]\n",
        "\n",
        "        #POS tag features: current tag, previous and next 2 tags.\n",
        "        wordfeats['tag'] = sen_tags[i][1]\n",
        "        if i == 0:\n",
        "            wordfeats[\"prevTag\"] = wordfeats[\"prevSecondTag\"] = \"<S>\"\n",
        "        elif i == 1:\n",
        "            wordfeats[\"prevTag\"] = sen_tags[0][1]\n",
        "            wordfeats[\"prevSecondTag\"] = \"</S>\"\n",
        "        else:\n",
        "            wordfeats[\"prevTag\"] = sen_tags[i - 1][1]\n",
        "\n",
        "            wordfeats[\"prevSecondTag\"] = sen_tags[i - 2][1]\n",
        "            # next two words as features\n",
        "        if i == len(sentence) - 2:\n",
        "            wordfeats[\"nextTag\"] = sen_tags[i + 1][1]\n",
        "            wordfeats[\"nextNextTag\"] = \"</S>\"\n",
        "        elif i == len(sentence) - 1:\n",
        "            wordfeats[\"nextTag\"] = \"</S>\"\n",
        "            wordfeats[\"nextNextTag\"] = \"</S>\"\n",
        "        else:\n",
        "            wordfeats[\"nextTag\"] = sen_tags[i + 1][1]\n",
        "            wordfeats[\"nextNextTag\"] = sen_tags[i + 2][1]\n",
        "        #That is it! You can add whatever you want!\n",
        "        feats.append(wordfeats)\n",
        "    return feats"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kBNaaDJce102"
      },
      "source": [
        "### Extracting Features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "LmB5emVicnA1"
      },
      "outputs": [],
      "source": [
        "#Extract features from the conll data, after loading it.\n",
        "def get_feats_conll(conll_data):\n",
        "    feats = []\n",
        "    labels = []\n",
        "    for sentence in conll_data:\n",
        "        feats.append(sent2feats(sentence[0]))\n",
        "        labels.append(sentence[1])\n",
        "    return feats, labels"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S7TO_UXBe6Lg"
      },
      "source": [
        "### Training a Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "7bo-XzixcnBA"
      },
      "outputs": [],
      "source": [
        "\n",
        "#Train a sequence model\n",
        "def train_seq(X_train,Y_train,X_dev,Y_dev):\n",
        "   # crf = CRF(algorithm='lbfgs', c1=0.1, c2=0.1, max_iterations=50, all_possible_states=True)\n",
        "    crf = CRF(algorithm='lbfgs', c1=0.1, c2=10, max_iterations=50)#, all_possible_states=True)\n",
        "    #Just to fit on training data\n",
        "    crf.fit(X_train, Y_train)\n",
        "    labels = list(crf.classes_)\n",
        "    #testing:\n",
        "    y_pred = crf.predict(X_dev)\n",
        "    sorted_labels = sorted(labels, key=lambda name: (name[1:], name[0]))\n",
        "    print(metrics.flat_f1_score(Y_dev, y_pred, average='weighted', labels=labels))\n",
        "    print(metrics.flat_classification_report(Y_dev, y_pred, labels=sorted_labels, digits=3))\n",
        "    #print(metrics.sequence_accuracy_score(Y_dev, y_pred))\n",
        "    get_confusion_matrix(Y_dev, y_pred, labels=sorted_labels)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8OyfLwkYe_lz"
      },
      "source": [
        "Confusion Matrix helper function . Source for this function is [here](https://gist.github.com/zachguo/10296432)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "khGXUWBScnBL"
      },
      "outputs": [],
      "source": [
        "def print_cm(cm, labels):\n",
        "    print(\"\\n\")\n",
        "    \"\"\"pretty print for confusion matrixes\"\"\"\n",
        "    columnwidth = max([len(x) for x in labels] + [5])  # 5 is value length\n",
        "    empty_cell = \" \" * columnwidth\n",
        "    # Print header\n",
        "    print(\"    \" + empty_cell, end=\" \")\n",
        "    for label in labels:\n",
        "        print(\"%{0}s\".format(columnwidth) % label, end=\" \")\n",
        "    print()\n",
        "    # Print rows\n",
        "    for i, label1 in enumerate(labels):\n",
        "        print(\"    %{0}s\".format(columnwidth) % label1, end=\" \")\n",
        "        sum = 0\n",
        "        for j in range(len(labels)):\n",
        "            cell = \"%{0}.0f\".format(columnwidth) % cm[i, j]\n",
        "            sum =  sum + int(cell)\n",
        "            print(cell, end=\" \")\n",
        "        print(sum) #Prints the total number of instances per cat at the end."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "5vy2dLg2cnBe"
      },
      "outputs": [],
      "source": [
        "#python-crfsuite does not have a confusion matrix function,\n",
        "#so writing it using sklearn's confusion matrix and print_cm from github\n",
        "def get_confusion_matrix(y_true,y_pred,labels):\n",
        "    trues,preds = [], []\n",
        "    for yseq_true, yseq_pred in zip(y_true, y_pred):\n",
        "        trues.extend(yseq_true)\n",
        "        preds.extend(yseq_pred)\n",
        "    print_cm(confusion_matrix(trues,preds,labels=labels),labels)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IYAlUXyRfPzf"
      },
      "source": [
        "### Call all our functions inside the main method"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 592
        },
        "id": "GtkLBZF-cnBr",
        "outputId": "221fc487-9bdc-46dc-f511-0e559005da1a"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-d68001ed-7224-4e9c-b3a3-3e380203efb0\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-d68001ed-7224-4e9c-b3a3-3e380203efb0\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script>// Copyright 2017 Google LLC\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "/**\n",
              " * @fileoverview Helpers for google.colab Python module.\n",
              " */\n",
              "(function(scope) {\n",
              "function span(text, styleAttributes = {}) {\n",
              "  const element = document.createElement('span');\n",
              "  element.textContent = text;\n",
              "  for (const key of Object.keys(styleAttributes)) {\n",
              "    element.style[key] = styleAttributes[key];\n",
              "  }\n",
              "  return element;\n",
              "}\n",
              "\n",
              "// Max number of bytes which will be uploaded at a time.\n",
              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
              "\n",
              "function _uploadFiles(inputId, outputId) {\n",
              "  const steps = uploadFilesStep(inputId, outputId);\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  // Cache steps on the outputElement to make it available for the next call\n",
              "  // to uploadFilesContinue from Python.\n",
              "  outputElement.steps = steps;\n",
              "\n",
              "  return _uploadFilesContinue(outputId);\n",
              "}\n",
              "\n",
              "// This is roughly an async generator (not supported in the browser yet),\n",
              "// where there are multiple asynchronous steps and the Python side is going\n",
              "// to poll for completion of each step.\n",
              "// This uses a Promise to block the python side on completion of each step,\n",
              "// then passes the result of the previous step as the input to the next step.\n",
              "function _uploadFilesContinue(outputId) {\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  const steps = outputElement.steps;\n",
              "\n",
              "  const next = steps.next(outputElement.lastPromiseValue);\n",
              "  return Promise.resolve(next.value.promise).then((value) => {\n",
              "    // Cache the last promise value to make it available to the next\n",
              "    // step of the generator.\n",
              "    outputElement.lastPromiseValue = value;\n",
              "    return next.value.response;\n",
              "  });\n",
              "}\n",
              "\n",
              "/**\n",
              " * Generator function which is called between each async step of the upload\n",
              " * process.\n",
              " * @param {string} inputId Element ID of the input file picker element.\n",
              " * @param {string} outputId Element ID of the output display.\n",
              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
              " */\n",
              "function* uploadFilesStep(inputId, outputId) {\n",
              "  const inputElement = document.getElementById(inputId);\n",
              "  inputElement.disabled = false;\n",
              "\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  outputElement.innerHTML = '';\n",
              "\n",
              "  const pickedPromise = new Promise((resolve) => {\n",
              "    inputElement.addEventListener('change', (e) => {\n",
              "      resolve(e.target.files);\n",
              "    });\n",
              "  });\n",
              "\n",
              "  const cancel = document.createElement('button');\n",
              "  inputElement.parentElement.appendChild(cancel);\n",
              "  cancel.textContent = 'Cancel upload';\n",
              "  const cancelPromise = new Promise((resolve) => {\n",
              "    cancel.onclick = () => {\n",
              "      resolve(null);\n",
              "    };\n",
              "  });\n",
              "\n",
              "  // Wait for the user to pick the files.\n",
              "  const files = yield {\n",
              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
              "    response: {\n",
              "      action: 'starting',\n",
              "    }\n",
              "  };\n",
              "\n",
              "  cancel.remove();\n",
              "\n",
              "  // Disable the input element since further picks are not allowed.\n",
              "  inputElement.disabled = true;\n",
              "\n",
              "  if (!files) {\n",
              "    return {\n",
              "      response: {\n",
              "        action: 'complete',\n",
              "      }\n",
              "    };\n",
              "  }\n",
              "\n",
              "  for (const file of files) {\n",
              "    const li = document.createElement('li');\n",
              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
              "    li.append(span(\n",
              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
              "        `last modified: ${\n",
              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
              "                                    'n/a'} - `));\n",
              "    const percent = span('0% done');\n",
              "    li.appendChild(percent);\n",
              "\n",
              "    outputElement.appendChild(li);\n",
              "\n",
              "    const fileDataPromise = new Promise((resolve) => {\n",
              "      const reader = new FileReader();\n",
              "      reader.onload = (e) => {\n",
              "        resolve(e.target.result);\n",
              "      };\n",
              "      reader.readAsArrayBuffer(file);\n",
              "    });\n",
              "    // Wait for the data to be ready.\n",
              "    let fileData = yield {\n",
              "      promise: fileDataPromise,\n",
              "      response: {\n",
              "        action: 'continue',\n",
              "      }\n",
              "    };\n",
              "\n",
              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
              "    let position = 0;\n",
              "    do {\n",
              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
              "      const chunk = new Uint8Array(fileData, position, length);\n",
              "      position += length;\n",
              "\n",
              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
              "      yield {\n",
              "        response: {\n",
              "          action: 'append',\n",
              "          file: file.name,\n",
              "          data: base64,\n",
              "        },\n",
              "      };\n",
              "\n",
              "      let percentDone = fileData.byteLength === 0 ?\n",
              "          100 :\n",
              "          Math.round((position / fileData.byteLength) * 100);\n",
              "      percent.textContent = `${percentDone}% done`;\n",
              "\n",
              "    } while (position < fileData.byteLength);\n",
              "  }\n",
              "\n",
              "  // All done.\n",
              "  yield {\n",
              "    response: {\n",
              "      action: 'complete',\n",
              "    }\n",
              "  };\n",
              "}\n",
              "\n",
              "scope.google = scope.google || {};\n",
              "scope.google.colab = scope.google.colab || {};\n",
              "scope.google.colab._files = {\n",
              "  _uploadFiles,\n",
              "  _uploadFilesContinue,\n",
              "};\n",
              "})(self);\n",
              "</script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training a Sequence classification model with CRF\n",
            "0.9255103670420659\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           O      0.973     0.981     0.977     38323\n",
            "       B-LOC      0.694     0.765     0.728      1668\n",
            "       I-LOC      0.738     0.482     0.584       257\n",
            "      B-MISC      0.648     0.309     0.419       702\n",
            "      I-MISC      0.626     0.505     0.559       216\n",
            "       B-ORG      0.670     0.561     0.611      1661\n",
            "       I-ORG      0.551     0.704     0.618       835\n",
            "       B-PER      0.773     0.766     0.769      1617\n",
            "       I-PER      0.819     0.886     0.851      1156\n",
            "\n",
            "    accuracy                          0.928     46435\n",
            "   macro avg      0.721     0.662     0.679     46435\n",
            "weighted avg      0.926     0.928     0.926     46435\n",
            "\n",
            "\n",
            "\n",
            "                O  B-LOC  I-LOC B-MISC I-MISC  B-ORG  I-ORG  B-PER  I-PER \n",
            "         O  37579    118      3     22     32    193    224     88     64 38323\n",
            "     B-LOC    143   1276      1     36      1     95     14     98      4 1668\n",
            "     I-LOC     32      6    124      1      5      0     52      0     37 257\n",
            "    B-MISC    344     48      1    217      2     56     13     19      2 702\n",
            "    I-MISC     58      1      4      4    109      2     29      0      9 216\n",
            "     B-ORG    265    236      0     48      3    932     20    151      6 1661\n",
            "     I-ORG     76     15     18      2     15     21    588      8     92 835\n",
            "     B-PER     86    138      1      5      3     90     44   1238     12 1617\n",
            "     I-PER     26      1     16      0      4      2     83      0   1024 1156\n",
            "Done with sequence model\n"
          ]
        }
      ],
      "source": [
        "from sklearn_crfsuite.utils import flatten\n",
        "\n",
        "def main():\n",
        "\n",
        "    try:\n",
        "        from google.colab import files\n",
        "        uploaded = files.upload()\n",
        "        # files are present in Data/conlldata\n",
        "        train_path = 'train.txt'\n",
        "        test_path = 'test.txt'\n",
        "    except:\n",
        "        train_path = 'Data/conlldata/train.txt'\n",
        "        test_path = 'Data/conlldata/test.txt'\n",
        "\n",
        "    conll_train = load__data_conll(train_path)\n",
        "    conll_dev = load__data_conll(test_path)\n",
        "\n",
        "    print(\"Training a Sequence classification model with CRF\")\n",
        "    feats, labels = get_feats_conll(conll_train)\n",
        "    devfeats, devlabels = get_feats_conll(conll_dev)\n",
        "    train_seq(feats, labels, devfeats, devlabels)\n",
        "    print(\"Done with sequence model\")\n",
        "\n",
        "if __name__==\"__main__\":\n",
        "    main()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-xwubY50cnB1"
      },
      "source": [
        "This is pretty good. We already have a model which has an F-score of 92%!!!"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "5jWKX7zBw1E2"
      },
      "execution_count": 28,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}