examples/convert-examples.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "convert-examples.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/PolyAI-LDN/polyai-models/blob/master/examples/convert-examples.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6VslZQSrMHCp",
        "colab_type": "text"
      },
      "source": [
        "# ConveRT\n",
        "## More Efficient and Accurate Conversational Representations from Transformers\n",
        "\n",
        "This notebook demonstrates how to use the ConveRT model, to encode sentences to a meaningful semantic space, and to rank responses in a conversation."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YVFi1UXcLwdo",
        "colab_type": "text"
      },
      "source": [
        "# Install dependencies\n",
        "\n",
        "The encoder model requires the subword tokenization ops from the [tensorflow.text](https://github.com/tensorflow/text) module to run. This is released for tensorflow 1.14, 1.15, and 2.0. In this colab we opt for tf 1.14."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5v96iAgaGyKt",
        "colab_type": "code",
        "outputId": "3e612a0c-be5e-4acc-b2c9-171a532adfea",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 734
        }
      },
      "source": [
        "!pip install tensorflow-gpu==1.14.0\n",
        "!pip install --no-deps tensorflow_text==0.1.0"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tensorflow-gpu==1.14.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/76/04/43153bfdfcf6c9a4c38ecdb971ca9a75b9a791bb69a764d652c359aca504/tensorflow_gpu-1.14.0-cp36-cp36m-manylinux1_x86_64.whl (377.0MB)\n",
            "\u001b[K     |████████████████████████████████| 377.0MB 41kB/s \n",
            "\u001b[?25hRequirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.8.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.1.7)\n",
            "Collecting tensorflow-estimator<1.15.0rc0,>=1.14.0rc0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/3c/d5/21860a5b11caf0678fbc8319341b0ae21a07156911132e0e71bffed0510d/tensorflow_estimator-1.14.0-py2.py3-none-any.whl (488kB)\n",
            "\u001b[K     |████████████████████████████████| 491kB 42.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.1.0)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.15.0)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.8.1)\n",
            "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.2.2)\n",
            "Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.17.3)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.12.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.33.6)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.11.2)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.1.0)\n",
            "Collecting tensorboard<1.15.0,>=1.14.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/91/2d/2ed263449a078cd9c8a9ba50ebd50123adf1f8cfbea1492f9084169b89d9/tensorboard-1.14.0-py3-none-any.whl (3.1MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2MB 59.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (3.10.0)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.0.8)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow-gpu==1.14.0) (0.16.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow-gpu==1.14.0) (41.4.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow-gpu==1.14.0) (3.1.1)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow-gpu==1.14.0) (2.8.0)\n",
            "\u001b[31mERROR: tensorflow 1.15.0 has requirement tensorboard<1.16.0,>=1.15.0, but you'll have tensorboard 1.14.0 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: tensorflow 1.15.0 has requirement tensorflow-estimator==1.15.1, but you'll have tensorflow-estimator 1.14.0 which is incompatible.\u001b[0m\n",
            "Installing collected packages: tensorflow-estimator, tensorboard, tensorflow-gpu\n",
            "  Found existing installation: tensorflow-estimator 1.15.1\n",
            "    Uninstalling tensorflow-estimator-1.15.1:\n",
            "      Successfully uninstalled tensorflow-estimator-1.15.1\n",
            "  Found existing installation: tensorboard 1.15.0\n",
            "    Uninstalling tensorboard-1.15.0:\n",
            "      Successfully uninstalled tensorboard-1.15.0\n",
            "Successfully installed tensorboard-1.14.0 tensorflow-estimator-1.14.0 tensorflow-gpu-1.14.0\n",
            "Collecting tensorflow_text==0.1.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f0/85/46411195f3961f4906c7e519d7a5cf017882d59149393d7c311e97cf7080/tensorflow_text-0.1.0-cp36-cp36m-manylinux1_x86_64.whl (6.2MB)\n",
            "\u001b[K     |████████████████████████████████| 6.2MB 8.9MB/s \n",
            "\u001b[?25hInstalling collected packages: tensorflow-text\n",
            "Successfully installed tensorflow-text-0.1.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KSyjAZvYHrRI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Import the required dependencies\n",
        "\n",
        "import tensorflow_hub as tfhub\n",
        "import tensorflow as tf\n",
        "import tensorflow_text  # required for tokenization ops\n",
        "\n",
        "sess = None\n",
        "\n",
        "assert tf.__version__ == \"1.14.0\", (\n",
        "    f\"found tf version {tf.__version__}, but need 1.14.0\")\n",
        "assert tf.test.is_gpu_available(), (\n",
        "    \"GPU not available. please use a GPU runtime\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fTlzHg_xMnNz",
        "colab_type": "text"
      },
      "source": [
        "# Computing sentence encodings\n",
        "\n",
        "The default signature for ConveRT encodes sentences to a meaningful semantic space. Sentences can be compared for semantic similarity in this space, and NLP classifiers can be trained on top of these encodings.\n",
        "\n",
        "This section demonstrates encoding texts with ConveRT, and computing similarities."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mLRhwAmSAhAp",
        "colab_type": "code",
        "outputId": "ccea72a4-d87c-41de-a549-d797ed312270",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "# Load the tensorflow hub module into a fresh session.\n",
        "\n",
        "if sess is not None:\n",
        "    sess.close()\n",
        "\n",
        "sess = tf.InteractiveSession(graph=tf.Graph())\n",
        "\n",
        "module = tfhub.Module(\"http://models.poly-ai.com/convert/v1/model.tar.gz\")\n",
        "\n",
        "text_placeholder = tf.placeholder(dtype=tf.string, shape=[None])\n",
        "encoding_tensor = module(text_placeholder)\n",
        "encoding_dim = int(encoding_tensor.shape[1])\n",
        "print(f\"ConveRT encodes text to {encoding_dim}-dimensional vectors\")\n",
        "\n",
        "sess.run(tf.tables_initializer())\n",
        "sess.run(tf.global_variables_initializer())\n",
        "\n",
        "\n",
        "def encode(texts):\n",
        "    \"\"\"Encode the given texts to the encoding space.\"\"\"\n",
        "    return sess.run(encoding_tensor, feed_dict={text_placeholder: texts})"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "ConveRT encodes text to 1024-dimensional vectors\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BFGC5EITDoqU",
        "colab_type": "code",
        "outputId": "7043917b-e0cc-45cb-d49b-1807499a194e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 392
        }
      },
      "source": [
        "# Compute the similarity between a collection of sentences.\n",
        "\n",
        "import numpy as np\n",
        "from sklearn.metrics import pairwise\n",
        "import seaborn\n",
        "\n",
        "texts = [\n",
        "         # booking a ride\n",
        "         \"can i book a car?\",\n",
        "         \"i need a ride from my place\",\n",
        "         \"book a taxi\",\n",
        "\n",
        "         # asking for ETA\n",
        "         \"when will it arrive?\",\n",
        "         \"what is the ETA?\",\n",
        "         \"when will it be here?\",\n",
        "\n",
        "         # asking to talk to a human\n",
        "         \"can i talk to a real person\",\n",
        "         \"i don't want to talk to a robot.\",\n",
        "         \"let me speak to a human.\",\n",
        "]\n",
        "\n",
        "# Pad with \"\" to make heatmap display better.\n",
        "texts = [\"\"] + texts + [\"\"]\n",
        "\n",
        "encodings = encode(texts)\n",
        "\n",
        "similarities = pairwise.cosine_similarity(encodings)\n",
        "seaborn.set(font_scale=0.8)\n",
        "g = seaborn.heatmap(\n",
        "    similarities,\n",
        "    xticklabels=texts,\n",
        "    yticklabels=texts,\n",
        "    vmin=0.,\n",
        "    vmax=1.,\n",
        "    cmap=\"BuGn\",\n",
        "    square=True,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbwAAAF3CAYAAADen/6oAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeVxUZf/4/9cMqwSIuJDKnaCCuILm\njVpat6nVzyU1NwwHVLRcWExEMVs0Q1wwI8xcU8R9gbs0b+/MLLWfoZWGJhAqLuCKCqjINsz3Dz/O\n7QgiMDMJzvvZYx5xzrnO+1znzDjvuc5yXQqNRqNBCCGEeMopn3QFhBBCiL+DJDwhhBAmQRKeEEII\nkyAJTwghhEmQhCeEEMIkSMITQghhEiThCSGEqJaKiorw8fGhY8eO7N69u9Tyffv2MWzYMHx8fEhK\nSnpsPIU8hyeEEKI60mg0XLt2jc2bN+Pm5sbrr7+uXaZWqxk0aBDr1q3jzp07TJo0iY0bN5Ybz9zY\nFRZCCCEelJubS25ubqn59vb22Nvba6cVCgUNGjQoM8bZs2dxcXHB1tYWW1tbiouLKSgowMrK6pHb\nlYQnDCZfXfKkqyCEqAJrM/2vbil6OVe47OdvTGPx4sWl5gcGBhIUFFShGDk5OTrJ0d7enuzsbJyc\nnB65jiQ8IYQQfyt/f38GDhxYav6DCexxateuza1bt7TTt27dwsHBodx1JOEJIYTQn0JR4aIPn7qs\niiZNmnD27Fny8vK4c+cOZmZm5Z7OBEl4QgghDMGs4gmvMkJCQjhx4gQ2NjYkJSXh4OBAz549adq0\nKYGBgYwaNQqFQsH06dMfG0vu0hQGI9fwhKiZDHINr/dzFS6r2XVe7+1VhbTwhBBC6K8SpzSfFEl4\nQggh9FcDujGRhCeEEEJ/0sITQghhEqp/vpOEJ4QQwgCMdJemIUnCE0IIoT85pSmEfmq9XvFbnStj\n8JRQo8S9knPOKHEjXn7X4DGdbeoZPCZAYUmRUeIezko3Slwvx38YPGaxRm3wmAAta9c3SlyDqP75\nThKeEEIIA1BW/4wnCU8IIYT+qn++k4QnhBDCAAzQW4uxScITQgihP2nhCSGEMAlyl6YQQgiTUP3z\nnSQ8IYQQBlAD7tKs/lcZa4jly5dz5swZnXkxMTHs3r27yjETExP5+OOP9a1apa1duxYfHx8GDhzI\nsWPH/vbtCyFqIEUlXk+ItPAM5O23337SVTAItVpNt27d8PPzIykpidjYWLy8vJ50tYQQ1Z10LVb9\nzZ07l2PHjmFhYUF4eDhWVlbMmjWLkpIS7O3tiY6OxtLSkl69euHt7U1ycjJ9+vQhICBAJ054eDi+\nvr60bdtWZ/5///tftm3bRl5eHosWLcLJyYn169fz9ddfo9FoGDduHD169OD06dN89NFHlJSU0LRp\nU2bPnq2NoVarmTVrFm3btmXIkCHa+YcOHWLJkiUUFRXRtGlT5syZo7PtwsJC3n//fTIzM1EqlURF\nRXHmzJlS62RkZDBlyhQaN26Mo6MjM2bMAODIkSO0aNHC0IdcCPE0kptWqrd9+/aRk5PDpk2bgHuJ\npaioiNjYWJRKJQsXLuT777+nd+/eZGVlERYWho2NDf369SuV8B6lbt26LFq0iD179rBixQrGjx9P\nQkICmzZtIj8/n6FDh9K9e3eioqIIDw+nTZs2zJw5k71792JnZ0dhYSFhYWG89tprvPbaazqxPT09\niYuLA2Dy5Mn88ccfeHp6apdv2bKF5557jvnz5wNQUlKCnZ1dqXXq1q1LZmYmq1evplatWgD8+uuv\nHDx4kOXLl+t3kIUQpqH65zvTvoZ36tQpOnXqpJ02MzPj4sWLjBs3jhEjRrBnzx4uX74MQKNGjXBw\ncMDS0hILC4sKb6Ndu3ba/589e5aMjAw8PDwwNzfH1taWBg0acPPmTTIyMmjTpg0A7du3Jz39Xr+B\n+/fvR61Wl0p2AKmpqYwaNYoRI0Zw7NgxbV0f3L/OnTtrp5VK5SPXadGihTbZAXz77bdMmjSpUvsq\nhDBhCkXFX0+ISSc8Nzc3jhw5op0uKSlhw4YNDB06lHXr1tGzZ080Gg0Aiiq+SSdOnADg+PHjNGnS\nBGdnZ1JSUiguLub27dtcuXKFOnXq0LhxY23Zo0eP4urqCkDPnj1p0aIFCxYsKBV72bJlTJ06lXXr\n1uHp6amt64P7d/jwYZ39e9Q6ZmZmOut26dIFZ2fnKu2zEMIEKSvxekJMOuH961//wtbWFh8fH/z8\n/Dh58iTdu3fn008/Zfz48Vy9elXvbWRnZxMQEMCqVasYM2YMdevWZcCAAQwfPpyRI0cyefJklEol\nU6ZMYc6cObz11lsUFhbyyiuvaGNMmDCBZ555hqioKJ3Yr7/+OlOmTCEoKIiiotI91A8ZMoT09HR8\nfX3x8/Pj2rVrj13nvrNnz5KTk6P3/gshTIRSUfHXE6LQPNwsEKKK8tUlBo8pwwPdI8MDyfBAYLzh\ngawN0A+mIrBNhctqFp/Qe3tVYdI3rQghhDAQuUtTCCGESaj++U4SnhBCCP1V9ca+v5MkPCGEEHqT\nhCeEEMIkmNWAzqMl4Ylq7e7u86i+jzZ43G1RCw0eE2BaxBdGietoZWvwmHnqfOpY2hk8rhrD360L\n0NrhWaPEVRvhjkoFYKYwe2y5p4m08ITQkzGSnbjHGMlO3GNqyQ4k4QkhhDARkvCEEEKYhBqQ7yTh\nCSGE0J+08IQQQpgEpaL6d80sCU8IIYTepIUnhBDCJNSAfCcJTwghhP6UNSDjGeSk67Vr11i40DgP\n8t4XExPD7t279YoRHx+vMyDqfX379q3Q+hqNhgkTJqBSqThz5oxedfm7JCYm8vHHHz/pagghnnIK\nhaLCr8rYsmULPj4+qFQqLly4oLMsISGBQYMGMWTIEGJjYx8byyAtvPr16xMaapzxxQxFrVbz5ptv\n6hXj2rVrFBQUEBcXVyr2wyOGCyGEKVEaoWux7Oxstm7dysaNGzl58iRRUVFER/+vM4ply5YRHx+P\ntbU1ffv2Zfjw4VhaWj4ynkESXkZGBrNnz2bZsmU683v16oW3tzfJycn06dOHgIAAcnJy+OCDD8jO\nzsbKyop58+bh6OjImjVr+O6771Cr1YwbN47u3btz5MgRIiIiePbZZ9FoNLi5uenEP3XqFLNmzaKk\npAR7e3uio6N1djYxMZFVq1ZhZWVF69atKSoqws3Njddff5158+Zx9OhRXF1dtSN/P6pu982cOZMT\nJ06gUqkIDAzUid2rVy8++ugjSkpKaNq0KbNnz+bw4cMsXboUW1tbTp8+TVBQEF9//TUZGRnMmzeP\n1q1b6xzDd999l3/84x+kpqby9ttvs3fvXs6cOUNYWBienp4EBQVpk+2MGTN48803ef7557Ux+vTp\nQ6dOnUhOTqZt27a89957Osdr/vz5HD9+nDt37jBx4kR69OhBdnY27733Hrdu3UKpVLJy5UquXr3K\nzJkzKSgowMHBgQULFmBlZaXnp0QI8TSrTMstNzeX3NzcUvPt7e2xt7fXTiclJeHt7Y25uTnt2rUj\nPV13EOCmTZuSl5cHgLW19WMbHka9hpeVlUVYWBg2Njb069ePgIAAli9fTv/+/enRowc//PADq1ev\npn///vz+++9s2LCBgoIChg8fTvfu3Zk/fz7Lli2jQYMG+Pv7l4rv7OxMbGwsSqWShQsX8v3339O7\nd+9Sddi6dStmZmbExMQAcPLkSdLT09m0aRPnz5/XtvzKqtuDLdf33ntPm9gTExN1Yo8fP57w8HDa\ntGnDzJkz2bt3L3Z2duTn5/PVV1+xd+9evvjiC7Zv305iYiKbN28udarx+vXrrF+/njNnzjBmzBi+\n//57rl69yqxZs1i1ahX16tXj9OnTNG7cmNTUVJ1kB3D37l0GDx5Mq1atmDBhAn/++afO8sDAQGxs\nbMjJyWHUqFH06NGDZcuW8eqrrzJgwABKSkpQKpUsWLCA0NBQPDw8iIuLIyEhAR8fn6p/EIQQT73K\nJLzY2FgWL15can5gYCBBQUHa6ZycHGrXrq2d1mg0OuVfe+01BgwYgJmZGQEBAU824TVq1AgHBwcA\nLCwsAEhLS+O3335jzZo1FBcX4+bmRlpaGidPnkSlUgGQl5fH7du3yc/Px8nJCYB27dqVin/x4kXm\nzp1LXl4eWVlZ1KlTp1SZtm3bljoIZ8+exdPTE4DnnntOu15ZdSvPg7EzMjJo0+beEPft27cnPT2d\ndu3a4eHhgUKhwMnJCXd3d5RKJQ0aNCA7O7tUvObNm2NpaUmDBg1wcXHB2toaJycnbdnBgweTkJCA\nu7s7r776aqn1zc3NadWqlbZu586do27dutrlcXFx/Pjjj5ibm3Px4kXgXivZz88PAKXy3iXd06dP\nExERAUBBQQEvv/xyucdBCCEqk/D8/f0ZOHBgqfkPtu7uT6empmqn739HAdy+fZtly5axe/duLC0t\nGT16ND179qRRo0aP3K5RE15ZB6BZs2a88MILdOvWDYDCwkLS09Px9PTU3vhSWFiIpaUlVlZWXL16\nlfr163PixAltQrlvw4YNDB06lJ49exIVFVUq+wNlZvwmTZqwY8cOAC5cuMDNmzcfWbfyPBi7cePG\n2joePXqUrl27ljoGj/tAPKrs/f3q0qULMTExJCcnExkZWWr94uJiUlJS8PDw4MSJE3Tr1o07d+4A\n986F7969m+3bt5Obm6ttCTdv3pzDhw/Tv39/bQvP1dWVSZMm0bRp0wodByGEqEzCe/jU5aN4enqy\nZMkS1Go1KSkpNGnSRLtMqVRiYWGBjY0NSqUSa2trbt++XW68v/2xhHHjxvHhhx+ycuVKSkpKGDp0\nKP369aNdu3b4+vqiVCpp1KgR8+bNIywsjLfffpsGDRpga1t6eJTu3bsTERHB9u3bsbOzK7OFV5bW\nrVvTpEkTfHx8cHd3p169euXWrSKmTJnChx9+CICLiwuvvPIKR44cqeBRqRilUknXrl05fvw4DRo0\nKLW8Vq1axMfHk5SURJs2bWjTpg2JiYkA1K5dm0aNGuHr60urVq2ws7vXU/4777zD9OnTtadmV65c\nybRp05g9e7b23Pj48ePp0qWLQfdFCPF0McZTCQ4ODgwYMABfX1/Mzc2JiIggPj4eZ2dnvL29GThw\nIMOGDUOhUODl5YW7u3v5ddSU1SwS1dYXX3yBu7s7vXr1KrWsb9++7Ny58wnU6p58teHHQTPW8EA1\nbTw8lXtXg8c01vBAeeoCo8TNLSz/13tVGaNLLGMND+Rm7/j4QlVgbab/MWgc8a8Kl82c8aPe26uK\n6t/5mdCaM2cOx44d45VXXnnSVRFCCB1KhaLCrydFelqpQR5+zOBhT7J1J4QwbTWgoxVJeEIIIfQn\nnUcLIYQwCQok4QkhhDAB0sITQk9Xcs4ZJa6x7qacN2OiUeKO3v7n4wtVkhG6PgTA3sLaKHFLNIa/\nCxjA2uzRfS9WVV5xvsFjVnfG6EvT0CThCSGE0Ju08IQQQpgEhRGeZzQ0SXhCCCH0Ji08IYQQJkGh\nlBaeEEIIE6BUVv9BsCXhCSGE0JtcwxNCCGESasI1vOqfkp8CiYmJpUY3r6y+fftWed3c3Fzt+H+V\nce3aNe0YhUIIUR6FQlnh15MiCc8E5ObmVqlj6fr16xMaGmqEGgkhnjYKhaLCrydFEt7f5MyZM4wf\nP54BAwbwyy+/APDbb78xbNgwfHx8+PzzzwHIy8sjKCiIESNGEBAQQFZWlk6cbdu28cEHH6BWq7Xz\nbty4gb+/PyqVCj8/P27cuKGzztq1azl27BgqlYrDhw+zevVq/Pz8GDhwIJs3bwbg3//+N5988gkA\nH3zwAXv27CEjI4N33nnHaMdECPH0kBae0MrNzWXx4sUsX76cqKgoACIjI/n888/ZtGkTJ06cIDk5\nmS1bttC+fXvWrVvHoEGDWL58uTbGihUrOH36NLNnz8bM7H93RNnZ2bFy5Uri4uLo3bs327Zt09m2\nn58fXl5exMXF4e3tzbBhw1i7di2bN28mNjaWkpISBgwYwM2bN/nkk08oKSkpc4BZIYR4FKXSrMKv\nJ0VuWvmbtGrVCjMzMxo0aEBBwb0RoQsKCnBycgLAy8uL9PR00tPTeeONNwDo0KEDCQkJwL3raQkJ\nCaWSGUBOTg6zZs3ixo0b3L59m44dO5Zbl127dpGQkIBCoeDq1avk5ORQp04dRo4cyeDBg/n+++8N\nuetCCFNQA/rSlBbe3+TkyZOUlJSQlZWFpeW9zmqtrKy4cuUKAMeOHcPV1RUXFxf++OMPAH7//Xdc\nXFyAe9fTZsyYQUhICHfv3tWJvWPHDp5//nnWr1+Pj48PGo1GZ7mFhQXFxcXa6WXLlrF69Wq++uor\n7O3t0Wg0FBcXM3/+fD755BPmzp1rrMMghHhK1YRTmtLC+5vUrVuXCRMmcPnyZaZNmwZAeHg4QUFB\nKBQKOnfuTMuWLXnuuecICwtj7969WFlZMW/ePG2MF198EYCQkBCio6OpVasWAF26dCEsLIxDhw7h\n5OSEubnu21q/fn0AgoODGT16NF27dmX48OG4u7tjb28PQExMDK+//jpDhgzh4sWLbNiwgZdeesno\nx0UI8XSoCY8lKDQPNweEqKJ8teGHb3l1+2SDxwR4waW7UeIaa3igVCMMD2RvUcvgMQGURvriyy7M\nM0rcmjQ8kIttbaPEtTbTv9X1/Ff+FS772+hYvbdXFdLCE0IIoTfpaUUIIYRJkL40hRBCmISacA1P\nEp4QQgi9ySlNIYQQJkFaeEIIIUyCDAArhJ4iXn7XKHEdrWyNEne0ER4fAGgxqLXBY/YOedvgMQGS\nkvYZJe7PIbuMEldphP436lga5/NVnclNK0IIIUyCnNIUQghhEuSmFSGEECZBWnhCCCFMgrTwhBBC\nmAZJeEIIIUyB3KUphBDCJBhrlAxDqv5tUCPJyMjgnXfe+Vu2FRERwa1bt0hMTOTjjz8GQKVScePG\nDZ1y8fHxHD58mNzcXHbs2GGQbScnJxMbW7mhOHbt2oWPjw/9+/eX0c+FEBWiqMR/lbFlyxZ8fHxQ\nqVRcuHBBZ9n169cJDg7Gz8+P0NDQx8aSFt7fYMaMGRUq9+abbwL3kvHOnTvp169fpbdVUlKC8v96\nPFCr1bRs2ZKWLVtWKkarVq3YuHEjN27cYOLEifTs2bPS9RBCmBZj3LSSnZ3N1q1b2bhxIydPniQq\nKoro6Gjt8rlz5zJlyhSee+65CsV7qlt4MTEx7N+/n7t379KmTRv++usvMjIyCA4OBuDmzZuEhobS\nr18/vv32WwAyMzMZO3Ysfn5+BAcHU1BQQGJiIgEBAQQHB9O3b18OHz6ss53ExESioqIAeOutt1i3\nbh0AgwcPRqPRlNmae1R9d+/ezdq1azl27BgqlarUtqZOnYpKpWLQoEEcO3YMuDdy+syZMxk7diyp\nqan06tWL999/n6lTp2pblSkpKTq/gAICAsjMzCQlJQV/f39UKhUffvghGo0GFxcXFAoFR44cwd3d\nvYpHXwhhShQKRYVfubm5ZGRklHrl5ubqxExKSsLb2xtzc3PatWtHenq6dplarebMmTPExMQwYsQI\ndu16fE88T3ULz9vbmwMHDmBhYUHXrl05fPgwzzzzDJ06dQIgKyuLdevWcfv2bcaMGUOfPn1YsGAB\noaGheHh4EBcXR0JCAq6uruTl5bFy5UpOnjzJ0qVL8fb21m7Hy8uLxYsXk5+fj729Pb///jvdu3en\ncePGVXo2xc/Pj3PnzrFs2bJSy2bOnImNjQ1nzpxhzpw5rFy5EgAXFxdmzpwJwJUrVwgODqZBgwYk\nJiYC4OHhQWZmJnl5eeTl5VFUVETjxo1RqVR8+umn1K9fn7lz53Lw4EG6detGeno6X331FatWrap0\n/YUQpkepqPhNK7GxsSxevLjU/MDAQIKCgrTTOTk51K79v1HeNRqN9u/r16+TmprKggULePbZZ3nr\nrbd44YUXcHBweOR2n+qE5+XlRUxMDObm5owfP57Y2FisrKwYM2YMAM2bN8fS0hJHR0cKCwsBOH36\nNBEREQAUFBTw8ssv4+rqioeHBwqFgoYNG5Kdna2zHSsrK8zNzTl48CBdu3bl559/JjExUZtYDUWt\nVvPZZ59x/PhxzM3NderRvn177d+NGzemQYMGpdbv0aMHe/bsIScnhz59+gCQlpbG5MmTAbhz5w5u\nbm4A7N27l5EjR2JnZ2fQfRBCPJ2UlTil6e/vz8CBA0vNt7e3LzWdmpr6v2080EF17dq1adSoEU2b\nNgWgdevWnD9/3nQTnpWVFRYWFiQlJRESEkJ+fj6ZmZk0a9aMjIyMMltfrq6uTJo0SXsQCwsLOXr0\n6GNbah06dODLL78kMjKSa9eusX79eubPn1+leltYWFBcXFxqfkpKCpmZmWzcuJHTp08TEhKiXfbg\nB8HMrOxfWv369ePDDz8kLy+PL774AgB3d3c+++wzHB0dASgqKgLufXicnJyqVH8hhOmpzM0o9vb2\npZJbWTw9PVmyZAlqtZqUlBSaNGmiXWZlZYWTkxNZWVnUqVOHtLQ0GjVqVG68pzrhwb1ElJaWhkKh\nwM3NDSsrq3LLT5s2jdmzZ5OXlwfA+PHjdZLJo3h7e7Nhwwbc3Nzo1KkT27dvp1mzZlWqc/369QEI\nDg5m9OjReHl5AfeScU5ODn5+fnTo0KHScZ999lmKi4upU6eO9jTBe++9R2hoKMXFxSiVSt5//33c\n3Ny4dOkSZmZm2sQvhBDlMcZNKw4ODgwYMABfX1/Mzc2JiIggPj4eZ2dnvL29mTp1KiEhIRQVFdGv\nXz/q1atXfh01D54UFUIP+eoSg8c8knXh8YWqwFjDA1kojfMbUoYHqlnDA1kYaWy42pbGebjb2kz/\n+r6xa06Fy37T+z29t1cVT30LTwghhPFV9vm6J0ESnhBCCL2ZVeIuzSdFEp4QQgi9yWgJQgghTIKM\nhyeEEMIkKGpAx12S8IQQQuhNWnhC6MnZpvznaqrKyszCKHGVRvo3b4xHCHZFLzd4TID+k4MeX6gK\nrM2M83VV/dslNYNcwxNCCGESzCThCSGEMAVyDU8IIYRJkGt4QgghTIJcwxNCCGESpGsxIYQQJsFM\nKV2LCSGEMAE14aaV6l/DJywjI4N33nmnwmV/+umnMpdNnz690uv37du3YpV8KEanTp1QqVSoVCre\nffddtm7dikqlon///nTr1g2VSsXUqVO167z55ptERUVpp3ft2oWPjw/9+/fn+++/r3QdhBCmR6FQ\nVPj1pEgLz4AyMzP56aefePnll0sti4yM1Gv9yvDy8mLZsmU684YMGUJiYiL//e9/+fDDD7Xz09PT\nee6550hMTESj0aBQKGjVqhUbN27kxo0bTJw4kZ49e+pVHyHE068mXMOTFh4QExPD/v37uXv3Lm3a\ntOGvv/4iIyOD4OBgAG7evEloaCj9+vXj22+/BWDHjh2oVCoGDx5MdHQ0AGvXrmXPnj2oVCrOnDmj\ns437rbXNmzczaNAg/Pz8WL9+vU6Zh9fXaDRERkbi4+NDREQEAIWFhUyfPh0/Pz/8/Pw4d+6cXvv+\n7bffMnDgQNq3b8/Ro0cBcHFxQaFQcOTIEdzd3fWKL4QwDQqFssKvJ0VaeIC3tzcHDhzAwsKCrl27\ncvjwYZ555hk6deoEQFZWFuvWreP27duMGTOGPn360KNHD/r164dGo2H48OG89dZb+Pn5lWpBPWzH\njh2sWLECR0dHSkp0Rwh/eP27d+/i4+PD9OnTeeONN7h9+zbffPMNLVu2JDIyktTUVKKjo/n00091\n4hw7dgyVSgVA69atCQ8Pf2R9Dh06xLhx46hfvz7bt2+nQ4cOwL2W31dffcWqVasqf0CFECZHnsOr\nIby8vIiJicHc3Jzx48cTGxuLlZUVY8aMAaB58+ZYWlri6OhIYWEhAL/88gurV69Go9Fw4cIFrly5\nUqFtTZs2jaioKAoLC/H19aV9+/aPLFurVi1cXV0BaNiwITk5OaSlpfHHH3+wZ88ebZmy9ufhU5pl\nOXnyJBcvXtReozx//jxqtRozMzP27t3LyJEjsbOzq9B+CSFMmwwAW0NYWVlhYWFBUlISISEh5Ofn\nk5mZSbNmzcjIyCjzl0t0dDSxsbHUrl2b4cOHo9FosLCwoLi4uNxtNW/enDlz5nD58mUmTZrEpk2b\ntMseXv/h7Wo0Gpo3b06LFi3w8fEB0Cbgqti5cydz5syhS5cuACxatIhffvmFF198kdatW+Pk5FTl\n2EII06KsAS08uYb3fzp06ICtrS0KhQI3NzcaNWpUbvk+ffrg5+dHaGgo1tbWALi7u3Pq1CmCg4O5\ncOFCmevNnDmTESNGEBwcrE1a91Vk/SFDhnD8+HH8/PxQqVRs2LChVJn7pzRVKpW2lfowjUbDDz/8\nwPPPP6+d9+KLL7Jz504ALl26RFZWVrnHQAgh7lOiqPDrSVFoNBrNE9u6eKrkq0seX6iSLuXdNXhM\nqHnDAwXsnWvwmDVteKDl3ScbJW5N+tVva2Gc04bWZvofheD/f0uFy37+wlC9t1cVckpTCCGE3mrC\nKU1JeEIIIfRWE3pakYQnhBBCb+ZKSXhCCCFMgDyHJ4QQwiQ8ybsvK0oSnqjWGtrU4tztXIPHVWP4\nO0oB7C2sjRI3KWmfwWM6d3fj+favGzzu15/GGDwmQHQ349z9aW2EO3aNdbdudSYtPCH0ZIxkJ+4x\nRrITpkspI54LIYQwBWaS8IQQQpgCeQ5PCCGESagJ4+FJwhNCCKE3aeEJIYQwCZLwhBBCmATpWkwI\nIYRJMKsBXYtV/xpWUUZGhnYkb2OLiIjg1q1bJCYm8vHHHwOgUqm4ceOGTrn4+HgOHz5Mbm4uO3bs\nKDNW3759K739+Ph4Vq1aVfmKP8LatWvx8fFh4MCBHDt2zGBxhRBPL6VCUeFXZWzZsgUfHx9UKlWZ\n44TeunWLTp06sXv37sfXsVJbFmWaMWMGdnZ2jy335ptv4u3tTW5urnag1SdJrVaXOb9bt25s2rSJ\nWbNmERcX9zfXSghRExljANjs7Gy2bt3KunXrCAsLIyoqqlSZVatW4enpWcE61mAxMTHs37+fu3fv\n0qZNG/766y8yMjIIDg4G4FoaLrIAACAASURBVObNm4SGhtKvXz++/fZbADIzMxk7dix+fn4EBwdT\nUFBAYmIiAQEBBAcH07dvXw4fPqyzncTERO2Bfuutt1i3bh0AgwcPRqPRlNmae1R9d+/ezdq1a7Wj\nkj+8rYKCAsLCwhg8eLC21ZaTk0NwcDB+fn6MHTu2zG0dO3aMCRMm8MYbb3Dq1CkADh06hK+vL2+9\n9RaLFy/W1iE8PJxx48Zx8ODBMsu4uroCcOTIEVq0aFGBd0IIYeoUCkWFX7m5uWRkZJR65ebq9qyU\nlJSEt7c35ubmtGvXjvT0dJ3lWVlZXLhwgbZt21aojjX6Gp63tzcHDhzAwsKCrl27cvjwYZ555hk6\ndeoE3DsY69at4/bt24wZM4Y+ffqwYMECQkND8fDwIC4ujoSEBFxdXcnLy2PlypWcPHmSpUuX4u3t\nrd2Ol5cXixcvJj8/H3t7e37//Xe6d+9O48aNq9R/nJ+fH+fOnWPZsmWlll2+fJmpU6fi6OiIj48P\nAwYM4KuvvqJ///706NGDH374gdWrVxMaGqqznqWlJQsXLuS7775j27ZtTJs2jaioKOLi4rCxsSE4\nOJgzZ84AYGdnx9y5c9FoNAwePLhUmaZNm/Lrr79y8OBBli83zqjYQoinS2W6FouNjdX+wH5QYGAg\nQUH/6zM1JyeH2rVra6c1Go1O+aVLlzJ27Fj27NlToe3W6ITn5eVFTEwM5ubmjB8/ntjYWKysrBgz\nZgwAzZs3x9LSEkdHRwoLCwE4ffo0ERERwL3W1Msvv4yrqyseHh4oFAoaNmxIdna2znasrKwwNzfn\n4MGDdO3alZ9//pnExERtYjWkhg0bUr9+fQBatGhBRkYGaWlp/Pbbb6xZs4bi4mLc3NxKrdeyZUvt\n+jk5Ody8eVPnOmZubi6XLl0CoH379gCPLNO0aVO+/fZbJk2ahIWF4TvWFUI8fSrz4Lm/vz8DBw4s\nNd/e3r7UdGpqqnZa+cCNMRcuXCA3NxcPDw/TSHhWVlZYWFiQlJRESEgI+fn5ZGZm0qxZMzIyMsps\nfbm6ujJp0iSaNm0KQGFhIUePHn1sS61Dhw58+eWXREZGcu3aNdavX8/8+fOrVG8LCwuKi4vLXHb5\n8mWysrJwdHTkr7/+onHjxjRr1owXXniBbt26aev8sAfrr9FoqFOnDi4uLqxYsQJra2tKSkrQaDT8\n/vvv2g/No8oAdOnSBWdn5yrtnxDC9FRmAFh7e/tSya0snp6eLFmyBLVaTUpKCk2aNNEuS05O5vz5\n8wQEBHD+/HmeeeYZmjVrVmaDQFvHCtewmurQoQNpaWkoFArc3NywsrIqt/y0adOYPXs2eXl5AIwf\nP17nV8OjeHt7s2HDBtzc3OjUqRPbt2+nWbNmVarz/RZccHAwo0ePxsvLS7vs2WefZeHChaSlpfHa\na69Rr149xo0bx4cffsjKlSspKSlh6NCh9OvXr9xtKBQKJk2axNixYwEwNzcvdcH3UWXq1q3L2bNn\nad68OXXr1q3SPgohTIsxuhZzcHBgwIAB+Pr6Ym5uTkREBPHx8Tg7O/Pqq6/y6quvAvfuTXBzcys3\n2QEoNA+fFBWiivLVhh9jzljDA5kpzYwS11jj4T3/6auGj2mk4YGMNR7e2YS/jBK3Jo2HZ2dhnM+t\ntZn+9y8uTfmlwmXHeXTWe3tVUeNbeEIIIZ48hQwPJIQQwhRU5vm6J0USnhBCCL2ZGes8rgFJwhNC\nCKE3aeEJIYQwCVXphOPvJndpCoMxxl2aW8/+YfCYAK0dnjVKXFsLG6PEtTazNEJM4/zevVtcZJS4\nLgPdjRJ38JTQxxeqpEs3zxg8JsD3Q6KNEtcQd2nGnT5a4bKqZu313l5VSAtPCCGE3uSUphBCCJNQ\nE05pSsITQgihNzN5Dk8IIYQpqOzArk+CJDwhhBB6k2t4QgghTEINaOBJwhNCCKG/mnBKs/pfZSzD\n8uXLtaN335eYmMipU6fKXS88PJzjx4+TmJjIxx9//MhyFYn1d4iPj2fVqlVPuhpCCPFYZgplhV9P\nSo1MeG+//bZ2ANf7Dh8+bLAkZchYD1Kr1QaPWRO2LYR4+ilRVPj1pPytpzTnzp3LsWPHsLCwIDw8\nHCsrK2bNmkVJSQn29vZER0djaWlJr1698Pb2Jjk5mT59+hAQEKATJzw8HF9fX9q2bQvArVu3SEhI\noFatWmzbto2VK1cyevRoioqKKCoqYu7cubi4uJSqj1qtZtasWbRt25YhQ4Y8MtaiRYtITEwE7g0g\n2769bi8BU6dO5dKlS+Tl5fHBBx/oDOgK0KdPH1566SWSk5NZs2YN8+bN48SJExQXFzN9+nTatWvH\n/PnzOX78OHfu3GHixIn06NGjzGOYmJjIl19+Sa1atbh06RLh4eF07tyZlJQUIiMjKSkpwdXVlVmz\nZpGQkMD+/fspKCjglVde4dKlSxw8eBBra2t8fX157bXXytw3lUpFq1atSE5Opnbt2sTEGGd8MyHE\n00Oew3vAvn37yMnJYdOmTcC9ZFNUVERsbCxKpZKFCxfy/fff07t3b7KysggLC8PGxoZ+/fqVSngP\ns7OzY+DAgbi5ufH66/cGtVy8eDE2NjYcOHCA1atXM2vWLJ11CgsLCQsL47XXXuO11157ZKyTJ0+S\nmprKpk2buHTpEsHBwWzdulUn1syZM7GxseHMmTPMmTOHlStX6iy/e/cub7zxBtOmTeOnn36ipKSE\nuLg4bty4QUhICHFxcQQGBmJjY0NOTg6jRo16ZMIDyM3NZdWqVVy/fp0JEyawbds2IiIi+PTTT6lf\nvz5z587l4MGD2v388ssvARgwYABbtmzB0tKSkpKScvetW7duTJ8+nXHjxpGamkqLFi3KfQ+EEKat\nJlzD+9sS3qlTp+jUqZN22szMjHPnzjF37lzy8vLIysqiTp06ADRq1AgHBwcALCwqPxrx3bt3+fjj\njzl//jzFxcU4OjqWKrN//37at2+vk+zKcvbsWTw9PQFo2LAh+fn5OsvVajWfffYZx48fx9zcnOzs\n7FIxrKysaNmyJQBpaWkcPHgQlUoFwJ07dwCIi4vjxx9/xNzcnIsXL5Zbp1atWmFmZkaDBg0oKCjQ\nxp08ebI2ppubGwqFQqe1GRYWxgcffIBCoWDs2LHl7lurVq2Ae+9FTk5OufURQgiFPJbwP25ubuzZ\ns4cBAwYAUFJSwoYNGxg6dCg9e/YkKiqK+/1YV6VpbGFhob1OdeDAAezs7NiwYQP79+9n/fr1pcr3\n7NmTevXqsWDBAsLCwh4Zq0mTJnzzzTcAXLp0CWtra52yKSkpZGZmsnHjRk6fPk1ISEipbZmZmWn/\nbt68Oa+88gqhofc6rC0sLCQ7O5vdu3ezfft2cnNz6d27d7n7evLkSUpKSrhx4waWlvc6FXZ3d+ez\nzz7TJveioiJ27Nihs+0OHTrw4osv8uuvv7JkyRJGjx5d7r7dJ/2LCyEeR1p4D/jXv/7FoUOH8PHx\nwdLSkqlTp9K9e3ciIiLYvn07dnZ22hZeVXTp0oW5c+fy448/MmXKFJYuXUpAQADNmzd/5DoTJkxg\nyZIlREVFMWXKlDJjLViwADc3N3x8fNBoNISHh+vEcHV1JScnBz8/Pzp06FCh4/Drr79qW3jt2rVj\nypQpNGrUCF9fX1q1aoWdnV25MerWrcuECRO4fPky06ZNA+C9994jNDSU4uJilEol77//fqn1AgMD\nKSwspLCwkKCgIFq3bl3uvgkhREXVhK7FZHigGiYxMZH//ve/fPjhh0+6KqXI8EAyPBDI8EBgmsMD\n/Xg5vcJl//Wsq97bqwp58FwIIYTe5BqeMLhOnTrp3PwjhBDVgVzDE0IIYRIk4QkhhDAJckpTCCGE\nSTBTVv+7NCXhCSGE0JuMhyeEnrwc/2GUuGqNcTrTNsbjAwBKI/Tzbqzf49Zmle8dqSKM8fgAwLao\nhQaP6a36/wwes7qTa3hCCCFMQvVPd5LwhBBCGET1T3mS8IQQQuhNTmkKIYQwCfJYghBCCJNQAxp4\nkvCEEEIYQvXPeJLwhBBC6K0mnNIs91Gca9eusXBh+c+oqFQqbty4UamNxsfHEx8fD1Dm4KyG9Kj4\n8fHx3L17t0IxMjIy+Omnn8otEx8fz6pVqwDo27evXrEMLTw8nOPHj1eobGWOixBC3KeoxKsytmzZ\ngo+PDyqVigsXLmjn5+bm4u/vz1tvvcXw4cP5888/Hxur3IRXv3597cjcxrJx48YnEj8hIaHCX+yZ\nmZkGS1KGjPWwkhL9x6OrzHERQoj7FApFhV8VlZ2dzdatW1m3bh1hYWFERUVpl1laWjJ//nw2bNjA\nJ598orPsUco9pZmRkcHs2bNZtmyZzvzY2Fi++eYb/vGPf5CbmwtAXl4e06ZN4+bNm1hZWTFv3jzq\n1atHr1698Pb2Jjk5mT59+hAQEECvXr20cTIyMlCpVAwfPpzevXtrt7tw4UIWLVpEWFgY9erVY9q0\naYwcOZLFixezdetW9u3bx61bt/Dx8WHYsGHEx8ezb98+1Go1GRkZfPrpp/z111/a+D179sTf3x+A\nw4cPk5yczIQJE/jnP//J+PHjy6z7fWvXriUpKYm0tDRmzZrFtm3bOH78OHfu3GHixIn06NGjzOO3\nbds2/vjjD2bOnImZmVmZsU6fPs3y5csBGDRoED4+PjoxVq9eXWpfHxQeHo61tTWZmZlMnjyZ33//\nna+//hqNRsO4ceO0dduwYQNXrlxBqVTy2WefYWtry6JFi0hMTARg2rRpFBUV6RwXY//YEUI8PSpz\nSjM3N1ebOx5kb2+Pvb29djopKQlvb2/Mzc1p164d6en/G2TW2toaa2tr4F7yu/8dW55KX8O7fv06\nO3bsYPPmzeTl5Wm/ULds2UL79u0ZPXo0u3btYvny5bz33ntkZWURFhaGjY0N/fr1IyAgADs7OwD8\n/f3ZunUrcXFxOttwdnbm4sWLANy5c4dbt25RWFhIUVERtra2DBs2jFGjRlFYWMiAAQMYMmSIdqcX\nLlzId999x7Zt2wgPD2fJkiWl4nt7e9OyZUuio6NxdHRkzZo1Zdb9Pj8/P51RxgMDA7GxsSEnJ4dR\no0aVmfBWrFjBjRs3mD17ts78B2Op1WoCAwPZunUrlpaW+Pj48Oqrr+Lo6KgtX9a+Kh/qpNXFxYWZ\nM2dy/fp1EhIS2LRpE/n5+QwdOpTu3bsD0KJFCyIjI1mzZg1bt26lU6dOpKamsmnTJi5dukRwcDBb\nt27VOS5CCFFRlWm5xcbGsnjx4lLzAwMDCQoK0k7n5ORQu3Zt7bRGoym1jkajITIykjFjxjx2u5VO\neBkZGXh4eGBubo69vT2urveGak9PT+eNN94AoEOHDiQkJADQqFEjHBwcALCwqHgfe66uruzbt48m\nTZpw/fp1Dhw4gJeXFwC7du0iISEBhULB1atXycnJAaBly5YANGzYUDuvIh5V90eJi4vjxx9/xNzc\nXJuYH3Tt2jUSEhLYtm1buXFu3ryJk5MTzzzzDAAeHh5kZGToJJuy9rVOnTo6cdq3bw/ovje2trY0\naNCAmzdvAtC2bVvt/7/55hucnJzw9PQE7h2v/Pz8cusqhBDlqcy1OX9/fwYOHFhq/oOtu/vTqamp\n2umHf+wDRERE4O3tTefOnR+73Ur3H+vs7ExqaipqtZpbt25x9uxZ4F4r448//gDg999/x8XFBXh8\n1n/Ucm9vbxYvXoy3tzcdO3ZkyZIl2pG+ly1bxurVq/nqq6+wt7fXZv0HY5U170EWFhao1epy6/5g\n2eLiYuDeOeXdu3ezfv16YmJiynwD6tevz4wZMwgJCSl1PezBWHXq1OHKlSvcuXNHezrR2dlZp/yj\n9vVB9+vg7OxMSkoKxcXF3L59mytXrmiT44kTJ7T/b9KkCU2aNNHu86VLl7SnBh48LkIIUVGKSvxn\nb2+Ps7NzqdfDCc/T05MjR46gVqv5888/adKkic7ypUuXYmZmxsiRIytUx0q38OrWrUvv3r0ZMmQI\nrq6uNG7cGIChQ4cSFhbG3r17tdfBKsLDw4OJEycyaNAgXnnlFe18b29v3n//fTp27MiNGzeYNWsW\nzz//PABdu3Zl+PDhuLu7lzpAD/vXv/7F22+/Ta9evbSnPgF69OhBWFgYL7zwAr6+vuXW3d3dnVOn\nThEcHMyUKVNo1KgRvr6+tGrVSnt69mEvvvgiACEhIURHR1OrVq1SscLCwggJCcHf3x+FQsHgwYNL\nnUqszL7WrVuXAQMGMHz4cDQaDZMnT9Ymw9OnTzNq1CiUSiXR0dHY2tri5uaGj48PGo2G8PDwUsel\nW7duHD58WHvtUwgh/k4ODg4MGDAAX19fzM3NiYiIID4+HmdnZ/7xj38QHR3N888/j0qlokGDBo99\nqkChKavJIEQV5Kv1v0v0YWm5lXvkpaKMNTyQo1X5P0qqyhjDA1mbGWeAILWRvlEC95e+5mMINWl4\noJ98lz2+UBUY4rOQVsZNKI/i9pgf78YiD54LIYTQW/V/7FwSnhBCCEOoAZ1pSsITQgiht+qf7iTh\nCSGEMICa0JemJDwhhBB6kwFghRBCmAhJeELopdhIjw+YKx7f715V5BUbp8eaOpa2RolrDEojfe9d\nunnGKHGN8QjB4bj/GDwmAL7GCWsI1T/dScITQghhAJXpS/NJkYQnhBBCb3LTihBCCJNQ/dOdJDwh\nhBAGIKc0hRBCmISacErTOL3HCiGEENWMtPCEEELorfq37/7GFl5GRgbvvPPOI5cnJyfz+++//13V\n0Vvfvn0fuSw3N5cdO3YYbFsxMTHs3r3bYPGEEMLQFIqKv56UanNKMzk5maNHjz7pahhEbm4uO3fu\nfNLVEEII8YAnckozJSWFyMhISkpKcHV1ZdasWaxdu5bbt2/z448/smTJEu1I4omJiSxduhRbW1tO\nnz5NUFAQX3/9NRkZGcybN4/WrVtz6NAhFi9ejEaj4YUXXiAwMFBne59//jkHDx7E2toaX19fWrdu\nzaRJk2jUqBEXLlwgICCAvn37kpmZycyZMykoKMDBwYEFCxZw584d3n33XUpKSlAoFHz22Wc6o5L/\n9NNPbN++nXnz5mlHNV+7di3Hjh1DpVIRFBSEmZkZ8+fPR6FQ8MILLxAcHKxTvx07drBlyxbu3r1L\nt27dCAkJKXXMvv/+e/7973+TlZXF0qVLqVevHn379tUm1vt/x8TEcPbsWW7dukVubi4+Pj58/fXX\n3L17l5UrV2Jra8vo0aMpKiqiqKiIuXPn4uLigkqlolWrViQnJ1O7dm1iYmIM+p4LIZ5uctPKI0RE\nRBAVFUVcXBw2NjYcPHgQPz8/hg8fTlxcnDbZ3Zefn8/nn3/O5MmTWb58OUuWLGHGjBls3rwZjUZD\nVFQUK1asYMOGDfz111+cOaPbBdEPP/zAunXrWLt2Lb169QLg8uXLzJs3jw0bNvDll1+iVqtZsGAB\noaGhrF27ln/+858kJCRgZ2fHypUriYuLo3fv3mzbtk0bd8eOHXz77bcsXLhQm+wA/Pz88PLyIi4u\nDm9vbyIjI/n888/ZtGkTJ06cIDk5Wad+PXr0IC4ujq1bt3Lo0CGuXbtW6pg1atSIpUuX0qdPH/7z\nn/K7LWrcuDHLly+nXbt2nDhxgtWrV9O5c2d+/PFHABYvXkxcXBwTJ05k9erV2vW6devG2rVrKSoq\nIjU1tdxtCCHEg2rCKc0n0sJLS0tj8uTJANy5cwc3N7dyn+Hw8PBAoVDg5OSEu7s7SqWSBg0akJ2d\nzc2bN3WuD+bm5nLp0iWaNm2qXT8sLIwPPvgAhULB2LFjsbKyws3NTZukGjRowM2bNzl9+jQREREA\nFBQU8PLLL5OTk8OsWbO4ceMGt2/fpmPHjgAUFRURHR3Nxo0bsbCwKHd/CwoKcHJyAsDLy4v09HRa\ntmypXf7LL7+wevVqNBoNFy5c4MqVK9SvX18nxv3yDRs25NSpU6W2odFoSpV1cnLCwcFB+3dOTg53\n797l448/5vz58xQXF+u0Vlu1agXcS645OTnl7pMQQtQ0TyThubu765waLCoqYvfu3ajVZXcU/GAy\nfDgx1qlTBxcXF1asWIG1tTUlJSU6X/4AHTp04MUXX+TXX39lyZIlvPvuu5w6dYr8/Hsd/V69epU6\nderg6urKpEmTtMmysLCQ9evX8/zzzzNy5Eg2btxIWloaABYWFkRHRzN58mSio6N1EoeFhQXFxcXa\naSsrK65cuYKTkxPHjh2je/fuOvWLjo4mNjaW2rVrM3z48FL1f3i/7y+/f1ryzp07XLx48bHHS6PR\ncODAAezs7NiwYQP79+9n/fr1ZR7zsuoghBCPUv1PaD6hhPfee+8RGhpKcXExSqWS999/n/bt27N+\n/XpOnDjBnDlzsLWtWO/wCoWCSZMmMXbsWADMzc2Jioqibt262jKBgYEUFhZSWFhIUFAQcK8VM23a\nNM6dO8c777yDmZkZ06ZNY/bs2eTl5QEwfvx4unTpQlhYGIcOHcLJyQlz8/8dMg8PD2bMmEFISIhO\n0rvfOgsODmb06NGEh4cTFBSEQqGgc+fOOq07gD59+uDn50fz5s2xtrau8HEcPHgww4YNo127dqVa\nhI/i6enJ0qVLCQgIoHnz5uWWvXbtGmvXriU0NLTCdRJCmKaakPAUGhP8KZ+RkcHs2bNZtmzZk67K\nUyVfXWLwmMk5pa9nGoKxhgeyUBonrjGGBzIz0jeUsb5QBv57slHiFhUafkgnYw0PdHf3eaPEtTbT\n/3aO6wVFFS5b16r8y0DGIg+eCyGEMIDq38YzyYTn7OwsrTshhDCg6p/uqtGD50IIIYQxmWQLTwgh\nhGHVhBaeJDwhhBB6qwHD4ZnmXZrCOIxxl6YQwvgMcZdmTmHx4wv9n9qWT6atJdfwhBBCmAQ5pSmE\nEEJv5XUPWV1IC08IIYRJkBaeEEIIvVX/9p208IQQQpgIaeEJIYTQm7IGNPGkhSeEEMIkSAtPCCGE\n3mpAA09aeEIIIQxBUYlXxW3ZsgUfHx9UKhUXLlzQWZaUlISPjw/Dhg1j3759j40lLTwhhBB6M8Zj\neNnZ2WzdupWNGzdy8uRJoqKiiI6O1i6PjIwkOjoaW1tbfH19eemllzAze/SYlJLwhBBC6K0y+S43\nN5fc3NxS8+3t7bG3t9dOJyUl4e3tjbm5Oe3atSM9PV27rKCgALVajZOTEwAuLi6cPXuWZs2aPXK7\nkvCEwRiiPz4hRM1UmX//K2JjWbx4can5gYGBBAUFaadzcnKoXbu2dvrBrp+zs7Oxs7PTTtvb25OT\nk1PudiXhCSGE+Fv5+/szcODAUvMfbN3dn05NTdVOK5X/S6q1a9fm1q1b2ulbt27pJMeySMITQgjx\nt3r41OWjeHp6smTJEtRqNSkpKTRp0kS7zNraGjMzM65evYqtrS3nzp3TWV4WGR5ICCFEtbVx40a+\n/vprzM3NiYiI4LfffsPZ2Rlvb2/++OMPIiMj0Wg0vP322/To0aPcWJLwhBBCmAS5y0AIIYRJkIQn\nhBDCJEjCE0IIYRIk4QkhhDAJkvCEEEKYBEl4QgghTIIkPFEjJSUlERoayooVK1Cr1QaPX1hYyObN\nm/nyyy8pLi4mMTFRr3iLFi0CYNy4caVe+vjhhx/w8fFh5syZ3L59W69YxowJxn/PANLS0vj11185\ncuQIR44cMco24uLiDB7z/udDGJc8hydqpDfffJOZM2fy66+/8vPPPxMSEoKzszO5ubm4uLjoHX/S\npEl07NiRnTt3smnTJkaOHMmaNWuqHC8rK4t69eqRmZlZalnjxo2rHHfgwIGsWrWKw4cPs2bNGvz9\n/WnZsiVZWVl07Nix2sQE479nwcHBFBUV0ahRIwAUCgXvv/++3nEflpKSgoeHh0Fj3v98COOSrsVE\njWRpaUm7du20r3//+9906dKFs2fPMnbsWL3jZ2dnM2LECL777jtAt9Paqrj/ZZacnEzPnj218zdt\n2oSPj0+V41pZWeHo6Mjrr7+Oh4cHe/fupU6dOly6dKnKyckYMcH471lOTg6xsbF6x3nYokWLePfd\nd7XTBw4cMEjCS0tLIycnR/vZkoRnfJLwRI3Up08fbd95HTt21OuLuCyWlpb89ddfAJw9e5ZatWoZ\nJO7+/fv56aefeOedd5g7dy4tWrTQK56/vz/nz5/nueeew8XFhYCAAL3raIyYYPz3rGXLlhw6dAgP\nDw8U/zc4m4ODQ5XjFRQUkJeXx5EjR7SJqbi4mEOHDumdoMtqjf7zn//UK6Z4PDmlKWqskpIS5s2b\nx/Tp0w0e++LFiyxYsIBTp07RrFkzpk6dqv1y0ldERATr1q3jk08+YdCgQQaJCfDzzz9z9epV3njj\njceOC1YZGo2GQ4cOsXPnTubMmWOQmGfPnuXSpUt07tyZmzdv4ujoqHdMlUqlM61QKFi7dm2V4yUk\nJBAfH09KSgotW7ZEo9FgaWlJjx49eOutt/Sqq7+/v1Fao6J8kvBEjTZlyhTmzJmDpaWlQeOq1Wqd\nkZMfnq6qTz75hOzsbEaOHMm8efPo06ePXqc07/voo4+wsbHhyJEjbNu2jVGjRrF69Wq9Yh47doyd\nO3dy4MABevfuzQsvvGCQVsgXX3xBamoqFy5cID4+njFjxrBq1Sq94xrLkSNHDN76mjt3Li+//LLB\nWqOiYuSUpqjRMjIy6NWrFy1atNCOlbV06VK9444ZM0abMDQajc60Ptq2bUv//v0BWL16NTExMXrH\nBEhPT2ft2rXaVo4+d0F++umnJCYm0r59ewYNGsTZs2cJCQkxSD0BDh06xLp161CpVCgUCoqKigwS\n948//iAiIoL8/Hxq1arF9OnT8fLy0juuhYUFQ4cONWjcP//8kz///FM7rW9rVFSMJDxRoy1cuNAo\ncYuLi7V/KxQKnWl93E92968JjRo1yiBxlUol169fR6FQkJ2djbl51f9p//zzzzz33HO8+OKLtGjR\nQtsCMaT8/HwUCgWFrldn6AAAIABJREFUhYV63xB039y5c/n888959tlnuXTpEu+++y6bNm3SO+68\nefMMHtcYjzaIx5OEJ2q0xo0bk5SUxJUrV7RfnPrc5n+fvb09X3/9NZ07dyYxMRE7Ozu9YwJs3bqV\nNWvWcO3aNWxtbXFwcCA+Pl7vuOHh4bz77rukpaUxadIkwsPDqxxr+/btnD17lp07d7Js2TIuXrzI\nL7/8wvPPP4+FhYXedQ0MDGTEiBFcuHABPz8/Jk2apHfM+5599lkAGjZsaLCYxoh7/vx5Vq9ezeXL\nl7WfW0OcmRDlk2t4okZ7//33uXv3LseOHcPT05Pi4mI+//xzvePm5OSwdOlSTp8+jZubG2+//Ta1\na9fWO+6bb77J5s2bGT16NKtWreKjjz4iMjJS77iXLl0y+Jf8fcnJyezcuZN9+/axa9cuvWKVlJRw\n8OBBXnrpJW7cuGGQm1XuW7RoEWlpaXh6enL06FHc3d2ZPHmyweO2aNFC5zGFqhg+fDgTJ04kJiaG\nwMBAfv75Z71+pIgK0ghRg/n4+Gg0Go1mxIgRGo1GowkKCnqS1XksPz8/jUaj0fj6+mqKioo0gwcP\nNkjcgIAAjb+/v2bLli2a3NxcvWJNmzbNIHV6lHHjxhkt9smTJzXffvut5uTJkwaPu2vXLs2ff/5p\nkHgqlUqj0dz7HGg0994/YXxySlPUaPdPsVlYWJCSksKZM2cMEnffvn3ExMRw+fJllEoldnZ2/Oc/\n/9E7bvfu3cnPz2fAgAH07t2bbt26GaC2sHLlSq5fv86uXbuYMGECjo6OREdHVynWxYsXDVKnR3Fw\ncCAyMhIvL6//196dB0V5ZX8D/9LQjOwiKgr6CygqGAbROIIT1ygxGjWCCLhgiIpLcAGURahAVEpF\niBu4oyKIKLiOaJxgxN0RNajRAgIGElxQZBEXmsXu9w/efkJrg4bnPrSt51OVGru76nhnJtWnn3vv\nOYe7+fr555/zjpueng4nJydoa2tj06ZNGD9+PPr378877osXL3D58mUUFhbi4cOH+Oijj6Cnp8cr\nplgshkQiQefOnbFixQqUl5fzXid5M0p4RK19++23kEgkmD9/PtatW8ekYwcAxMbGIj4+Hj4+PoiN\njWV2m9LW1hatWrWCq6srXF1dcfPmTSZxAUBPTw9GRkbQ1dXl1QOzsLAQERERSj9j0apLfsZ6584d\n7j0WCS8pKQlOTk6Ii4vDxIkTERUVxSTh+fr64tNPP8XYsWNx69YtLFiwAHFxcbxiysswwsPDce7c\nOWbF/aRplPCIWuvSpQtatWoFe3t7bNy4ESUlJUzi6uvrw9DQEFKpFEZGRsjOzmYSd/369QrXz7dv\n397sJ7GGFi1ahHv37mH48OFYunQpTE1Nmx2rTZs2TBJQY+bOnYuCggIUFxfDwcEBFRUVTOJWVVWh\nrq4OMpmM2QUboL7E4+uvvwYA9OnTB6dPn+Yd8/nz5zhz5gwqKiogk8mQnp6OyZMn845LmkYJj6i1\ngIAArmOFhoYGAgMDeTV5lrO1tYVEIsGgQYPg4uLCu8tKWloa0tLSkJuby01IYFXqAADe3t6825TJ\nGRgYoF+/fkxiKfNq4XlAQACTwvMBAwbAzc0NAQEBkEgkvJsRyOvkTE1NsWvXLtjZ2eHWrVvo3Lkz\n77XOmDEDjo6OvH6YkL+PEh5Ra68WWLNKIgEBAQCAWbNmwcPDg/cNzcGDB6N3795ISEjA1KlTAdSf\n47Rr145X3MTERHh6eiIlJeW1ernmbj/KawWFIkThuVQqhZmZmUKJB98kunv3bu7POTk5yMnJ4RWv\nIR0dHabF/OTtUMIjas3c3BwbN26Eg4MDMjMzeT+JLVu2rNFCaz7nVwYGBjAwMGDe91Pe8mrEiBHM\nYl66dAmurq4A6ouug4KCAAAzZ87E1q1bmfwdrAvPRSIRTp06xbQ36avlIizLKL766issX74cNjY2\n3L9v48aNYxKbNI4SHlFry5YtQ2pqKtLS0mBlZdXoZYu3xTJxtARra2tIpVKkpKQgOjqaScxHjx5x\nf27Y/qqqqopJfKEKz7W1tTF37lyF258sOtlkZGQgOjoapqamePjwIfz9/TFs2DBeMZOTk9GrVy+6\nndnCKOERtaatrc30sL/h2VV2djYKCwvx0UcfoWfPnsz+DtZEIhF0dXVRWloKExMT3vEae8Jl1WLM\n0dER+/fvZ154PmjQIGaxGtq8eTNSUlKgp6eHZ8+eYdq0abwTnpGRkSBTPkjTKOERosTKlStx9+5d\n2Nra4tixYzAzM0NISAjvuOXl5Thw4ACKi4u591hc9f/f//6H//znPzA3N+eaaB89erRZsRperGn4\nZ/l8QL7u3LmD/fv3Kww/ZdFtxtnZGefPn0dJSQnGjBmDP/74g3dMOV1dXYX/5EtfXx9r1qxBz549\nuR8SQt6MJfUo4RGixM2bN7Fnzx7u9cSJE5nE9fPzw8iRI3Hs2DG4u7szKXeQyWRYv349kyncAJj0\n9mzKwoUL4ePjw7wVWsMRSc7OzoiIiGAy4eKrr76Cu7s7rK2tkZOTw+RSj6WlJYD6qedylPCERwmP\nqL1Xm0ez+OKwsbFBTk4OrK2tkZubi169evGOCdTfJnR3d0daWho8PDy4pyc+NDQ0sGbNGmzZsoXB\nCtk0335TfCcnJ+ZxWY5IkpPJZOjTpw+++OIL3L17F506dWKyDTt37lzeMcjfRwmPqDVlzaNZJLzL\nly8jIyMDYrEYtbW10NXVxZgxYwA0f6sQqD9ve/nyJYyNjZGQkMCsjZeZmRkSEhIULmx8/PHHTGKz\nIn/a0tTUxKJFixS281hcLmE5Ikmu4Y8JlueN8n+XAODevXto3749Tpw4wSw+UY6mJRC1NnHiRCQn\nJ8PT0xOJiYmYP38+k2kJQikvL4ehoSEqKipw9OhR9O/fn0nBuLILECzOxVg6dOhQo585Ozvzjp+T\nk4Ply5cjLy8PPXr0QHBwMJNt3iVLlsDS0lKwHxMVFRVYt24dwsPDmcUkytETHlFrQjWPFoqxsTEA\nwMTEBF5eXsziCpHcioqKsHbtWjx//hyxsbFISkriWmw1B4uk1hRra2tBpoZLJBJkZ2crnLey/N/b\nwMAAv/76K7N4pHGU8IhaE6p5tLq5efMmYmJiFM4y+Wy9AsB3332HkJAQREREQEtLC6dOneKV8ISW\nmZmJqKgoVFdX4x//+AcWLVoEBwcH3nGF+DHRcEuzrq6OacE8aRwlPKLWHB0dAQD29vbYtGkTs7hZ\nWVno3bs3s3hyMplMoZ5N/uXM17Jly7Bq1SqEhYVhyZIlvJOdXPfu3ZnEaQkrV67Eli1b0K5dOzx+\n/BgzZ84U/MZpc7H6/4f8PSJVL4CQd9GBAwfg5uaGuLg4PH78mFnchuc0EomEWT9FHR0dWFpaQiqV\nokuXLsjKyuIds2PHjti9ezeeP3+OlJQUJk2TgfpzzNWrVyM8PBx1dXU4duwYk7gdOnTgepO2bdv2\nnW7MXFZWhqSkJGzYsAGxsbGIjY1V9ZI+CJTwCFEiIiICiYmJMDU1RXBwMGbPno2TJ0/y7vtoZ2eH\nqKgobhtW3kiar44dO0IikcDW1hZz585lciV/6dKl0NTU5G6/srpUERwcjN69eyM/Px9aWlpISUlh\nEreqqgrjx49HWFgYXF1d8fLlS0RERPBuNyf35MkTVFRUMBlnNHv2bNTW1sLKygrdunVDt27dGKyQ\nvAltaRK1tn//fq7RMQCsXr0a/v7+TGLfv38fv/32G549e4YBAwbgt99+Q2pqKq96N1dXV8TFxcHV\n1RWhoaFMBpQC9U2eASAoKAg5OTlcYTMfYrEYY8eORWVlJWQyGUpKSng35wbqt3GHDh3KlSmwalk2\nZ84c7s+jR49mEhMAUlNTER8fj5KSEujr66N169a8t0rbtGnD9NISeTuU8Ihau3nzJoyMjODk5ITo\n6Gjo6ekxievp6Yn27dtj/Pjx8Pf3576Um3uBoeElBQAoKSnB8uXLAbA9zxGJRMz6fi5fvhyZmZno\n0KED997mzZt5xzU0NERGRgbq6upw4cIFtG7dmndMAILN8EtOTsbhw4cxbdo0bN++ndeT7k8//QSg\n/mYmtRZreVSHR9SaVCpFQEAApFIpevTowaRzCVC/fcV3Bp668/DwwN69e5nHraysxNatW5GXlwcr\nKyvMmjULhoaGzP8eVr7++mvs2rULU6ZMQXx8PCZOnIjU1NRmxWrqrI66rwiPnvCIWmo4t05XVxdn\nz56FiYkJIiIimDRjvnHjBnbt2oXKykpIpVJoaGhg//79vOOWlZVh69atKCgogIWFBWbOnMlkwsHV\nq1fRt29f7vXNmzdhZ2fHK6a9vT3+/PNP/N///R/f5SkwNDTEtGnTuFZd73KyA4ChQ4dCIpFg3Lhx\nGDVqFAYOHNjsWJTUVIue8IhayszMbPQzFltbY8aMwcaNGxVu+mlra/OO6+3tDRcXF/Tp0wfXrl3D\ngQMHeE/mBoCpU6cqFF0vWLAA69ata1Ys+farTCZDVVWVwoQAFtuvO3bswH//+190794deXl5+Pzz\nzzFt2rRmx2uqQTSLlmXk/UFPeEQtNUxqp06d4p6Y+M4pk7O0tISpqSmTJNeQRCLByJEjAQCjRo1S\nmMjQHGlpaUhLS1MY41NXV8crpjypFRQUKFx+KSoq4hVX7sSJE9i7dy9EIhGkUik8PDx4JTxWZ4Dk\n/UcJj6i1sLAwaGlpwd7eHufPn0dGRgaTa+jDhw/HZ599xn3ha2hoMGlbZWxsjPj4ePTq1QvXr1/n\nfU44ePBg9O7dGwkJCVyJg1gs5urR+AgPD1f47xwdHd3sp8aGLC0tUVFRgTZt2qCiogJWVla84gnd\nskwopaWlCp1x3rVm3+8jSnhErf3+++/YvXs3AGDs2LGYMmUKk7jbtm1DXFwc8+LlyMhI7Nu3D0eO\nHIGFhQWioqJ4xcvNzUXfvn3Rp08f3L59W+Gz5t76O3PmDM6ePYvCwkLux0NtbS3vAnz5Vml1dTWG\nDBmCjh07ori4WOEWKB8ZGRmIiYlBcXExRCIRDAwM8OOPPzKJXV5ezp05yvuh8hETE4OLFy+iqKgI\nnTp1gq6uLnbs2MFgpaQplPCIWtPQ0EB2djZsbGxw+/Zt3oXhct27d0e3bt247vis6OjowNHREaam\nprC0tOQ9QbuoqAh9+/ZVGCQq19yEZ2VlBR0dHTx8+JCLIRaLeRdHC91OKzY2FvHx8fDx8UFsbCxi\nYmKYxN23bx9SU1O5AbATJkyAu7s7r5jnzp1DSkoKN+Vj4cKFTNZKmkYJj6i1JUuWIDo6mvv1vXTp\nUiZxi4qKMGLECIXtNhY1aDExMbh9+zZ69eqFw4cPo2fPnrzai8m381je/jM3N4e5ublgdW1C0dfX\nh6GhIaRSKYyMjJhMkwfqJ8Dv3bsXWlpaqK2txZQpU3gnvFatWgGo/8FWWVmJ/Px8Fkslb0AJj6i1\nLl26YOPGjSgrK2M6oHPNmjXMYjV08eJFJCcnc689PDyY9dP80Nna2kIikWDQoEFwcXFh0hUGqL+t\nWldXBy0tLdTV1UEqlfKO6ebmBolEAi8vL0ydOpVpZxjSOCpLIGrt1KlTWL16Ndq3b4+HDx/C39+f\n2U1NIXh4eCAmJgbt2rXDo0ePMG/ePOzbt0/Vy2pRQtQMvopl44D09HSsXbsWHTp0wMOHD7FgwQI4\nOTkxiU1amIwQNebm5iZ79uyZTCaTyZ4+fSqbMGGCilfUtKtXr8pcXV1lo0ePlk2YMEF25coVXvHK\ny8sb/YevpKQkmUwmk2VlZcnc3d1lR44c4R1TJpPJPD09FV7Pnz+fSdybN2/KPD09ZePHj5fV1tbK\noqOjmcQtKCiQvXz5Uvb48WPZy5cvZUVFRUzikpZHW5pE7ckvfvC9ANISPvnkk2a3pVJm3rx5St9n\nUUbx008/YdKkSdizZw/WrVuHBQsWYOzYsc2OJ0TNYEMrV65ETEwMFixYAC0tLdy4cYNJ3LCwMCQk\nJHAdcaKiopiUZ5CWRwmPqLWxY8fCzc0NNjY2yMnJwVdffcUkbnZ2NpKTk7lJAQCYfMmxnsqdmJjI\ne02NefbsGSoqKiAWi2FqagqxWMwrnpA1gwCgqamJNm3acC3nZDxPa4Qqz5DLyclBYWEhLCwsYG1t\nzSQmaRolPKLWJk+ejJEjR3K3NFldXAkMDERISAjzOjzWU7kb9hR9Fd+eohMmTMC8efMQHBwMiUQC\nc3NzXvGEqBlsyNbWFpGRkSgrK8OaNWtgb2/PK55Q5RlAfQnFr7/+Cnt7exw6dIj3bV3ydijhEbX2\n6NEjbN26lWty7O3tzSRJmZubM5tV1xDrqdwjRoxgsSyl3N3dFa7fr1y5klc8IWoGGwoMDMTZs2fR\ntm1bWFlZYfDgwbziCVmeceHCBbqtqwKU8Iha8/f3h5eXF7y9vfHrr7/C398fSUlJvOOOGzcOLi4u\n6NGjB/dec2fhAX89icmncn/88ce4ffs22rZty2udPXv2hL6+PpMp3HIhISFYvnz5azP8AH7F40LU\nDMpJpVLMmDEDO3bswKBBg5jHZ032/wfqym/r8t1+JW+HEh5Razo6Ohg+fDgAwNTUlNn8ttjYWPj5\n+THb0lT2JMai9iomJgaLFy/GvHnzoKGhwX1x8rm0IsRgWqGJRCL06NEDubm5Cj9SWJDJZArbxvLz\nVz4CAwPx7bffQiKRQEdHB0FBQXyXSd4C1eERtSSfHH3ixAm0bdsWdnZ2uHXrFioqKnhvvQHA/Pnz\nsX79et5xSMsZPXo0Xrx4AR0dHYhEIgBsknZYWBjXwUcikcDX15d3152WqEUkr6MnPKKWfvvtNwD1\nnVYAoLCwEPr6+tDX12cS/9mzZ5g8eTKsra25X/csBssKZcKECejUqRMcHBzg4OCgMNbnXdHUtiuL\nET9paWm8YyhjZ2eHqKgozJs3D/Pnz4eXlxfvmOvXr1d4At++fTuVOrQAesIjRAllA2bf9d6S9+/f\nx+XLl5GUlISHDx/i3LlzvOLJr8zLyW/CNpenp6fS91mNXhJSXFwcDh8+jNDQUF6XmeS1iFlZWejd\nuzeAv2oR4+LimKyVNI4SHiGNEGKwrFCSkpLwyy+/QCKRwMrKCo6OjrxvmbKcoq6OXr208+jRI7Rv\n3x5A87dKnz59isrKSqW1iI2VlxB2KOERokTDwbJZWVmora1lMlhWKJMnT4ahoSEGDBgABwcHXkNV\n5QXX6enpXLlAbW0t8vPzed2AFbJmkJC3QWd4RC3Jt9teLWAG2EyOFmqwrFCSkpJQXV2Nc+fOITg4\nGA8ePMCFCxeaFUuogmshawaB+osfMTExClPE+Vxa2blzZ6OfffPNN82OS1SHEh5RS+np6fD29uaS\nUkN86uXkhBosK5QffvgBN27cgJaWFoYMGcLrvLFhwXVpaSmXQP744w9ePyaEqBlsaNmyZVi1ahXC\nwsKwZMkS3jc0WVykacyrpQ3l5eVMJqmTptGWJiFK/P777wqDZRcuXIiuXbuqelmNyszMhL29PbS1\ntZnFjImJwcWLF1FUVIROnTpBV1cXO3bsaHa8FStWYPHixfD09GRWM9iQ/Mxx8uTJSEpKgpeXF+Lj\n43nHFcKCBQuwevVqaGpqoqysDH5+fti1a5eql/Xeoyc8Ql4hlUpx8uRJbNy4UdVLeWtC3CA9d+4c\nUlJS4OnpicTERCxcuJBXvMWLFwMQruF1x44dIZFIYGtri7lz5+Lly5dM4mZkZCAmJgbFxcUQiUQw\nMDDAjz/+yCumi4sLQkNDERAQAF9fX4SGhjJZK2kaJTxCXiESiXDr1i1VL0PlWrVqBaD+CayyshL5\n+flM4gpVMxgZGQkACAoKQk5ODrO4sbGxiI+Ph4+PD2JjYxETE8M75uDBg/H06VNMnDgRMTExzLvD\nEOUo4RGiRHV1NcaPHw87OztoamoC+PBuErq5uUEikcDLywtTp07Fl19+ySRuamoqVzMYEBDApGaw\nIZFIhJ49ezKLp6enB0NDQ0ilUhgZGSE7O7vZsV4tdRCJRFi0aBEA9Wrlpq4o4RG1tmPHDuzbt0/h\n7IrPF4f89uf06dNZLK/FXL16FQkJCXjy5AmkUimTczF5r8/PPvsMn332GYtlAlCsGfz000/h6OjI\nLLYQ/vnPf0IikWDQoEFwcXHhNSaJkppq0aUVotZcXV2RnJzMezipnPziw6tF1++6MWPGIDIyEh07\nduTe43vr7/jx49i+fTtqamq491h8YbOsGWwJDTvOPHnyBJWVlejcuTOvmOXl5Thw4ACKi4u59z60\nHQRVoCc8otZsbGwgkUiYJTwDAwNMnz4dubm5mD17tsJnfBsGC8nS0pLpNh4AbNmyBfHx8cyvy7Os\nGWzo+vXrWL58OTeBIDg4mGvfxUdYWBj348fIyAhhYWG8O874+flh5MiROHbsGNzd3Xltk5K3RwmP\nqLXLly9jyJAhMDMz497j8xSyYcMGPHr0CEuWLEFISAiLJbaI58+fM292bWVlBUNDQxbLU8CyZrCh\nyMhIrF+/Hh06dMCDBw/g5+fHa1yUvONMYWEh12WntrYWjx8/5r1WqVQKd3d3pKWlwcPD47UfV0QY\nlPCIWpOPCWKpffv22LBhA/O4Qpo1axazWPIv3/LycowYMUJhy5HFU+7AgQMxb948pjWDch06dAAA\nha3d5hKq4wxQf1nl5cuXMDY2RkJCAu7fv887JnkzOsMjaq22thYnT55UaCf1IbZ9qqmpwaFDh1BW\nVgZvb29cu3YNDg4OzYp17969Rj/jc2FDaGvWrEFeXh569eqF69evo1u3bvD391f1spQqLy+HoaEh\nKioqcPToUfTv359KE1oAPeERtebr64uuXbvi5MmTGDZsGEpKSlS9JJUIDAxE3759cebMGcyZMweb\nNm1qdsKTJ7UFCxbA0dERDg4O3NzBd5mfnx+ys7NRUFCAgQMHMj/TZElPTw/79+9HWVkZZsyYgV9+\n+UXVS/ogiFS9AEL4qKyshL+/P0xMTLBw4UI8ffqUSdzy8nKsXr0a4eHhqKurw7Fjx5jEFUpFRQWm\nTJnCbROy2LgJDAyEjo4O4uLi4OHhwbvTitBmzpwJGxsbjBo1Cj179kRQUJCql9SowMBA1NbW4syZ\nMxCLxdi0aZOql/RBoCc8otbkFzR0dXXx888/o7CwkEnc4OBgeHh4IC4uDlpaWkhJSWFWeC0EbW1t\nbgp8YWEhdHR0eMesqqrC8+fPUV1dDRMTE95X8eVY1wzeuXMH+fn5ePDgAXemW1dXh6KiIibrLSsr\nw5YtW7jyhJkzZ8LExIRXTPkPFPl66WSpZVDCI2otPDwcNTU1CA4ORlJSErOehNXV1Rg6dCg3IuZd\nH875/fffIyoqCuXl5Vi7di3CwsJ4x3R2dsaAAQMwZ84c2NnZMVhlvSVLlrxWM8hHaWkp8vLy8Pz5\ncy7pi8ViBAcHM4kfFBQEFxcXTJs2DdeuXUNgYCC2b9/OK6YQP1DIm1HCI2qtpqYGYrEYlpaWCA0N\nZVbPZGhoiIyMDNTV1eHChQuCjophwczMDGvWrGEa88qVK8jKysLp06exbt066OvrM5l4zrpmsF+/\nfujXrx+8vLygr6/PLK6cRCLByJEjAQCjRo3Cnj17eMcU4gcKeTNKeEStrVy5khuroqGhgVWrVjEZ\nCRMREYGtW7fCwMAAFy9exNKlS3nHFJIQXVGuXr2Ky5cv4/r168yu4wPC1AwCECTZAfUda+Lj47nb\nn0ZGRrxjmpmZYfXq1dwcvHd9B+F9QQmPqLWGX/BA/a9xFgwNDbmmvupAiK4o169fF6RmjmXNYEuI\njIzEvn37cOTIEVhYWCA6Opp3zGPHjmHz5s1o3749Hj16hNmzZ7/TZ8TvC0p4RK3Z29sjNDQUDg4O\n3BBUPuTd7Ovq6lBaWgpjY2OUl5ejbdu2OH78OIslC0KIrihz585lGk/O3t7+tZpBFvbv3w9XV1fu\n9erVq5nU4W3evBl+fn7c623btsHb25tXzO3btyM1NRWtWrWCRCLBpEmTKOG1AEp4RK0FBQXh9OnT\nyM/Ph5OTEwYPHswrnnwbMCQkBHPmzEHnzp1RVFSEbdu2sVguc0J3RRECy5rBhm7evAkjIyM4OTkh\nOjoaenp6vOJVV1fjxYsXuHLlCp48eQKZTIaXL1/i0qVLvBNe586dIRLVV4WJRCJ89NFHvOKRt0MJ\nj6i9IUOGYMiQIUxjFhQUcNfwO3fujLy8PKbxWfnuu+9UvYS/Tagr+d9//z0CAgJw/Phx9OjRg3d/\nyuPHj+PgwYPIy8vDvHnzIJPJoK2tjeHDh/Ne6/379zF8+HB069YNeXl5MDMz49b7rv5QeR9QwiNE\nCXt7e8yZMwf29va4ceMG761SoahjVxTWV/KXLVumUI959uxZmJiYICIigtdlGGdnZzg7O+PKlSv4\n17/+xWuNr1q7di3TeOTtUC9NQhohb1NlYWHxTrepAur7X165cgWZmZn4/fffYW5ujh9++EHVy1Lq\n/v37iIqKQn5+Prp27YrAwECFaRd/V2ZmZqOfsZrEwFp6ejqcnJxw584dbNq0CS4uLvj3v/+t6mW9\n9+gJj5AGrl69ir59+3LbbVpaWrh79y7u3r3Ldcx/FwnVFUUIrGsG+Z7VqUJSUhKcnJwQFxeHiRMn\nIioqihJeC6CER0gDRUVF6Nu3L7fl1tC7nPCE6ooiBNY1g7t37270sxUrVjQ7rlxiYiI8PT251wcO\nHMD48eN5xayqqkJdXR1kMhk++eQTZgOMSdMo4RHSgLOzM6RSKbS1tTFz5kxVL+etCdUVRQisawZZ\nJLWmpKenKyS89PR03gnv008/hZubGwICAiCRSASZDUheRwmPkFeIRCLcunVL1cv4W4TqiiIEoSap\n//nnn9i5cyeKi4u5m598bjzu2bMHycnJuHv3LlefKRKJeJe+AMD8+fMxf/587jXf3pzk7dClFUKU\nmDVrFh4/fgzj8LMqAAAJTElEQVQ7OztoamoCYNP+SiixsbHo168f7O3t39mnhYY1g6WlpcxrBidO\nnAgfHx/ExMRg7ty5uHDhApMG0sePH8eoUaN4xyGqR094hCgxffp0VS/hbxGqKwpLQtcMisViDBgw\nAJs3b8bAgQO5Hqt89ejRA5GRkVzxOSD8NioRBiU8QpR4V6+zqzOhawbFYjEkEgk6d+6MFStWoLy8\nnEnchQsXwsfHh9k4I7nCwkI8ePAAjo6OKC8vR5s2bZjGJ6+jLU1CSIsSumZQIpHg3LlzsLe3R7t2\n7XjH8/HxwYYNGxis7C8bNmxAbm4uioqKcPDgQcyYMYPO8VqASNULIIR8WISqGZTfqm3VqhXXT5MF\nTU1NLFq0CDt27MDOnTu5ocB8XLp0CevXr4e+vj40NDRQW1vLYKXkTWhLkxAlamtrcfLkSTx8+JA7\nt/nmm29UvKr3A+uawTt37iA/Px8PHjzgGgbU1dWhqKiId2wAGDp0KJM4r5JIJNDQ0EBNTQ2zfqKk\naZTwCFHC19cXXbt2xcmTJzFs2DCUlJSoeknvDdY1g6WlpcjLy8Pz58+5hgFisZjJDU2gPkEDULi0\nwtfcuXMxZcoUFBUVYerUqfD19WUSlzSNzvAIUcLT05PrsJGYmCjIOc6H6vz58wo1g3369GFyy/TZ\ns2eCTD1PTU1FfHw8SkpKoK+vj9atW+PgwYNMYpeVldFllRZET3iEKNGw+/7PP/+MwsJC1S7oPSLU\nJHUhkh0AJCcn4/Dhw5g2bRq2b9+O8PBw3jGzs7Oxd+9ehafGd7UzzvuEEh4hSoSHh6OmpgbBwcFI\nSkpCaGioqpf03lCHmsGGDAwMIBaLIZPJIBKJkJ+fzztmYGAgQkJCYGpqymCF5G1RwiNEia5duwIA\nLC0t3+kOK6S+ns3CwgK3b99+7bOPP/6Yd/yhQ4dCIpFg3LhxGDVqFAYOHMg7prm5Ofr37887Dvl7\n6AyPEKLWtm3bBm9vbyxevPi1z1h0RHn+/DnzEUQnTpzA1q1b0aNHD+496t4iPEp4hBDShG+++QZ1\ndXWwt7eHg4MDPvnkE95T2kePHg0/Pz+FLU1bW1u+SyVvQFuahDQgr+NS5l2eh0eEs3PnTtTU1OD8\n+fNYu3YtcnJyeE/T6NKlC4YNG8ZoheRtUcIjpAF5HdetW7dQVVUFOzs73Lp1CwYGBpTwPlCrVq1C\nYWEhDA0N4e7uDgcHB94xnz17hsmTJ8Pa2pq7EUxnxcKjhEdIA/IbhHPmzOG67ctkMsyZM0eVyyIq\n9PjxY2hoaMDY2Bjt27dnUjcnH5VEWhYlPEKUKC4uRklJCdq1a4fS0lI8fPhQ1Usib7Bjxw7s27dP\nobbv6NGjvOOuWrUKAHD69GnExMQw2dKkaRyqQQmPECVCQkLg4+ODqqoq6OjoKL0BSN4tx48fR1pa\nGsRiMdO4/v7+uH//PiwtLTF58mRKVmqMbmkS0gQhrqQTYXz33XcIDAyEgYEB07j37t3jZvmx8mob\ntBcvXkBXV5fp30FeR+OBCFHizJkzGD9+PNzc3FBXV0dPeGrg8uXLGDJkCMaMGcP9wwLrZAcA3377\nrcJr+verZdCWJiFKbNq0CcnJyZg+fTq0tLRw7949VS+JvEFTJSXviqysLGRlZaG4uJibq1dbW4tH\njx6peGUfBkp4hCihqakJbW1t7sq4VCpV8YrImwg1w/DHH3/EyJEjudenT5/GkCFDmhVLR0cHxsbG\n0NPTg7GxMWQyGcRiMbNhtaRptKVJiBJOTk6YP38+Hjx4gICAAKrBUwO+vr7Izs5GSkoKysrKkJub\nyyRucnKywusDBw40O5a1tTWcnZ1x6NAhtG3bFgDwxRdfQCKR8FojeTv0hEeIEl5eXhg4cCDy8vLQ\npUsXdO/eXdVLIm9QWVkJf39/ZGVlYeHChfDx8eEVb8+ePUhOTsbdu3e580ANDQ3Y2NjwXmt4eDh0\ndXVx5coVODs7IyIigtviJMKhhEeIEuXl5Thy5AiePHmC4cOH49ixY/jyyy9VvSzSBNYzDCdNmoRJ\nkyYhISEBU6dOZbDCvxQUFCAhIQGenp4AgJcvXzKNT5SjLU1ClAgODkbv3r2Rn58PLS0tpKSkqHpJ\n5A0azjC8dOkS7xmGhw8fxuHDh+Hi4sJohX8RiUQoLS2FhoYGKioqoKVFzx4tgRIeIUpUV1dj6NCh\n0NTUBPDX0wN5d9XU1EAsFsPS0hKhoaFo3bo1r3gymQxClSkHBwfDz88PeXl58PX1RWBgoCB/D1FE\nPysIUcLQ0BAZGRmoq6vDhQsXeH95EuGtXLmS63+qoaGBVatWIT4+vtnxnJ2dGa3sddbW1khISBAs\nPlGOEh4hSkRERGDr1q0wMDDAxYsXsXTpUlUvibxBTU2NwmtWNx9Z9uhsqmn05s2bmxWTvD1qLUYI\neS9ERkaisrISDg4OyMzMhL6+PoKDg3nHdXV1RXJyMpMenU01MBCiowtRRE94hCghVOd9IpygoCCc\nPn0a+fn5cHJywuDBg5nEtbGxgUQiYZLwKKmpFj3hEaIEy1/1RL19/vnnKC0thZmZGfce/fhRT/SE\nR4gSLH/VE/WmDj06yduhJzxClKBf9USurKwMW7duRUFBASwsLDBz5kyYmJioelmkGSjhEUJIE7y9\nveHi4oI+ffrg2rVrOHDgALZv367qZZFmoMJzQghpgkQiwciRI2FqaopRo0ahurpa1UsizURneIQQ\n0gRjY2PEx8ejV69euH79OoyMjFS9JNJMtKVJCCFNqKqqwr59+1BYWAgLCwu4ublBV1dX1csizUAJ\nj5AG1qxZAz8/P6UdMagTxoeloqKi0c+o1Zx6ooRHSAOPHz9G27ZtlXbEoKLhD4unpyc0NDTw9OlT\n3L9/H5aWligoKIC5uTkOHjyo6uWRZqAzPEIakE+hpuRGEhMTAQB+fn5ISEiAgYEBnj59iu+//161\nCyPNRrc0CSGkCUVFRdDX1wcA6Ovr448//lDxikhz0RMeIYQ0YcyYMXB3d4e1tTVyc3MxevRoVS+J\nNBOd4RFCyBuUlZXh7t276NSpE9q0aaPq5ZBmooRHCCHkg0BneIQQQj4IlPAIIYR8EOjSCiGEKEFN\nCN4/dIZHCCFKUBOC9w8lPEIIIR8EOsMjhBDyQaCERwgh5INACY8QQsgHgRIeIYSQDwIlPEIIIR+E\n/wcO0kYdL69f3QAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "URPyrfw0nT0x",
        "colab_type": "text"
      },
      "source": [
        "# Response Ranking\n",
        "\n",
        "ConveRT is trained on the response ranking task, so it can be used to find good responses to a given conversational context.\n",
        "\n",
        "This section demonstrates how to rank responses, by computing cosine similarities of context and response representations in the shared response ranking space. Response representations for a fixed candidate list are first pre-computed. When a new context is provided, it is encoded and then compared to the pre-computed response representations."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ocu1flTekqbU",
        "colab_type": "code",
        "outputId": "26e13a78-9234-4df7-9f14-a2848505e294",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "if sess is not None:\n",
        "    sess.close()\n",
        "\n",
        "sess = tf.InteractiveSession(graph=tf.Graph())\n",
        "\n",
        "module = tfhub.Module(\"http://models.poly-ai.com/convert/v1/model.tar.gz\")\n",
        "\n",
        "text_placeholder = tf.placeholder(dtype=tf.string, shape=[None])\n",
        "context_encoding_tensor = module(text_placeholder, signature=\"encode_context\")\n",
        "response_encoding_tensor = module(text_placeholder, signature=\"encode_response\")\n",
        "\n",
        "encoding_dim = int(context_encoding_tensor.shape[1])\n",
        "print(f\"ConveRT encodes contexts & responses to {encoding_dim}-dimensional vectors\")\n",
        "\n",
        "sess.run(tf.tables_initializer())\n",
        "sess.run(tf.global_variables_initializer())\n",
        "\n",
        "def encode_contexts(texts):\n",
        "    return sess.run(context_encoding_tensor, feed_dict={text_placeholder: texts})\n",
        "\n",
        "def encode_responses(texts):\n",
        "    return sess.run(response_encoding_tensor, feed_dict={text_placeholder: texts})"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "ConveRT encodes contexts & responses to 512-dimensional vectors\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p5EIo6dsnt_V",
        "colab_type": "code",
        "outputId": "72eedbc3-7a90-4735-d891-e3c620526276",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "# Pre-encode responses.\n",
        "# Download a file of random facts to serve as candidate responses for the demo.\n",
        "!wget -q https://raw.githubusercontent.com/assaf/dailyhi/a35ba9fc1744dfb399d82abbc5f2d4e4216fbd45/facts.txt -O facts.txt\n",
        "\n",
        "import numpy as np\n",
        "import random\n",
        "\n",
        "responses = []\n",
        "with open(\"facts.txt\") as f:\n",
        "    for line in f:\n",
        "        responses.append(line.strip())\n",
        "\n",
        "print(\"\\n\\t- \".join([\"Three random facts:\"] + random.sample(responses, 3)))\n",
        "\n",
        "# Encode the responses in batches of 64.\n",
        "batch_size = 64\n",
        "response_encodings = []\n",
        "for i in range(0, len(responses), batch_size):\n",
        "    batch = responses[i:i + batch_size]\n",
        "    response_encodings.append(encode_responses(batch))\n",
        "\n",
        "response_encodings = np.concatenate(response_encodings)\n",
        "print(f\"Encoded {response_encodings.shape[0]} candidate responses.\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Three random facts:\n",
            "\t- Chocolate accounts for less than two percent of the fat in the American diet\n",
            "\t- During World War II, there was not enough sugar in the U.S. to make candy as it was sent to the troops overseas. At this time, popcorn was consumed three times more than its usual amount\n",
            "\t- There are an estimated 2,500 collisions between birds and planes each year in the US\n",
            "Encoded 3090 candidate responses.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RkBdMitBrA5u",
        "colab_type": "code",
        "cellView": "both",
        "outputId": "9630c899-e02b-4fc3-c9e2-c42bb4dd8ce3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#@title Rank responses { run: \"auto\" }\n",
        "#@markdown Find the best response in a list of around 3 thousand random facts. \n",
        "context = \"what can you tell me about forests?\" # @param {type:\"string\"}\n",
        "if context:\n",
        "    context_encoding = encode_contexts([context])\n",
        "    scores = np.dot(response_encodings, context_encoding.T)\n",
        "    top_index = np.argmax(scores)\n",
        "    top_score = float(scores[top_index])\n",
        "    print(f\"[{top_score:.3f}] {responses[top_index]}\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[0.374] An acre of trees can remove about 13 tons of dust and gases every year from the surrounding environment\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "--fvxsGurWZR",
        "colab_type": "text"
      },
      "source": [
        "# Multi-Context ConveRT\n",
        "\n",
        "The multi-context version of ConveRT has the same interface as ConveRT, except that when encoding contexts, it also takes extra dialogue history as input.\n",
        "\n",
        "The extra context must be passed as a single string, where earlier contexts are joined with spaces from most recent to oldest.\n",
        "\n",
        "An example from the ConveRT paper, where extra contexts would help is:\n",
        "\n",
        "> Student:  I’m looking for good courses to take.\n",
        ">\n",
        "> Advisor: Are you looking for courses in a specific area?\n",
        ">\n",
        "> Student: Not in particular.\n",
        ">\n",
        "> Advisor: Are you looking to take a very difficult class?\n",
        "\n",
        "This section will demonstrate how Multi-context ConveRT can be used to select that last response from the Advisor. (Without more context that \"Not in particular\", this would be hard)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9xUwIHwBsUlq",
        "colab_type": "code",
        "outputId": "a8e97e03-9e41-45ea-8049-45ebccfc73d6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "if sess is not None:\n",
        "    sess.close()\n",
        "\n",
        "sess = tf.InteractiveSession(graph=tf.Graph())\n",
        "\n",
        "module = tfhub.Module(\"http://models.poly-ai.com/multi_context_convert/v1/model.tar.gz\")\n",
        "\n",
        "text_placeholder = tf.placeholder(dtype=tf.string, shape=[None])\n",
        "extra_text_placeholder = tf.placeholder(dtype=tf.string, shape=[None])\n",
        "\n",
        "# The encode_context signature now also takes the extra context.\n",
        "context_encoding_tensor = module(\n",
        "    {\n",
        "        'context': text_placeholder,\n",
        "        'extra_context': extra_text_placeholder,\n",
        "    },\n",
        "    signature=\"encode_context\"\n",
        ")\n",
        "response_encoding_tensor = module(text_placeholder, signature=\"encode_response\")\n",
        "\n",
        "encoding_dim = int(context_encoding_tensor.shape[1])\n",
        "print(f\"ConveRT encodes contexts & responses to {encoding_dim}-dimensional vectors\")\n",
        "\n",
        "sess.run(tf.tables_initializer())\n",
        "sess.run(tf.global_variables_initializer())\n",
        "\n",
        "def encode_context(dialogue_history):\n",
        "    \"\"\"Encode the dialogue context to the response ranking vector space.\n",
        "\n",
        "    Args:\n",
        "        dialogue_history: a list of strings, the dialogue history, in\n",
        "            chronological order.\n",
        "    \"\"\"\n",
        "\n",
        "    # The context is the most recent message in the history.\n",
        "    context = dialogue_history[-1]\n",
        "\n",
        "    extra_context = list(dialogue_history[:-1])\n",
        "    extra_context.reverse()\n",
        "    extra_context_feature = \" \".join(extra_context)\n",
        "\n",
        "    return sess.run(\n",
        "        context_encoding_tensor,\n",
        "        feed_dict={\n",
        "            text_placeholder: [context],\n",
        "            extra_text_placeholder: [extra_context_feature],\n",
        "        }\n",
        "    )[0]\n",
        "\n",
        "def encode_responses(texts):\n",
        "    return sess.run(response_encoding_tensor, feed_dict={text_placeholder: texts})"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "ConveRT encodes contexts & responses to 512-dimensional vectors\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tosSAnB0tTGM",
        "colab_type": "code",
        "outputId": "3c24167f-47a9-4a5f-965d-7fa84b97c544",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "context_encoding = encode_context(\n",
        "    [\n",
        "     \"I'm looking for good courses to take.\",\n",
        "     \"Are you looking for courses in a specific area?\",\n",
        "     \"Not in particular.\",\n",
        "     ]\n",
        ")\n",
        "\n",
        "candidate_responses = [\n",
        "    \"Me neither.\",\n",
        "    # We hope it selects the following sentence:\n",
        "    \"Are you looking to take a very difficult class?\",\n",
        "    \"Why not?\",\n",
        "    \"Please finish the exercise first.\",\n",
        "    \"Nothing in particular?\",\n",
        "    \"Would you like fries with that?\",\n",
        "    \"School's for fools, look at me!\",\n",
        "    \"Our higher education system is one of the things that makes America exceptional.\",\n",
        "]\n",
        "response_encodings = encode_responses(candidate_responses)\n",
        "\n",
        "scores = context_encoding.dot(response_encodings.T)\n",
        "top_idx = np.argmax(scores)\n",
        "\n",
        "print(\n",
        "    f\"Best response:\\n\\t'{candidate_responses[top_idx]}'\\n\"\n",
        "    f\"\\t\\tscore: {scores[top_idx]:.3f}\"\n",
        ")\n",
        "\n",
        "indices = np.argsort(scores)[::-1][1:]\n",
        "print(\"Other responses:\")\n",
        "for index in indices:\n",
        "    print(\n",
        "        f\"\\t'{candidate_responses[index]}'\\n\"\n",
        "        f\"\\t\\tscore: {scores[index]:.3f}\"\n",
        "    )\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Best response:\n",
            "\t'Are you looking to take a very difficult class?'\n",
            "\t\tscore: 0.439\n",
            "Other responses:\n",
            "\t'Nothing in particular?'\n",
            "\t\tscore: 0.285\n",
            "\t'Why not?'\n",
            "\t\tscore: 0.137\n",
            "\t'Me neither.'\n",
            "\t\tscore: 0.113\n",
            "\t'Please finish the exercise first.'\n",
            "\t\tscore: 0.084\n",
            "\t'Our higher education system is one of the things that makes America exceptional.'\n",
            "\t\tscore: 0.055\n",
            "\t'Would you like fries with that?'\n",
            "\t\tscore: 0.025\n",
            "\t'School's for fools, look at me!'\n",
            "\t\tscore: 0.016\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}