apps/web-crawl-q-and-a/web-qa.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://someweirdsin.github.io/\n",
      "https://someweirdsin.github.io/008_Contacts.html\n",
      "https://someweirdsin.github.io/010_Augments.html\n",
      "https://someweirdsin.github.io/Augments/010b_AugPack.html\n",
      "https://someweirdsin.github.io/HackProg/011c_Jockeying.html\n",
      "https://someweirdsin.github.io/Inventory/009c_Weapons.html\n",
      "https://someweirdsin.github.io/Templates/Tweeker.html\n",
      "https://someweirdsin.github.io/002_Combat.html\n",
      "https://someweirdsin.github.io/Augments/010a_AugmentList.html\n",
      "https://someweirdsin.github.io/Inventory/009f_Drugs.html\n",
      "https://someweirdsin.github.io\n",
      "https://someweirdsin.github.io/Inventory/009b_Armor.html\n",
      "https://someweirdsin.github.io/003_Death.html\n",
      "https://someweirdsin.github.io/999_Credits.html\n",
      "https://someweirdsin.github.io/goblinpunch.blogspot.com\n",
      "HTTP Error 404: Not Found\n",
      "https://someweirdsin.github.io/Templates/ConsoleJockey.html\n",
      "https://someweirdsin.github.io/Templates/Investigator.html\n",
      "https://someweirdsin.github.io/Inventory/009d_Equipment.html\n",
      "https://someweirdsin.github.io/Templates/Augmented.html\n",
      "https://someweirdsin.github.io/Templates/Face.html\n",
      "https://someweirdsin.github.io/Inventory/009e_Drones.html\n",
      "https://someweirdsin.github.io/HackProg/011b_Programming.html\n",
      "https://someweirdsin.github.io/004_Healing.html\n",
      "https://someweirdsin.github.io/009_Inventory.html\n",
      "https://someweirdsin.github.io/011_HackAndProg.html\n",
      "https://someweirdsin.github.io/001_Overview.html\n",
      "https://someweirdsin.github.io/Templates/SteppingRaz.html\n",
      "https://someweirdsin.github.io/005_CharGen.html\n",
      "https://someweirdsin.github.io/007_Templates.html\n",
      "https://someweirdsin.github.io/998_AppendixN.html\n",
      "https://someweirdsin.github.io/Inventory/009a_PriceList.html\n",
      "https://someweirdsin.github.io/Templates/PeopleChamp.html\n",
      "https://someweirdsin.github.io/Augments/010c_AugmentPrice.html\n",
      "https://someweirdsin.github.io/HackProg/011d_Warez.html\n",
      "https://someweirdsin.github.io/Templates/Fixer.html\n",
      "https://someweirdsin.github.io/006_FailedCareers.html\n",
      "https://someweirdsin.github.io/HackProg/011a_Hacking.html\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import re\n",
    "import urllib.request\n",
    "from bs4 import BeautifulSoup\n",
    "from collections import deque\n",
    "from html.parser import HTMLParser\n",
    "from urllib.parse import urlparse\n",
    "import os\n",
    "\n",
    "# Regex pattern to match a URL\n",
    "HTTP_URL_PATTERN = r'^http[s]*://.+'\n",
    "\n",
    "# Define root domain to crawl\n",
    "domain = \"someweirdsin.github.io/\"\n",
    "full_url = \"https://someweirdsin.github.io/\"\n",
    "\n",
    "# Create a class to parse the HTML and get the hyperlinks\n",
    "class HyperlinkParser(HTMLParser):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        # Create a list to store the hyperlinks\n",
    "        self.hyperlinks = []\n",
    "\n",
    "    # Override the HTMLParser's handle_starttag method to get the hyperlinks\n",
    "    def handle_starttag(self, tag, attrs):\n",
    "        attrs = dict(attrs)\n",
    "\n",
    "        # If the tag is an anchor tag and it has an href attribute, add the href attribute to the list of hyperlinks\n",
    "        if tag == \"a\" and \"href\" in attrs:\n",
    "            self.hyperlinks.append(attrs[\"href\"])\n",
    "\n",
    "# Function to get the hyperlinks from a URL\n",
    "def get_hyperlinks(url):\n",
    "    \n",
    "    # Try to open the URL and read the HTML\n",
    "    try:\n",
    "        # Open the URL and read the HTML\n",
    "        with urllib.request.urlopen(url) as response:\n",
    "\n",
    "            # If the response is not HTML, return an empty list\n",
    "            if not response.info().get('Content-Type').startswith(\"text/html\"):\n",
    "                return []\n",
    "            \n",
    "            # Decode the HTML\n",
    "            html = response.read().decode('utf-8')\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        return []\n",
    "\n",
    "    # Create the HTML Parser and then Parse the HTML to get hyperlinks\n",
    "    parser = HyperlinkParser()\n",
    "    parser.feed(html)\n",
    "\n",
    "    return parser.hyperlinks\n",
    "\n",
    "# Function to get the hyperlinks from a URL that are within the same domain\n",
    "def get_domain_hyperlinks(local_domain, url):\n",
    "    clean_links = []\n",
    "    for link in set(get_hyperlinks(url)):\n",
    "        clean_link = None\n",
    "\n",
    "        # If the link is a URL, check if it is within the same domain\n",
    "        if re.search(HTTP_URL_PATTERN, link):\n",
    "            # Parse the URL and check if the domain is the same\n",
    "            url_obj = urlparse(link)\n",
    "            if url_obj.netloc == local_domain:\n",
    "                clean_link = link\n",
    "\n",
    "        # If the link is not a URL, check if it is a relative link\n",
    "        else:\n",
    "            if link.startswith(\"/\"):\n",
    "                link = link[1:]\n",
    "            elif link.startswith(\"#\") or link.startswith(\"mailto:\"):\n",
    "                continue\n",
    "            clean_link = \"https://\" + local_domain + \"/\" + link\n",
    "\n",
    "        if clean_link is not None:\n",
    "            if clean_link.endswith(\"/\"):\n",
    "                clean_link = clean_link[:-1]\n",
    "            clean_links.append(clean_link)\n",
    "\n",
    "    # Return the list of hyperlinks that are within the same domain\n",
    "    return list(set(clean_links))\n",
    "\n",
    "\n",
    "def crawl(url):\n",
    "    # Parse the URL and get the domain\n",
    "    local_domain = urlparse(url).netloc\n",
    "\n",
    "    # Create a queue to store the URLs to crawl\n",
    "    queue = deque([url])\n",
    "\n",
    "    # Create a set to store the URLs that have already been seen (no duplicates)\n",
    "    seen = set([url])\n",
    "\n",
    "    # Create a directory to store the text files\n",
    "    if not os.path.exists(\"text/\"):\n",
    "            os.mkdir(\"text/\")\n",
    "\n",
    "    if not os.path.exists(\"text/\"+local_domain+\"/\"):\n",
    "            os.mkdir(\"text/\" + local_domain + \"/\")\n",
    "\n",
    "    # Create a directory to store the csv files\n",
    "    if not os.path.exists(\"processed\"):\n",
    "            os.mkdir(\"processed\")\n",
    "\n",
    "    # While the queue is not empty, continue crawling\n",
    "    while queue:\n",
    "\n",
    "        # Get the next URL from the queue\n",
    "        url = queue.pop()\n",
    "        print(url) # for debugging and to see the progress\n",
    "\n",
    "        # Save text from the url to a <url>.txt file\n",
    "        with open('text/'+local_domain+'/'+url[8:].replace(\"/\", \"_\") + \".txt\", \"w\") as f:\n",
    "\n",
    "            # Get the text from the URL using BeautifulSoup\n",
    "            soup = BeautifulSoup(requests.get(url).text, \"html.parser\")\n",
    "\n",
    "            # Get the text but remove the tags\n",
    "            text = soup.get_text()\n",
    "\n",
    "            # If the crawler gets to a page that requires JavaScript, it will stop the crawl\n",
    "            if (\"You need to enable JavaScript to run this app.\" in text):\n",
    "                print(\"Unable to parse page \" + url + \" due to JavaScript being required\")\n",
    "            \n",
    "            # Otherwise, write the text to the file in the text directory\n",
    "            f.write(text)\n",
    "\n",
    "        # Get the hyperlinks from the URL and add them to the queue\n",
    "        for link in get_domain_hyperlinks(local_domain, url):\n",
    "            if link not in seen:\n",
    "                queue.append(link)\n",
    "                seen.add(link)\n",
    "\n",
    "crawl(full_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_newlines(serie):\n",
    "    serie = serie.str.replace('\\n', ' ')\n",
    "    serie = serie.str.replace('\\\\n', ' ')\n",
    "    serie = serie.str.replace('  ', ' ')\n",
    "    serie = serie.str.replace('  ', ' ')\n",
    "    return serie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/kz/7gjjgm1d5bs72ymzgwgkmst00000gp/T/ipykernel_7606/2931785837.py:3: FutureWarning: The default value of regex will change from True to False in a future version.\n",
      "  serie = serie.str.replace('\\\\n', ' ')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fname</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>n.github.io Templates Fixer.html</td>\n",
       "      <td>n.github.io Templates Fixer.html.   Fixer - So...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>n.github.io Inventory 009f Drugs.html</td>\n",
       "      <td>n.github.io Inventory 009f Drugs.html.   Drugs...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>n.github.io HackProg 011b Programming.html</td>\n",
       "      <td>n.github.io HackProg 011b Programming.html.   ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>n.github.io Inventory 009c Weapons.html</td>\n",
       "      <td>n.github.io Inventory 009c Weapons.html.   Wea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>n.github.io 009 Inventory.html</td>\n",
       "      <td>n.github.io 009 Inventory.html.   Inventory - ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        fname  \\\n",
       "0            n.github.io Templates Fixer.html   \n",
       "1       n.github.io Inventory 009f Drugs.html   \n",
       "2  n.github.io HackProg 011b Programming.html   \n",
       "3     n.github.io Inventory 009c Weapons.html   \n",
       "4              n.github.io 009 Inventory.html   \n",
       "\n",
       "                                                text  \n",
       "0  n.github.io Templates Fixer.html.   Fixer - So...  \n",
       "1  n.github.io Inventory 009f Drugs.html.   Drugs...  \n",
       "2  n.github.io HackProg 011b Programming.html.   ...  \n",
       "3  n.github.io Inventory 009c Weapons.html.   Wea...  \n",
       "4  n.github.io 009 Inventory.html.   Inventory - ...  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Create a list to store the text files\n",
    "texts=[]\n",
    "\n",
    "# Get all the text files in the text directory\n",
    "for file in os.listdir(\"text/\" + domain + \"/\"):\n",
    "\n",
    "    # Open the file and read the text\n",
    "    with open(\"text/\" + domain + \"/\" + file, \"r\") as f:\n",
    "        text = f.read()\n",
    "\n",
    "        # Omit the first 11 lines and the last 4 lines, then replace -, _, and #update with spaces.\n",
    "        texts.append((file[11:-4].replace('-',' ').replace('_', ' ').replace('#update',''), text))\n",
    "\n",
    "# Create a dataframe from the list of texts\n",
    "df = pd.DataFrame(texts, columns = ['fname', 'text'])\n",
    "\n",
    "# Set the text column to be the raw text with the newlines removed\n",
    "df['text'] = df.fname + \". \" + remove_newlines(df.text)\n",
    "df.to_csv('processed/scraped.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot: >"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAncElEQVR4nO3df3DU5YHH8c+GbBaCbCBESKIJxB8H5fcpPy7FUhgCMaUgvV5LDefl8AZPjVUuHgr2kMQfhdoZh9Zy2rubyt1cA95dDXpasTkkUE5AAkSk2ki8WKgYKFKyQGRds8/9wWTHNYHsxt0n+X55v2Z2dL/fZ/d5PnmW5DP7I/EYY4wAAAAsSentBQAAgMsL5QMAAFhF+QAAAFZRPgAAgFWUDwAAYBXlAwAAWEX5AAAAVlE+AACAVam9vYDPC4fDOnbsmAYNGiSPx9PbywEAADEwxujMmTPKzc1VSsqln9voc+Xj2LFjysvL6+1lAACAHjh69KiuvvrqS47pc+Vj0KBBki4s3u/3dzs+FArpV7/6lebOnSuv15vs5Vnn9nyS+zOSz9ncnk9yf0by2REIBJSXlxf5OX4pfa58dLzU4vf7Yy4f6enp8vv9rn1QuTmf5P6M5HM2t+eT3J+RfHbF8pYJ3nAKAACsonwAAACrKB8AAMAqygcAALCK8gEAAKyifAAAAKsoHwAAwCrKBwAAsIryAQAArKJ8AAAAqygfAADAKsoHAACwivIBAACsonwAAACrUnt7AYjNuMpXFWzv/s8U9xXvr53X20sAAPRRPPMBAACsonwAAACrKB8AAMAqygcAALCK8gEAAKyifAAAAKsoHwAAwCrKBwAAsIryAQAArKJ8AAAAqygfAADAKsoHAACwivIBAACsonwAAACrKB8AAMAqygcAALAq7vKxY8cOzZ8/X7m5ufJ4PNq8eXOnMe+8844WLFigjIwMDRw4UFOmTNGRI0cSsV4AAOBwcZePc+fOaeLEiVq/fn2X59977z3ddNNNGj16tOrq6nTw4EGtWrVK/fv3/8KLBQAAzpca7w1KSkpUUlJy0fPf+9739LWvfU1PPPFE5Ni1117bs9UBAADXibt8XEo4HNbLL7+sBx54QMXFxTpw4IAKCgq0cuVKLVy4sMvbBINBBYPByPVAICBJCoVCCoVC3c7ZMSaWsU7UkcuXYnp5JfGJZz8ulz0knzO5PZ/k/ozks7uOWHiMMT3+qebxeFRTUxMpFi0tLcrJyVF6eroee+wxzZo1S1u2bNFDDz2kbdu26atf/Wqn+6isrFRVVVWn49XV1UpPT+/p0gAAgEVtbW0qLS1Va2ur/H7/JccmtHwcO3ZMV111lW699VZVV1dHxi1YsEADBw7Uxo0bO91HV8985OXl6eTJk90uXrrQtGprazVnzhx5vd6eRumzOvKtqk9RMOzp7eXE7FBlccxjL5c9JJ8zuT2f5P6M5LMjEAgoKysrpvKR0JddsrKylJqaqjFjxkQd/9KXvqSdO3d2eRufzyefz9fpuNfrjeuLGO94pwmGPQq2O6d89GQv3L6H5HM2t+eT3J+RfMmfP1YJ/T0faWlpmjJlihobG6OOv/vuuxoxYkQipwIAAA4V9zMfZ8+eVVNTU+R6c3OzGhoalJmZqfz8fC1fvlyLFi3SjBkzIu/5+O///m/V1dUlct0AAMCh4i4f9fX1mjVrVuR6RUWFJKmsrEwbNmzQN77xDT3zzDNas2aN7r33Xo0aNUq/+MUvdNNNNyVu1QAAwLHiLh8zZ85Ud+9Rvf3223X77bf3eFEAAMC9+NsuAADAKsoHAACwivIBAACsonwAAACrKB8AAMAqygcAALCK8gEAAKyifAAAAKsoHwAAwCrKBwAAsIryAQAArKJ8AAAAqygfAADAKsoHAACwivIBAACsonwAAACrKB8AAMAqygcAALCK8gEAAKyifAAAAKsoHwAAwCrKBwAAsIryAQAArKJ8AAAAqygfAADAKsoHAACwivIBAACsonwAAACrKB8AAMAqygcAALAq7vKxY8cOzZ8/X7m5ufJ4PNq8efNFx955553yeDxat27dF1giAABwk7jLx7lz5zRx4kStX7/+kuNqamq0e/du5ebm9nhxAADAfVLjvUFJSYlKSkouOeaDDz7Qd7/7Xb366quaN29ejxcHAADcJ+7y0Z1wOKzbbrtNy5cv19ixY7sdHwwGFQwGI9cDgYAkKRQKKRQKdXv7jjGxjHWijly+FNPLK4lPPPtxuewh+ZzJ7fkk92ckn911xMJjjOnxTzWPx6OamhotXLgwcmzNmjXatm2bXn31VXk8Ho0cOVLLli3TsmXLuryPyspKVVVVdTpeXV2t9PT0ni4NAABY1NbWptLSUrW2tsrv919ybEKf+di3b59+9KMfaf/+/fJ4PDHdZuXKlaqoqIhcDwQCysvL09y5c7tdvHShadXW1mrOnDnyer09Xntf1ZFvVX2KguHYvqZ9waHK4pjHXi57SD5ncns+yf0ZyWdHxysXsUho+fj1r3+tEydOKD8/P3Ksvb1d999/v9atW6f333+/0218Pp98Pl+n416vN64vYrzjnSYY9ijY7pzy0ZO9cPseks/Z3J5Pcn9G8iV//lgltHzcdtttKioqijpWXFys2267TUuWLEnkVAAAwKHiLh9nz55VU1NT5Hpzc7MaGhqUmZmp/Px8DR06NGq81+tVdna2Ro0a9cVXCwAAHC/u8lFfX69Zs2ZFrne8X6OsrEwbNmxI2MIAAIA7xV0+Zs6cqXg+INPV+zwAAMDli7/tAgAArKJ8AAAAqygfAADAKsoHAACwivIBAACsonwAAACrKB8AAMAqygcAALCK8gEAAKyifAAAAKsoHwAAwCrKBwAAsIryAQAArKJ8AAAAqygfAADAKsoHAACwivIBAACsonwAAACrKB8AAMAqygcAALCK8gEAAKyifAAAAKsoHwAAwCrKBwAAsIryAQAArKJ8AAAAqygfAADAKsoHAACwivIBAACsonwAAACrKB8AAMCquMvHjh07NH/+fOXm5srj8Wjz5s2Rc6FQSA8++KDGjx+vgQMHKjc3V3/1V3+lY8eOJXLNAADAweIuH+fOndPEiRO1fv36Tufa2tq0f/9+rVq1Svv379fzzz+vxsZGLViwICGLBQAAzpca7w1KSkpUUlLS5bmMjAzV1tZGHfvJT36iqVOn6siRI8rPz+/ZKgEAgGvEXT7i1draKo/Ho8GDB3d5PhgMKhgMRq4HAgFJF17CCYVC3d5/x5hYxjpRRy5fiunllcQnnv24XPaQfM7k9nyS+zOSz+46YuExxvT4p5rH41FNTY0WLlzY5fnz589r+vTpGj16tH7+8593OaayslJVVVWdjldXVys9Pb2nSwMAABa1tbWptLRUra2t8vv9lxybtPIRCoX0zW9+U7///e9VV1d30YV09cxHXl6eTp482e3iO+apra3VnDlz5PV6exqlz+rIt6o+RcGwp7eXE7NDlcUxj71c9pB8zuT2fJL7M5LPjkAgoKysrJjKR1JedgmFQvr2t7+t3/3ud3rttdcuuQifzyefz9fpuNfrjeuLGO94pwmGPQq2O6d89GQv3L6H5HM2t+eT3J+RfMmfP1YJLx8dxePw4cPatm2bhg4dmugpAACAg8VdPs6ePaumpqbI9ebmZjU0NCgzM1M5OTn6i7/4C+3fv18vvfSS2tvb1dLSIknKzMxUWlpa4lYOAAAcKe7yUV9fr1mzZkWuV1RUSJLKyspUWVmpF198UZI0adKkqNtt27ZNM2fO7PlKAQCAK8RdPmbOnKlLvUf1C7x/FQAAXAb42y4AAMAqygcAALCK8gEAAKyifAAAAKsoHwAAwCrKBwAAsIryAQAArKJ8AAAAqygfAADAKsoHAACwivIBAACsonwAAACrKB8AAMAqygcAALCK8gEAAKyifAAAAKsoHwAAwCrKBwAAsIryAQAArKJ8AAAAqygfAADAKsoHAACwivIBAACsonwAAACrKB8AAMAqygcAALCK8gEAAKyifAAAAKsoHwAAwCrKBwAAsCru8rFjxw7Nnz9fubm58ng82rx5c9R5Y4wefvhh5eTkaMCAASoqKtLhw4cTtV4AAOBwcZePc+fOaeLEiVq/fn2X55944gn9+Mc/1jPPPKM9e/Zo4MCBKi4u1vnz57/wYgEAgPOlxnuDkpISlZSUdHnOGKN169bpH/7hH3TLLbdIkv7t3/5Nw4cP1+bNm/Wd73zni60WAAA4Xtzl41Kam5vV0tKioqKiyLGMjAxNmzZNu3bt6rJ8BINBBYPByPVAICBJCoVCCoVC3c7ZMSaWsU7UkcuXYnp5JfGJZz8ulz0knzO5PZ/k/ozks7uOWHiMMT3+qebxeFRTU6OFCxdKkl5//XVNnz5dx44dU05OTmTct7/9bXk8Hj333HOd7qOyslJVVVWdjldXVys9Pb2nSwMAABa1tbWptLRUra2t8vv9lxyb0Gc+emLlypWqqKiIXA8EAsrLy9PcuXO7Xbx0oWnV1tZqzpw58nq9yVxqr+jIt6o+RcGwp7eXE7NDlcUxj71c9pB8zuT2fJL7M5LPjo5XLmKR0PKRnZ0tSTp+/HjUMx/Hjx/XpEmTuryNz+eTz+frdNzr9cb1RYx3vNMEwx4F251TPnqyF27fQ/I5m9vzSe7PSL7kzx+rhP6ej4KCAmVnZ2vr1q2RY4FAQHv27FFhYWEipwIAAA4V9zMfZ8+eVVNTU+R6c3OzGhoalJmZqfz8fC1btkyPPfaYrr/+ehUUFGjVqlXKzc2NvC8EAABc3uIuH/X19Zo1a1bkesf7NcrKyrRhwwY98MADOnfunO644w6dPn1aN910k7Zs2aL+/fsnbtUAAMCx4i4fM2fO1KU+IOPxePTII4/okUce+UILAwAA7sTfdgEAAFZRPgAAgFWUDwAAYBXlAwAAWEX5AAAAVlE+AACAVZQPAABgFeUDAABYRfkAAABWUT4AAIBVlA8AAGAV5QMAAFhF+QAAAFZRPgAAgFWUDwAAYBXlAwAAWEX5AAAAVlE+AACAVZQPAABgFeUDAABYRfkAAABWUT4AAIBVlA8AAGAV5QMAAFhF+QAAAFZRPgAAgFWUDwAAYBXlAwAAWEX5AAAAVlE+AACAVQkvH+3t7Vq1apUKCgo0YMAAXXvttXr00UdljEn0VAAAwIFSE32HP/jBD/T000/rX//1XzV27FjV19dryZIlysjI0L333pvo6QAAgMMkvHy8/vrruuWWWzRv3jxJ0siRI7Vx40a98cYbiZ4KAAA4UMJfdvnyl7+srVu36t1335Ukvfnmm9q5c6dKSkoSPRUAAHCghD/zsWLFCgUCAY0ePVr9+vVTe3u7Hn/8cS1evLjL8cFgUMFgMHI9EAhIkkKhkEKhULfzdYyJZawTdeTypTjrPTPx7Mflsofkcya355Pcn5F8dtcRC49J8DtBN23apOXLl+uHP/yhxo4dq4aGBi1btkxPPvmkysrKOo2vrKxUVVVVp+PV1dVKT09P5NIAAECStLW1qbS0VK2trfL7/Zccm/DykZeXpxUrVqi8vDxy7LHHHtO///u/67e//W2n8V0985GXl6eTJ092u3jpQtOqra3VnDlz5PV6ExOiD+nIt6o+RcGwp7eXE7NDlcUxj71c9pB8zuT2fJL7M5LPjkAgoKysrJjKR8Jfdmlra1NKSvRbSfr166dwONzleJ/PJ5/P1+m41+uN64sY73inCYY9CrY7p3z0ZC/cvofkcza355Pcn5F8yZ8/VgkvH/Pnz9fjjz+u/Px8jR07VgcOHNCTTz6p22+/PdFTAQAAB0p4+Xjqqae0atUq3X333Tpx4oRyc3P1t3/7t3r44YcTPRUAAHCghJePQYMGad26dVq3bl2i7xoAALgAf9sFAABYRfkAAABWUT4AAIBVlA8AAGAV5QMAAFhF+QAAAFZRPgAAgFWUDwAAYBXlAwAAWEX5AAAAVlE+AACAVZQPAABgFeUDAABYRfkAAABWUT4AAIBVlA8AAGAV5QMAAFhF+QAAAFZRPgAAgFWUDwAAYBXlAwAAWEX5AAAAVlE+AACAVZQPAABgFeUDAABYRfkAAABWUT4AAIBVlA8AAGAV5QMAAFhF+QAAAFZRPgAAgFVJKR8ffPCB/vIv/1JDhw7VgAEDNH78eNXX1ydjKgAA4DCpib7DP/7xj5o+fbpmzZqlV155RVdeeaUOHz6sIUOGJHoqAADgQAkvHz/4wQ+Ul5enZ599NnKsoKAg0dMAAACHSnj5ePHFF1VcXKxvfetb2r59u6666irdfffdWrp0aZfjg8GggsFg5HogEJAkhUIhhUKhbufrGBPLWCfqyOVLMb28kvjEsx+Xyx6Sz5ncnk9yf0by2V1HLDzGmIT+VOvfv78kqaKiQt/61re0d+9e3XfffXrmmWdUVlbWaXxlZaWqqqo6Ha+urlZ6enoilwYAAJKkra1NpaWlam1tld/vv+TYhJePtLQ0TZ48Wa+//nrk2L333qu9e/dq165dncZ39cxHXl6eTp482e3ipQtNq7a2VnPmzJHX601MiD6kI9+q+hQFw57eXk7MDlUWxzz2ctlD8jmT2/NJ7s9IPjsCgYCysrJiKh8Jf9klJydHY8aMiTr2pS99Sb/4xS+6HO/z+eTz+Tod93q9cX0R4x3vNMGwR8F255SPnuyF2/eQfM7m9nyS+zOSL/nzxyrhH7WdPn26Ghsbo469++67GjFiRKKnAgAADpTw8vF3f/d32r17t77//e+rqalJ1dXV+qd/+ieVl5cneioAAOBACS8fU6ZMUU1NjTZu3Khx48bp0Ucf1bp167R48eJETwUAABwo4e/5kKSvf/3r+vrXv56MuwYAAA7H33YBAABWUT4AAIBVlA8AAGAV5QMAAFhF+QAAAFZRPgAAgFWUDwAAYBXlAwAAWEX5AAAAVlE+AACAVZQPAABgFeUDAABYRfkAAABWUT4AAIBVqb29ANtGrni5t5cQF18/oyem9vYqAABIHJ75AAAAVlE+AACAVZQPAABgFeUDAABYRfkAAABWUT4AAIBVlA8AAGAV5QMAAFhF+QAAAFZRPgAAgFWUDwAAYBXlAwAAWEX5AAAAVlE+AACAVZQPAABgVdLLx9q1a+XxeLRs2bJkTwUAABwgqeVj7969+ulPf6oJEyYkcxoAAOAgSSsfZ8+e1eLFi/XP//zPGjJkSLKmAQAADpOarDsuLy/XvHnzVFRUpMcee+yi44LBoILBYOR6IBCQJIVCIYVCoW7n6RgTy1hJ8vUzMY3rK3wpJuq/ThHrfnx2bDy3cRLyOZvb80nuz0g+u+uIhccYk/Cfaps2bdLjjz+uvXv3qn///po5c6YmTZqkdevWdRpbWVmpqqqqTserq6uVnp6e6KUBAIAkaGtrU2lpqVpbW+X3+y85NuHl4+jRo5o8ebJqa2sj7/W4VPno6pmPvLw8nTx5stvFSxeaVm1trebMmSOv19vt+HGVr8Yepg/wpRg9OjmsVfUpCoY9vb2cpHB7xmTmO1RZnND764l4/w06jdvzSe7PSD47AoGAsrKyYiofCX/ZZd++fTpx4oRuuOGGyLH29nbt2LFDP/nJTxQMBtWvX7/IOZ/PJ5/P1+l+vF5vXF/EWMcH2535wy0Y9jh27bFye8Zk5OtL30jj/TfrNG7PJ7k/I/mSP3+sEl4+Zs+erbfeeivq2JIlSzR69Gg9+OCDUcUDAABcfhJePgYNGqRx48ZFHRs4cKCGDh3a6TgAALj88BtOAQCAVUn7qO1n1dXV2ZgGAAA4AM98AAAAqygfAADAKsoHAACwivIBAACsonwAAACrKB8AAMAqygcAALCK8gEAAKyifAAAAKsoHwAAwCrKBwAAsIryAQAArKJ8AAAAqygfAADAKsoHAACwivIBAACsonwAAACrKB8AAMAqygcAALCK8gEAAKyifAAAAKsoHwAAwCrKBwAAsIryAQAArKJ8AAAAqygfAADAKsoHAACwivIBAACsonwAAACrKB8AAMAqygcAALAq4eVjzZo1mjJligYNGqRhw4Zp4cKFamxsTPQ0AADAoRJePrZv367y8nLt3r1btbW1CoVCmjt3rs6dO5foqQAAgAOlJvoOt2zZEnV9w4YNGjZsmPbt26cZM2YkejoAAOAwCS8fn9fa2ipJyszM7PJ8MBhUMBiMXA8EApKkUCikUCjU7f13jIllrCT5+pmYxvUVvhQT9V83cnvGZOaL9XGfTPH+G3Qat+eT3J+RfHbXEQuPMSZp3/HD4bAWLFig06dPa+fOnV2OqaysVFVVVafj1dXVSk9PT9bSAABAArW1tam0tFStra3y+/2XHJvU8nHXXXfplVde0c6dO3X11Vd3OaarZz7y8vJ08uTJbhcvXWhatbW1mjNnjrxeb7fjx1W+GnuAPsCXYvTo5LBW1acoGPb09nKSwu0ZyedsHfli/R7jRPF+H3Ua8tkRCASUlZUVU/lI2ssu99xzj1566SXt2LHjosVDknw+n3w+X6fjXq83ri9irOOD7c785hgMexy79li5PSP5nC3e70lO5PaM5Ev+/LFKePkwxui73/2uampqVFdXp4KCgkRPAQAAHCzh5aO8vFzV1dV64YUXNGjQILW0tEiSMjIyNGDAgERPBwAAHCbhv+fj6aefVmtrq2bOnKmcnJzI5bnnnkv0VAAAwIGS8rILAADAxfC3XQAAgFWUDwAAYBXlAwAAWEX5AAAAVlE+AACAVZQPAABgFeUDAABYRfkAAABWUT4AAIBVlA8AAGAV5QMAAFhF+QAAAFZRPgAAgFWUDwAAYFVqby8AAJxgXOWrCrZ7ensZcXl/7bzeXkLcRq54OeH36etn9MRUZ+5hLHqSr7cfGzzzAQAArKJ8AAAAqygfAADAKsoHAACwivIBAACsonwAAACrKB8AAMAqygcAALCK8gEAAKyifAAAAKsoHwAAwCrKBwAAsIryAQAArKJ8AAAAqygfAADAqqSVj/Xr12vkyJHq37+/pk2bpjfeeCNZUwEAAAdJSvl47rnnVFFRodWrV2v//v2aOHGiiouLdeLEiWRMBwAAHCQp5ePJJ5/U0qVLtWTJEo0ZM0bPPPOM0tPT9bOf/SwZ0wEAAAdJTfQdfvLJJ9q3b59WrlwZOZaSkqKioiLt2rWr0/hgMKhgMBi53traKkk6deqUQqFQt/OFQiG1tbXpo48+ktfr7XZ86qfnYonRZ6SGjdrawkoNpag97Ont5SSF2zOSz9mcnO+jjz6KaVy830eTKRnfo528h7HoSb5YHxvxOHPmjCTJGNP9YJNgH3zwgZFkXn/99ajjy5cvN1OnTu00fvXq1UYSFy5cuHDhwsUFl6NHj3bbFRL+zEe8Vq5cqYqKisj1cDisU6dOaejQofJ4um9wgUBAeXl5Onr0qPx+fzKX2ivcnk9yf0byOZvb80nuz0g+O4wxOnPmjHJzc7sdm/DykZWVpX79+un48eNRx48fP67s7OxO430+n3w+X9SxwYMHxz2v3+935YOqg9vzSe7PSD5nc3s+yf0ZyZd8GRkZMY1L+BtO09LSdOONN2rr1q2RY+FwWFu3blVhYWGipwMAAA6TlJddKioqVFZWpsmTJ2vq1Klat26dzp07pyVLliRjOgAA4CBJKR+LFi3SH/7wBz388MNqaWnRpEmTtGXLFg0fPjzhc/l8Pq1evbrTSzdu4fZ8kvszks/Z3J5Pcn9G8vU9HmNi+UwMAABAYvC3XQAAgFWUDwAAYBXlAwAAWEX5AAAAVjm6fKxfv14jR45U//79NW3aNL3xxhu9vaSYVFZWyuPxRF1Gjx4dOX/+/HmVl5dr6NChuuKKK/TNb36z0y9tO3LkiObNm6f09HQNGzZMy5cv16effmo7SsSOHTs0f/585ebmyuPxaPPmzVHnjTF6+OGHlZOTowEDBqioqEiHDx+OGnPq1CktXrxYfr9fgwcP1t/8zd/o7NmzUWMOHjyor3zlK+rfv7/y8vL0xBNPJDuapO7z/fVf/3WnPb355pujxvTlfGvWrNGUKVM0aNAgDRs2TAsXLlRjY2PUmEQ9Luvq6nTDDTfI5/Ppuuuu04YNG5IdL6Z8M2fO7LSHd955Z9SYvprv6aef1oQJEyK/ZKqwsFCvvPJK5LyT965DdxmdvH+ft3btWnk8Hi1btixyzA17GCUhf9ClF2zatMmkpaWZn/3sZ+Y3v/mNWbp0qRk8eLA5fvx4by+tW6tXrzZjx441H374YeTyhz/8IXL+zjvvNHl5eWbr1q2mvr7e/Nmf/Zn58pe/HDn/6aefmnHjxpmioiJz4MAB88tf/tJkZWWZlStX9kYcY4wxv/zlL833vvc98/zzzxtJpqamJur82rVrTUZGhtm8ebN58803zYIFC0xBQYH5+OOPI2NuvvlmM3HiRLN7927z61//2lx33XXm1ltvjZxvbW01w4cPN4sXLzaHDh0yGzduNAMGDDA//elPez1fWVmZufnmm6P29NSpU1Fj+nK+4uJi8+yzz5pDhw6ZhoYG87Wvfc3k5+ebs2fPRsYk4nH5f//3fyY9Pd1UVFSYt99+2zz11FOmX79+ZsuWLb2e76tf/apZunRp1B62trY6It+LL75oXn75ZfPuu++axsZG89BDDxmv12sOHTpkjHH23sWa0cn791lvvPGGGTlypJkwYYK57777IsfdsIef5djyMXXqVFNeXh653t7ebnJzc82aNWt6cVWxWb16tZk4cWKX506fPm28Xq/5z//8z8ixd955x0gyu3btMsZc+EGYkpJiWlpaImOefvpp4/f7TTAYTOraY/H5H87hcNhkZ2ebH/7wh5Fjp0+fNj6fz2zcuNEYY8zbb79tJJm9e/dGxrzyyivG4/GYDz74wBhjzD/+4z+aIUOGRGV88MEHzahRo5KcKNrFysctt9xy0ds4KZ8xxpw4ccJIMtu3bzfGJO5x+cADD5ixY8dGzbVo0SJTXFyc7EhRPp/PmAs/vD77zf7znJTPGGOGDBli/uVf/sV1e/dZHRmNccf+nTlzxlx//fWmtrY2Ko8b99CRL7t88skn2rdvn4qKiiLHUlJSVFRUpF27dvXiymJ3+PBh5ebm6pprrtHixYt15MgRSdK+ffsUCoWiso0ePVr5+fmRbLt27dL48eOjfmlbcXGxAoGAfvOb39gNEoPm5ma1tLREZcrIyNC0adOiMg0ePFiTJ0+OjCkqKlJKSor27NkTGTNjxgylpaVFxhQXF6uxsVF//OMfLaW5uLq6Og0bNkyjRo3SXXfdFfUnq52Wr7W1VZKUmZkpKXGPy127dkXdR8cY2/9uP5+vw89//nNlZWVp3LhxWrlypdra2iLnnJKvvb1dmzZt0rlz51RYWOi6vZM6Z+zg9P0rLy/XvHnzOq3BjXvY63/VtidOnjyp9vb2Tr8xdfjw4frtb3/bS6uK3bRp07RhwwaNGjVKH374oaqqqvSVr3xFhw4dUktLi9LS0jr9cb3hw4erpaVFktTS0tJl9o5zfU3Hmrpa82czDRs2LOp8amqqMjMzo8YUFBR0uo+Oc0OGDEnK+mNx880368///M9VUFCg9957Tw899JBKSkq0a9cu9evXz1H5wuGwli1bpunTp2vcuHGR+RPxuLzYmEAgoI8//lgDBgxIRqQoXeWTpNLSUo0YMUK5ubk6ePCgHnzwQTU2Nur555+/5No7zl1qjI18b731lgoLC3X+/HldccUVqqmp0ZgxY9TQ0OCavbtYRsn5+7dp0ybt379fe/fu7XTOTf/+OjiyfDhdSUlJ5P8nTJigadOmacSIEfqP//gPq5uPxPnOd74T+f/x48drwoQJuvbaa1VXV6fZs2f34sriV15erkOHDmnnzp29vZSkuFi+O+64I/L/48ePV05OjmbPnq333ntP1157re1lxm3UqFFqaGhQa2ur/uu//ktlZWXavn17by8roS6WccyYMY7ev6NHj+q+++5TbW2t+vfv39vLscKRL7tkZWWpX79+nd7pe/z4cWVnZ/fSqnpu8ODB+pM/+RM1NTUpOztbn3zyiU6fPh015rPZsrOzu8zeca6v6VjTpfYrOztbJ06ciDr/6aef6tSpU47Mfc011ygrK0tNTU2SnJPvnnvu0UsvvaRt27bp6quvjhxP1OPyYmP8fr+V4n2xfF2ZNm2aJEXtYV/Ol5aWpuuuu0433nij1qxZo4kTJ+pHP/qRa/ZOunjGrjhp//bt26cTJ07ohhtuUGpqqlJTU7V9+3b9+Mc/VmpqqoYPH+6aPezgyPKRlpamG2+8UVu3bo0cC4fD2rp1a9Trf05x9uxZvffee8rJydGNN94or9cbla2xsVFHjhyJZCssLNRbb70V9cOstrZWfr8/8hRkX1JQUKDs7OyoTIFAQHv27InKdPr0ae3bty8y5rXXXlM4HI58EyksLNSOHTsUCoUiY2prazVq1KhefcmlK7///e/10UcfKScnR1Lfz2eM0T333KOamhq99tprnV7+SdTjsrCwMOo+OsYk+99td/m60tDQIElRe9hX83UlHA4rGAw6fu8upSNjV5y0f7Nnz9Zbb72lhoaGyGXy5MlavHhx5P9dt4fW3+KaIJs2bTI+n89s2LDBvP322+aOO+4wgwcPjnqnb191//33m7q6OtPc3Gz+93//1xQVFZmsrCxz4sQJY8yFj1Tl5+eb1157zdTX15vCwkJTWFgYuX3HR6rmzp1rGhoazJYtW8yVV17Zqx+1PXPmjDlw4IA5cOCAkWSefPJJc+DAAfO73/3OGHPho7aDBw82L7zwgjl48KC55ZZbuvyo7Z/+6Z+aPXv2mJ07d5rrr78+6qOop0+fNsOHDze33XabOXTokNm0aZNJT0+38lHUS+U7c+aM+fu//3uza9cu09zcbP7nf/7H3HDDDeb6668358+fd0S+u+66y2RkZJi6urqojyq2tbVFxiTicdnxUb/ly5ebd955x6xfv97KR/26y9fU1GQeeeQRU19fb5qbm80LL7xgrrnmGjNjxgxH5FuxYoXZvn27aW5uNgcPHjQrVqwwHo/H/OpXvzLGOHvvYsno9P3ryuc/veOGPfwsx5YPY4x56qmnTH5+vklLSzNTp041u3fv7u0lxWTRokUmJyfHpKWlmauuusosWrTINDU1Rc5//PHH5u677zZDhgwx6enp5hvf+Ib58MMPo+7j/fffNyUlJWbAgAEmKyvL3H///SYUCtmOErFt2zYjqdOlrKzMGHPh47arVq0yw4cPNz6fz8yePds0NjZG3cdHH31kbr31VnPFFVcYv99vlixZYs6cORM15s033zQ33XST8fl85qqrrjJr167t9XxtbW1m7ty55sorrzRer9eMGDHCLF26tFMR7sv5usomyTz77LORMYl6XG7bts1MmjTJpKWlmWuuuSZqjt7Kd+TIETNjxgyTmZlpfD6fue6668zy5cujfk9EX853++23mxEjRpi0tDRz5ZVXmtmzZ0eKhzHO3rsOl8ro9P3ryufLhxv28LM8xhhj73kWAABwuXPkez4AAIBzUT4AAIBVlA8AAGAV5QMAAFhF+QAAAFZRPgAAgFWUDwAAYBXlAwAAWEX5AAAAVlE+AACAVZQPAABgFeUDAABY9f8Ll0KRvvnv2QAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import tiktoken\n",
    "\n",
    "# Load the cl100k_base tokenizer which is designed to work with the ada-002 model\n",
    "tokenizer = tiktoken.get_encoding(\"cl100k_base\")\n",
    "\n",
    "df = pd.read_csv('processed/scraped.csv', index_col=0)\n",
    "df.columns = ['title', 'text']\n",
    "\n",
    "# Tokenize the text and save the number of tokens to a new column\n",
    "df['n_tokens'] = df.text.apply(lambda x: len(tokenizer.encode(x)))\n",
    "\n",
    "# Visualize the distribution of the number of tokens per row using a histogram\n",
    "df.n_tokens.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_tokens = 500\n",
    "\n",
    "# Function to split the text into chunks of a maximum number of tokens\n",
    "def split_into_many(text, max_tokens = max_tokens):\n",
    "\n",
    "    # Split the text into sentences\n",
    "    sentences = text.split('. ')\n",
    "\n",
    "    # Get the number of tokens for each sentence\n",
    "    n_tokens = [len(tokenizer.encode(\" \" + sentence)) for sentence in sentences]\n",
    "    \n",
    "    chunks = []\n",
    "    tokens_so_far = 0\n",
    "    chunk = []\n",
    "\n",
    "    # Loop through the sentences and tokens joined together in a tuple\n",
    "    for sentence, token in zip(sentences, n_tokens):\n",
    "\n",
    "        # If the number of tokens so far plus the number of tokens in the current sentence is greater \n",
    "        # than the max number of tokens, then add the chunk to the list of chunks and reset\n",
    "        # the chunk and tokens so far\n",
    "        if tokens_so_far + token > max_tokens:\n",
    "            chunks.append(\". \".join(chunk) + \".\")\n",
    "            chunk = []\n",
    "            tokens_so_far = 0\n",
    "\n",
    "        # If the number of tokens in the current sentence is greater than the max number of \n",
    "        # tokens, go to the next sentence\n",
    "        if token > max_tokens:\n",
    "            continue\n",
    "\n",
    "        # Otherwise, add the sentence to the chunk and add the number of tokens to the total\n",
    "        chunk.append(sentence)\n",
    "        tokens_so_far += token + 1\n",
    "\n",
    "    return chunks\n",
    "    \n",
    "\n",
    "shortened = []\n",
    "\n",
    "# Loop through the dataframe\n",
    "for row in df.iterrows():\n",
    "\n",
    "    # If the text is None, go to the next row\n",
    "    if row[1]['text'] is None:\n",
    "        continue\n",
    "\n",
    "    # If the number of tokens is greater than the max number of tokens, split the text into chunks\n",
    "    if row[1]['n_tokens'] > max_tokens:\n",
    "        shortened += split_into_many(row[1]['text'])\n",
    "    \n",
    "    # Otherwise, add the text to the list of shortened texts\n",
    "    else:\n",
    "        shortened.append( row[1]['text'] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot: >"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAi30lEQVR4nO3da3CU5f3/8c8GNhsibCJEcpAE0/GAiGAbBFZtqxhIqWczHumUUkZHjVaIrZVWhVDboJ166kSsLeL4IMViC4qVQyZqrDVBiFJBa0SHFiskiJosELOsyfV7wJ/9uwY1C7vfZeP7NcMMe993rr32y46+Z7ObeJxzTgAAAEbSkr0BAADw9UJ8AAAAU8QHAAAwRXwAAABTxAcAADBFfAAAAFPEBwAAMEV8AAAAUwOTvYHP6+np0fbt2zVkyBB5PJ5kbwcAAPSBc067d+9WQUGB0tK+/LWNIy4+tm/frsLCwmRvAwAAHIL33ntPI0aM+NJrjrj4GDJkiKT9m/f7/XFbNxwOa+3atZo6daq8Xm/c1kU05myDOdth1jaYs41EzjkYDKqwsDDy//Evc8TFx4Fvtfj9/rjHR2Zmpvx+P0/sBGLONpizHWZtgznbsJhzX94ywRtOAQCAKeIDAACYIj4AAIAp4gMAAJiKOT7ef/99/eAHP9CwYcM0aNAgnXrqqdqwYUPkvHNOd955p/Lz8zVo0CCVlpZqy5Ytcd00AABIXTHFx8cff6wzzzxTXq9Xq1at0ptvvqnf/e53OvrooyPX3HPPPXrwwQf18MMPa926dTrqqKNUVlamrq6uuG8eAACknpg+anv33XersLBQS5YsiRwrLi6O/N05p/vvv1+33367LrroIknS448/rtzcXK1YsUJXXnllnLYNAABSVUzx8fTTT6usrEyXXXaZGhoadOyxx+qGG27QNddcI0naunWrWltbVVpaGvmarKwsTZw4UY2NjQeNj1AopFAoFLkdDAYl7f8scjgcPqQHdTAH1ornmuiNOdtgznaYtQ3mbCORc45lTY9zzvX14oyMDElSZWWlLrvsMq1fv14333yzHn74Yc2YMUMvv/yyzjzzTG3fvl35+fmRr7v88svl8Xj0xBNP9Fpz/vz5qqqq6nW8trZWmZmZfX4gAAAgeTo7O3X11Vero6PjK39IaEzxkZ6ervHjx+vll1+OHPvJT36i9evXq7Gx8ZDi42CvfBQWFmrXrl1x/wmndXV1mjJlCj89L4GYsw3mbIdZ22DONhI552AwqJycnD7FR0zfdsnPz9fo0aOjjp188sn661//KknKy8uTJLW1tUXFR1tbm0477bSDrunz+eTz+Xod93q9CXkCJmpdRGPONpizHWZtgznbSMScY1kvpk+7nHnmmWppaYk69vbbb2vkyJGS9r/5NC8vT/X19ZHzwWBQ69atUyAQiOWuAABAPxXTKx9z5szRGWecod/85je6/PLL9corr+iRRx7RI488Imn/L5OZPXu27rrrLp1wwgkqLi7WHXfcoYKCAl188cWJ2D8AAEgxMcXH6aefruXLl2vu3LlasGCBiouLdf/992v69OmRa2699Vbt3btX1157rdrb23XWWWdp9erVkTerAgCAr7eY4kOSzj//fJ1//vlfeN7j8WjBggVasGDBYW0MAIBUcNxtf0/2FvrMN8DpngnJ3gW/2wUAABgjPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJiKKT7mz58vj8cT9WfUqFGR811dXaqoqNCwYcM0ePBglZeXq62tLe6bBgAAqSvmVz5OOeUU7dixI/LnpZdeipybM2eOVq5cqWXLlqmhoUHbt2/XpZdeGtcNAwCA1DYw5i8YOFB5eXm9jnd0dGjx4sWqra3V5MmTJUlLlizRySefrKamJk2aNOnwdwsAAFJezPGxZcsWFRQUKCMjQ4FAQNXV1SoqKlJzc7PC4bBKS0sj144aNUpFRUVqbGz8wvgIhUIKhUKR28FgUJIUDocVDodj3d4XOrBWPNdEb8zZBnO2w6xtpPKcfQNcsrfQZ760/XtNxJxjWdPjnOvz1FatWqU9e/bopJNO0o4dO1RVVaX3339fmzdv1sqVKzVz5syokJCkCRMm6JxzztHdd9990DXnz5+vqqqqXsdra2uVmZnZ5wcCAACSp7OzU1dffbU6Ojrk9/u/9NqY4uPz2tvbNXLkSN17770aNGjQIcXHwV75KCws1K5du75y87EIh8Oqq6vTlClT5PV647YuojFnG8zZDrO2kcpzHjN/TbK30Ge+NKdfje9JyJyDwaBycnL6FB8xf9vls7Kzs3XiiSfqnXfe0ZQpU7Rv3z61t7crOzs7ck1bW9tB3yNygM/nk8/n63Xc6/Um5AmYqHURjTnbYM52mLWNVJxzqNuT7C3ELBFzjmW9w/o5H3v27NG7776r/Px8lZSUyOv1qr6+PnK+paVF27ZtUyAQOJy7AQAA/UhMr3z89Kc/1QUXXKCRI0dq+/btmjdvngYMGKCrrrpKWVlZmjVrliorKzV06FD5/X7ddNNNCgQCfNIFAABExBQf//vf/3TVVVfpww8/1DHHHKOzzjpLTU1NOuaYYyRJ9913n9LS0lReXq5QKKSysjI99NBDCdk4AABITTHFx9KlS7/0fEZGhmpqalRTU3NYmwIAAP0Xv9sFAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAqcOKj4ULF8rj8Wj27NmRY11dXaqoqNCwYcM0ePBglZeXq62t7XD3CQAA+olDjo/169frD3/4g8aOHRt1fM6cOVq5cqWWLVumhoYGbd++XZdeeulhbxQAAPQPhxQfe/bs0fTp0/XHP/5RRx99dOR4R0eHFi9erHvvvVeTJ09WSUmJlixZopdffllNTU1x2zQAAEhdhxQfFRUVOu+881RaWhp1vLm5WeFwOOr4qFGjVFRUpMbGxsPbKQAA6BcGxvoFS5cu1auvvqr169f3Otfa2qr09HRlZ2dHHc/NzVVra+tB1wuFQgqFQpHbwWBQkhQOhxUOh2Pd3hc6sFY810RvzNkGc7bDrG2k8px9A1yyt9BnvrT9e03EnGNZM6b4eO+993TzzTerrq5OGRkZMW/sYKqrq1VVVdXr+Nq1a5WZmRmX+/isurq6uK+J3pizDeZsh1nbSMU53zMh2TuIXSLm3NnZ2edrPc65PifbihUrdMkll2jAgAGRY93d3fJ4PEpLS9OaNWtUWlqqjz/+OOrVj5EjR2r27NmaM2dOrzUP9spHYWGhdu3aJb/f3+cH8lXC4bDq6uo0ZcoUeb3euK2LaMzZBnO2w6xtpPKcx8xfk+wt9JkvzelX43sSMudgMKicnBx1dHR85f+/Y3rl49xzz9WmTZuijs2cOVOjRo3Sz3/+cxUWFsrr9aq+vl7l5eWSpJaWFm3btk2BQOCga/p8Pvl8vl7HvV5vQp6AiVoX0ZizDeZsh1nbSMU5h7o9yd5CzBIx51jWiyk+hgwZojFjxkQdO+qoozRs2LDI8VmzZqmyslJDhw6V3+/XTTfdpEAgoEmTJsVyVwAAoJ+K+Q2nX+W+++5TWlqaysvLFQqFVFZWpoceeijedwMAAFLUYcfHCy+8EHU7IyNDNTU1qqmpOdylAQBAP8TvdgEAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKZiio9FixZp7Nix8vv98vv9CgQCWrVqVeR8V1eXKioqNGzYMA0ePFjl5eVqa2uL+6YBAEDqiik+RowYoYULF6q5uVkbNmzQ5MmTddFFF+mNN96QJM2ZM0crV67UsmXL1NDQoO3bt+vSSy9NyMYBAEBqGhjLxRdccEHU7V//+tdatGiRmpqaNGLECC1evFi1tbWaPHmyJGnJkiU6+eST1dTUpEmTJsVv1wAAIGXFFB+f1d3drWXLlmnv3r0KBAJqbm5WOBxWaWlp5JpRo0apqKhIjY2NXxgfoVBIoVAocjsYDEqSwuGwwuHwoW6vlwNrxXNN9MacbTBnO8zaRirP2TfAJXsLfeZL27/XRMw5ljVjjo9NmzYpEAioq6tLgwcP1vLlyzV69Ght3LhR6enpys7Ojro+NzdXra2tX7hedXW1qqqqeh1fu3atMjMzY93eV6qrq4v7muiNOdtgznaYtY1UnPM9E5K9g9glYs6dnZ19vjbm+DjppJO0ceNGdXR06Mknn9SMGTPU0NAQ6zIRc+fOVWVlZeR2MBhUYWGhpk6dKr/ff8jrfl44HFZdXZ2mTJkir9cbt3URjTnbYM52mLWNVJ7zmPlrkr2FPvOlOf1qfE9C5nzgOxd9EXN8pKen6/jjj5cklZSUaP369XrggQd0xRVXaN++fWpvb4969aOtrU15eXlfuJ7P55PP5+t13Ov1JuQJmKh1EY0522DOdpi1jVScc6jbk+wtxCwRc45lvcP+OR89PT0KhUIqKSmR1+tVfX195FxLS4u2bdumQCBwuHcDAAD6iZhe+Zg7d66mTZumoqIi7d69W7W1tXrhhRe0Zs0aZWVladasWaqsrNTQoUPl9/t10003KRAI8EkXAAAQEVN87Ny5Uz/84Q+1Y8cOZWVlaezYsVqzZo2mTJkiSbrvvvuUlpam8vJyhUIhlZWV6aGHHkrIxgEAQGqKKT4WL178peczMjJUU1Ojmpqaw9oUAADov/jdLgAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwFRM8VFdXa3TTz9dQ4YM0fDhw3XxxRerpaUl6pquri5VVFRo2LBhGjx4sMrLy9XW1hbXTQMAgNQVU3w0NDSooqJCTU1NqqurUzgc1tSpU7V3797INXPmzNHKlSu1bNkyNTQ0aPv27br00kvjvnEAAJCaBsZy8erVq6NuP/bYYxo+fLiam5v1ne98Rx0dHVq8eLFqa2s1efJkSdKSJUt08sknq6mpSZMmTYrfzgEAQEqKKT4+r6OjQ5I0dOhQSVJzc7PC4bBKS0sj14waNUpFRUVqbGw8aHyEQiGFQqHI7WAwKEkKh8MKh8OHs70oB9aK55rojTnbYM52mLWNVJ6zb4BL9hb6zJe2f6+JmHMsa3qcc4c0tZ6eHl144YVqb2/XSy+9JEmqra3VzJkzo2JCkiZMmKBzzjlHd999d6915s+fr6qqql7Ha2trlZmZeShbAwAAxjo7O3X11Vero6NDfr//S6895Fc+KioqtHnz5kh4HKq5c+eqsrIycjsYDKqwsFBTp079ys3HIhwOq66uTlOmTJHX643buojGnG0wZzvM2kYqz3nM/DXJ3kKf+dKcfjW+JyFzPvCdi744pPi48cYb9cwzz+jFF1/UiBEjIsfz8vK0b98+tbe3Kzs7O3K8ra1NeXl5B13L5/PJ5/P1Ou71ehPyBEzUuojGnG0wZzvM2kYqzjnU7Un2FmKWiDnHsl5Mn3ZxzunGG2/U8uXL9dxzz6m4uDjqfElJibxer+rr6yPHWlpatG3bNgUCgVjuCgAA9FMxvfJRUVGh2tpaPfXUUxoyZIhaW1slSVlZWRo0aJCysrI0a9YsVVZWaujQofL7/brpppsUCAT4pAsAAJAUY3wsWrRIknT22WdHHV+yZIl+9KMfSZLuu+8+paWlqby8XKFQSGVlZXrooYfislkAAJD6YoqPvnwwJiMjQzU1NaqpqTnkTQEAgP6L3+0CAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAVMzx8eKLL+qCCy5QQUGBPB6PVqxYEXXeOac777xT+fn5GjRokEpLS7Vly5Z47RcAAKS4mONj7969GjdunGpqag56/p577tGDDz6ohx9+WOvWrdNRRx2lsrIydXV1HfZmAQBA6hsY6xdMmzZN06ZNO+g555zuv/9+3X777broooskSY8//rhyc3O1YsUKXXnllYe3WwAAkPLi+p6PrVu3qrW1VaWlpZFjWVlZmjhxohobG+N5VwAAIEXF/MrHl2ltbZUk5ebmRh3Pzc2NnPu8UCikUCgUuR0MBiVJ4XBY4XA4bns7sFY810RvzNkGc7bDrG2k8px9A1yyt9BnvrT9e03EnGNZM67xcSiqq6tVVVXV6/jatWuVmZkZ9/urq6uL+5rojTnbYM52mLWNVJzzPROSvYPYJWLOnZ2dfb42rvGRl5cnSWpra1N+fn7keFtbm0477bSDfs3cuXNVWVkZuR0MBlVYWKipU6fK7/fHbW/hcFh1dXWaMmWKvF5v3NZFNOZsgznbYdY2UnnOY+avSfYW+syX5vSr8T0JmfOB71z0RVzjo7i4WHl5eaqvr4/ERjAY1Lp163T99dcf9Gt8Pp98Pl+v416vNyFPwESti2jM2QZztsOsbaTinEPdnmRvIWaJmHMs68UcH3v27NE777wTub1161Zt3LhRQ4cOVVFRkWbPnq277rpLJ5xwgoqLi3XHHXeooKBAF198cax3BQAA+qGY42PDhg0655xzIrcPfMtkxowZeuyxx3Trrbdq7969uvbaa9Xe3q6zzjpLq1evVkZGRvx2DQAAUlbM8XH22WfLuS9+Z6/H49GCBQu0YMGCw9oYAADon/jdLgAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMDUwGRvAACAA8bMX6NQtyfZ20CC8coHAAAwRXwAAABTxAcAADBFfAAAAFPEBwAAMMWnXYD/57jb/p7sLfSZb4DTPRNS85MB/1l4XrK3ACDJeOUDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmPrafdolFT8dkEoOfAoDQPKl4ie48PXAKx8AAMBUwuKjpqZGxx13nDIyMjRx4kS98soriborAACQQhISH0888YQqKys1b948vfrqqxo3bpzKysq0c+fORNwdAABIIQmJj3vvvVfXXHONZs6cqdGjR+vhhx9WZmamHn300UTcHQAASCFxf8Ppvn371NzcrLlz50aOpaWlqbS0VI2Njb2uD4VCCoVCkdsdHR2SpI8++kjhcDhu+wqHw+rs7NTAcJq6e3jDaaIM7HHq7OzRhx9+KK/Xm+ztxGTgp3uTvYU+OzDnVHw+f/jhh8neQkwO/LeD53RipfJzOpUk8r/Ru3fvliQ55756H3G9Z0m7du1Sd3e3cnNzo47n5ubqrbfe6nV9dXW1qqqqeh0vLi6O99Zg5Opkb+BrIlXnnPO7ZO8AR6pUfU6nmkTPeffu3crKyvrSa5L+Udu5c+eqsrIycrunp0cfffSRhg0bJo8nfvUbDAZVWFio9957T36/P27rIhpztsGc7TBrG8zZRiLn7JzT7t27VVBQ8JXXxj0+cnJyNGDAALW1tUUdb2trU15eXq/rfT6ffD5f1LHs7Ox4byvC7/fzxDbAnG0wZzvM2gZztpGoOX/VKx4HxP0Np+np6SopKVF9fX3kWE9Pj+rr6xUIBOJ9dwAAIMUk5NsulZWVmjFjhsaPH68JEybo/vvv1969ezVz5sxE3B0AAEghCYmPK664Qh988IHuvPNOtba26rTTTtPq1at7vQnVks/n07x583p9iwfxxZxtMGc7zNoGc7ZxpMzZ4/rymRgAAIA44Xe7AAAAU8QHAAAwRXwAAABTxAcAADD1tYiPmpoaHXfcccrIyNDEiRP1yiuvJHtLKefFF1/UBRdcoIKCAnk8Hq1YsSLqvHNOd955p/Lz8zVo0CCVlpZqy5YtUdd89NFHmj59uvx+v7KzszVr1izt2bPH8FEc2aqrq3X66adryJAhGj58uC6++GK1tLREXdPV1aWKigoNGzZMgwcPVnl5ea8f6Ldt2zadd955yszM1PDhw/Wzn/1Mn376qeVDOaItWrRIY8eOjfyQpUAgoFWrVkXOM+PEWLhwoTwej2bPnh05xqzjY/78+fJ4PFF/Ro0aFTl/RM7Z9XNLly516enp7tFHH3VvvPGGu+aaa1x2drZra2tL9tZSyrPPPut++ctfur/97W9Oklu+fHnU+YULF7qsrCy3YsUK969//ctdeOGFrri42H3yySeRa773ve+5cePGuaamJvePf/zDHX/88e6qq64yfiRHrrKyMrdkyRK3efNmt3HjRvf973/fFRUVuT179kSuue6661xhYaGrr693GzZscJMmTXJnnHFG5Pynn37qxowZ40pLS91rr73mnn32WZeTk+Pmzp2bjId0RHr66afd3//+d/f222+7lpYW94tf/MJ5vV63efNm5xwzToRXXnnFHXfccW7s2LHu5ptvjhxn1vExb948d8opp7gdO3ZE/nzwwQeR80finPt9fEyYMMFVVFREbnd3d7uCggJXXV2dxF2lts/HR09Pj8vLy3O//e1vI8fa29udz+dzf/7zn51zzr355ptOklu/fn3kmlWrVjmPx+Pef/99s72nkp07dzpJrqGhwTm3f6Zer9ctW7Yscs2///1vJ8k1NjY65/ZHYlpammttbY1cs2jRIuf3+10oFLJ9ACnk6KOPdn/605+YcQLs3r3bnXDCCa6urs5997vfjcQHs46fefPmuXHjxh303JE65379bZd9+/apublZpaWlkWNpaWkqLS1VY2NjEnfWv2zdulWtra1Rc87KytLEiRMjc25sbFR2drbGjx8fuaa0tFRpaWlat26d+Z5TQUdHhyRp6NChkqTm5maFw+GoOY8aNUpFRUVRcz711FOjfqBfWVmZgsGg3njjDcPdp4bu7m4tXbpUe/fuVSAQYMYJUFFRofPOOy9qphLP53jbsmWLCgoK9I1vfEPTp0/Xtm3bJB25c076b7VNpF27dqm7u7vXT1bNzc3VW2+9laRd9T+tra2SdNA5HzjX2tqq4cOHR50fOHCghg4dGrkG/19PT49mz56tM888U2PGjJG0f4bp6em9fvHi5+d8sH+HA+ew36ZNmxQIBNTV1aXBgwdr+fLlGj16tDZu3MiM42jp0qV69dVXtX79+l7neD7Hz8SJE/XYY4/ppJNO0o4dO1RVVaVvf/vb2rx58xE7534dH0Cqqqio0ObNm/XSSy8leyv90kknnaSNGzeqo6NDTz75pGbMmKGGhoZkb6tfee+993TzzTerrq5OGRkZyd5OvzZt2rTI38eOHauJEydq5MiR+stf/qJBgwYlcWdfrF9/2yUnJ0cDBgzo9a7etrY25eXlJWlX/c+BWX7ZnPPy8rRz586o859++qk++ugj/i0+58Ybb9Qzzzyj559/XiNGjIgcz8vL0759+9Te3h51/efnfLB/hwPnsF96erqOP/54lZSUqLq6WuPGjdMDDzzAjOOoublZO3fu1Le+9S0NHDhQAwcOVENDgx588EENHDhQubm5zDpBsrOzdeKJJ+qdd945Yp/T/To+0tPTVVJSovr6+sixnp4e1dfXKxAIJHFn/UtxcbHy8vKi5hwMBrVu3brInAOBgNrb29Xc3By55rnnnlNPT48mTpxovucjkXNON954o5YvX67nnntOxcXFUedLSkrk9Xqj5tzS0qJt27ZFzXnTpk1RoVdXVye/36/Ro0fbPJAU1NPTo1AoxIzj6Nxzz9WmTZu0cePGyJ/x48dr+vTpkb8z68TYs2eP3n33XeXn5x+5z+mEvI31CLJ06VLn8/ncY4895t5880137bXXuuzs7Kh39eKr7d6927322mvutddec5Lcvffe61577TX33//+1zm3/6O22dnZ7qmnnnKvv/66u+iiiw76UdtvfvObbt26de6ll15yJ5xwAh+1/Yzrr7/eZWVluRdeeCHqI3OdnZ2Ra6677jpXVFTknnvuObdhwwYXCARcIBCInD/wkbmpU6e6jRs3utWrV7tjjjmGjyZ+xm233eYaGhrc1q1b3euvv+5uu+025/F43Nq1a51zzDiRPvtpF+eYdbzccsst7oUXXnBbt251//znP11paanLyclxO3fudM4dmXPu9/HhnHO///3vXVFRkUtPT3cTJkxwTU1Nyd5Synn++eedpF5/ZsyY4Zzb/3HbO+64w+Xm5jqfz+fOPfdc19LSErXGhx9+6K666io3ePBg5/f73cyZM93u3buT8GiOTAebryS3ZMmSyDWffPKJu+GGG9zRRx/tMjMz3SWXXOJ27NgRtc5//vMfN23aNDdo0CCXk5PjbrnlFhcOh40fzZHrxz/+sRs5cqRLT093xxxzjDv33HMj4eEcM06kz8cHs46PK664wuXn57v09HR37LHHuiuuuMK98847kfNH4pw9zjmXmNdUAAAAeuvX7/kAAABHHuIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmPo/P1O22ImganIAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = pd.DataFrame(shortened, columns = ['text'])\n",
    "df['n_tokens'] = df.text.apply(lambda x: len(tokenizer.encode(x)))\n",
    "df.n_tokens.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>n_tokens</th>\n",
       "      <th>embeddings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>n.github.io Templates Fixer.html.   Fixer - So...</td>\n",
       "      <td>486</td>\n",
       "      <td>[0.014662585221230984, 0.012262040749192238, 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>n.github.io Inventory 009f Drugs.html.   Drugs...</td>\n",
       "      <td>487</td>\n",
       "      <td>[0.033510904759168625, 0.00927431508898735, 0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Each dose gives you the feeling of getting 10 ...</td>\n",
       "      <td>488</td>\n",
       "      <td>[-0.011265072040259838, 0.004851039499044418, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>n.github.io HackProg 011b Programming.html.   ...</td>\n",
       "      <td>501</td>\n",
       "      <td>[0.017100855708122253, -0.007077397312968969, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>If executing a program on your own device, it ...</td>\n",
       "      <td>479</td>\n",
       "      <td>[-0.016866354271769524, -0.0008413073373958468...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  n_tokens  \\\n",
       "0  n.github.io Templates Fixer.html.   Fixer - So...       486   \n",
       "1  n.github.io Inventory 009f Drugs.html.   Drugs...       487   \n",
       "2  Each dose gives you the feeling of getting 10 ...       488   \n",
       "3  n.github.io HackProg 011b Programming.html.   ...       501   \n",
       "4  If executing a program on your own device, it ...       479   \n",
       "\n",
       "                                          embeddings  \n",
       "0  [0.014662585221230984, 0.012262040749192238, 0...  \n",
       "1  [0.033510904759168625, 0.00927431508898735, 0....  \n",
       "2  [-0.011265072040259838, 0.004851039499044418, ...  \n",
       "3  [0.017100855708122253, -0.007077397312968969, ...  \n",
       "4  [-0.016866354271769524, -0.0008413073373958468...  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import openai\n",
    "\n",
    "openai.api_key_path = '/Users/joshuamccrowell/Documents/GitHub/config.ini'\n",
    "\n",
    "df['embeddings'] = df.text.apply(lambda x: openai.Embedding.create(input=x, engine='text-embedding-ada-002')['data'][0]['embedding'])\n",
    "df.to_csv('processed/embeddings.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "ename": "RateLimitError",
     "evalue": "Rate limit reached for default-global-with-image-limits in organization org-QkIqVS4vwPzxNzDo6pfbr06g on requests per min. Limit: 60 / min. Please try again in 1s. Contact support@openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRateLimitError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[27], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mopenai\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39membeddings_utils\u001b[39;00m \u001b[39mimport\u001b[39;00m distances_from_embeddings\n\u001b[0;32m----> 3\u001b[0m df[\u001b[39m'\u001b[39m\u001b[39membeddings\u001b[39m\u001b[39m'\u001b[39m] \u001b[39m=\u001b[39m df\u001b[39m.\u001b[39;49mtext\u001b[39m.\u001b[39;49mapply(\u001b[39mlambda\u001b[39;49;00m x: openai\u001b[39m.\u001b[39;49mEmbedding\u001b[39m.\u001b[39;49mcreate(\u001b[39minput\u001b[39;49m\u001b[39m=\u001b[39;49mx, engine\u001b[39m=\u001b[39;49m\u001b[39m'\u001b[39;49m\u001b[39mtext-embedding-ada-002\u001b[39;49m\u001b[39m'\u001b[39;49m)[\u001b[39m'\u001b[39;49m\u001b[39mdata\u001b[39;49m\u001b[39m'\u001b[39;49m][\u001b[39m0\u001b[39;49m][\u001b[39m'\u001b[39;49m\u001b[39membedding\u001b[39;49m\u001b[39m'\u001b[39;49m])\n\u001b[1;32m      5\u001b[0m df\u001b[39m.\u001b[39mto_csv(\u001b[39m'\u001b[39m\u001b[39mprocessed/embeddings.csv\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m      6\u001b[0m df\u001b[39m.\u001b[39mhead()\n",
      "File \u001b[0;32m~/Documents/GitHub/openai-cookbook/apps/web-crawl-q-and-a/env/lib/python3.9/site-packages/pandas/core/series.py:4771\u001b[0m, in \u001b[0;36mSeries.apply\u001b[0;34m(self, func, convert_dtype, args, **kwargs)\u001b[0m\n\u001b[1;32m   4661\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mapply\u001b[39m(\n\u001b[1;32m   4662\u001b[0m     \u001b[39mself\u001b[39m,\n\u001b[1;32m   4663\u001b[0m     func: AggFuncType,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   4666\u001b[0m     \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs,\n\u001b[1;32m   4667\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m DataFrame \u001b[39m|\u001b[39m Series:\n\u001b[1;32m   4668\u001b[0m \u001b[39m    \u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m   4669\u001b[0m \u001b[39m    Invoke function on values of Series.\u001b[39;00m\n\u001b[1;32m   4670\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   4769\u001b[0m \u001b[39m    dtype: float64\u001b[39;00m\n\u001b[1;32m   4770\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[0;32m-> 4771\u001b[0m     \u001b[39mreturn\u001b[39;00m SeriesApply(\u001b[39mself\u001b[39;49m, func, convert_dtype, args, kwargs)\u001b[39m.\u001b[39;49mapply()\n",
      "File \u001b[0;32m~/Documents/GitHub/openai-cookbook/apps/web-crawl-q-and-a/env/lib/python3.9/site-packages/pandas/core/apply.py:1105\u001b[0m, in \u001b[0;36mSeriesApply.apply\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1102\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mapply_str()\n\u001b[1;32m   1104\u001b[0m \u001b[39m# self.f is Callable\u001b[39;00m\n\u001b[0;32m-> 1105\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mapply_standard()\n",
      "File \u001b[0;32m~/Documents/GitHub/openai-cookbook/apps/web-crawl-q-and-a/env/lib/python3.9/site-packages/pandas/core/apply.py:1156\u001b[0m, in \u001b[0;36mSeriesApply.apply_standard\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1154\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m   1155\u001b[0m         values \u001b[39m=\u001b[39m obj\u001b[39m.\u001b[39mastype(\u001b[39mobject\u001b[39m)\u001b[39m.\u001b[39m_values\n\u001b[0;32m-> 1156\u001b[0m         mapped \u001b[39m=\u001b[39m lib\u001b[39m.\u001b[39;49mmap_infer(\n\u001b[1;32m   1157\u001b[0m             values,\n\u001b[1;32m   1158\u001b[0m             f,\n\u001b[1;32m   1159\u001b[0m             convert\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mconvert_dtype,\n\u001b[1;32m   1160\u001b[0m         )\n\u001b[1;32m   1162\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(mapped) \u001b[39mand\u001b[39;00m \u001b[39misinstance\u001b[39m(mapped[\u001b[39m0\u001b[39m], ABCSeries):\n\u001b[1;32m   1163\u001b[0m     \u001b[39m# GH#43986 Need to do list(mapped) in order to get treated as nested\u001b[39;00m\n\u001b[1;32m   1164\u001b[0m     \u001b[39m#  See also GH#25959 regarding EA support\u001b[39;00m\n\u001b[1;32m   1165\u001b[0m     \u001b[39mreturn\u001b[39;00m obj\u001b[39m.\u001b[39m_constructor_expanddim(\u001b[39mlist\u001b[39m(mapped), index\u001b[39m=\u001b[39mobj\u001b[39m.\u001b[39mindex)\n",
      "File \u001b[0;32m~/Documents/GitHub/openai-cookbook/apps/web-crawl-q-and-a/env/lib/python3.9/site-packages/pandas/_libs/lib.pyx:2918\u001b[0m, in \u001b[0;36mpandas._libs.lib.map_infer\u001b[0;34m()\u001b[0m\n",
      "Cell \u001b[0;32mIn[27], line 3\u001b[0m, in \u001b[0;36m<lambda>\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mopenai\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39membeddings_utils\u001b[39;00m \u001b[39mimport\u001b[39;00m distances_from_embeddings\n\u001b[0;32m----> 3\u001b[0m df[\u001b[39m'\u001b[39m\u001b[39membeddings\u001b[39m\u001b[39m'\u001b[39m] \u001b[39m=\u001b[39m df\u001b[39m.\u001b[39mtext\u001b[39m.\u001b[39mapply(\u001b[39mlambda\u001b[39;00m x: openai\u001b[39m.\u001b[39;49mEmbedding\u001b[39m.\u001b[39;49mcreate(\u001b[39minput\u001b[39;49m\u001b[39m=\u001b[39;49mx, engine\u001b[39m=\u001b[39;49m\u001b[39m'\u001b[39;49m\u001b[39mtext-embedding-ada-002\u001b[39;49m\u001b[39m'\u001b[39;49m)[\u001b[39m'\u001b[39m\u001b[39mdata\u001b[39m\u001b[39m'\u001b[39m][\u001b[39m0\u001b[39m][\u001b[39m'\u001b[39m\u001b[39membedding\u001b[39m\u001b[39m'\u001b[39m])\n\u001b[1;32m      5\u001b[0m df\u001b[39m.\u001b[39mto_csv(\u001b[39m'\u001b[39m\u001b[39mprocessed/embeddings.csv\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m      6\u001b[0m df\u001b[39m.\u001b[39mhead()\n",
      "File \u001b[0;32m~/Documents/GitHub/openai-cookbook/apps/web-crawl-q-and-a/env/lib/python3.9/site-packages/openai/api_resources/embedding.py:33\u001b[0m, in \u001b[0;36mEmbedding.create\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[1;32m     32\u001b[0m     \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m---> 33\u001b[0m         response \u001b[39m=\u001b[39m \u001b[39msuper\u001b[39;49m()\u001b[39m.\u001b[39;49mcreate(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m     35\u001b[0m         \u001b[39m# If a user specifies base64, we'll just return the encoded string.\u001b[39;00m\n\u001b[1;32m     36\u001b[0m         \u001b[39m# This is only for the default case.\u001b[39;00m\n\u001b[1;32m     37\u001b[0m         \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m user_provided_encoding_format:\n",
      "File \u001b[0;32m~/Documents/GitHub/openai-cookbook/apps/web-crawl-q-and-a/env/lib/python3.9/site-packages/openai/api_resources/abstract/engine_api_resource.py:153\u001b[0m, in \u001b[0;36mEngineAPIResource.create\u001b[0;34m(cls, api_key, api_base, api_type, request_id, api_version, organization, **params)\u001b[0m\n\u001b[1;32m    127\u001b[0m \u001b[39m@classmethod\u001b[39m\n\u001b[1;32m    128\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mcreate\u001b[39m(\n\u001b[1;32m    129\u001b[0m     \u001b[39mcls\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    136\u001b[0m     \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mparams,\n\u001b[1;32m    137\u001b[0m ):\n\u001b[1;32m    138\u001b[0m     (\n\u001b[1;32m    139\u001b[0m         deployment_id,\n\u001b[1;32m    140\u001b[0m         engine,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    150\u001b[0m         api_key, api_base, api_type, api_version, organization, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mparams\n\u001b[1;32m    151\u001b[0m     )\n\u001b[0;32m--> 153\u001b[0m     response, _, api_key \u001b[39m=\u001b[39m requestor\u001b[39m.\u001b[39;49mrequest(\n\u001b[1;32m    154\u001b[0m         \u001b[39m\"\u001b[39;49m\u001b[39mpost\u001b[39;49m\u001b[39m\"\u001b[39;49m,\n\u001b[1;32m    155\u001b[0m         url,\n\u001b[1;32m    156\u001b[0m         params\u001b[39m=\u001b[39;49mparams,\n\u001b[1;32m    157\u001b[0m         headers\u001b[39m=\u001b[39;49mheaders,\n\u001b[1;32m    158\u001b[0m         stream\u001b[39m=\u001b[39;49mstream,\n\u001b[1;32m    159\u001b[0m         request_id\u001b[39m=\u001b[39;49mrequest_id,\n\u001b[1;32m    160\u001b[0m         request_timeout\u001b[39m=\u001b[39;49mrequest_timeout,\n\u001b[1;32m    161\u001b[0m     )\n\u001b[1;32m    163\u001b[0m     \u001b[39mif\u001b[39;00m stream:\n\u001b[1;32m    164\u001b[0m         \u001b[39m# must be an iterator\u001b[39;00m\n\u001b[1;32m    165\u001b[0m         \u001b[39massert\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39misinstance\u001b[39m(response, OpenAIResponse)\n",
      "File \u001b[0;32m~/Documents/GitHub/openai-cookbook/apps/web-crawl-q-and-a/env/lib/python3.9/site-packages/openai/api_requestor.py:227\u001b[0m, in \u001b[0;36mAPIRequestor.request\u001b[0;34m(self, method, url, params, headers, files, stream, request_id, request_timeout)\u001b[0m\n\u001b[1;32m    206\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mrequest\u001b[39m(\n\u001b[1;32m    207\u001b[0m     \u001b[39mself\u001b[39m,\n\u001b[1;32m    208\u001b[0m     method,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    215\u001b[0m     request_timeout: Optional[Union[\u001b[39mfloat\u001b[39m, Tuple[\u001b[39mfloat\u001b[39m, \u001b[39mfloat\u001b[39m]]] \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m,\n\u001b[1;32m    216\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m Tuple[Union[OpenAIResponse, Iterator[OpenAIResponse]], \u001b[39mbool\u001b[39m, \u001b[39mstr\u001b[39m]:\n\u001b[1;32m    217\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrequest_raw(\n\u001b[1;32m    218\u001b[0m         method\u001b[39m.\u001b[39mlower(),\n\u001b[1;32m    219\u001b[0m         url,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    225\u001b[0m         request_timeout\u001b[39m=\u001b[39mrequest_timeout,\n\u001b[1;32m    226\u001b[0m     )\n\u001b[0;32m--> 227\u001b[0m     resp, got_stream \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_interpret_response(result, stream)\n\u001b[1;32m    228\u001b[0m     \u001b[39mreturn\u001b[39;00m resp, got_stream, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mapi_key\n",
      "File \u001b[0;32m~/Documents/GitHub/openai-cookbook/apps/web-crawl-q-and-a/env/lib/python3.9/site-packages/openai/api_requestor.py:620\u001b[0m, in \u001b[0;36mAPIRequestor._interpret_response\u001b[0;34m(self, result, stream)\u001b[0m\n\u001b[1;32m    612\u001b[0m     \u001b[39mreturn\u001b[39;00m (\n\u001b[1;32m    613\u001b[0m         \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_interpret_response_line(\n\u001b[1;32m    614\u001b[0m             line, result\u001b[39m.\u001b[39mstatus_code, result\u001b[39m.\u001b[39mheaders, stream\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m\n\u001b[1;32m    615\u001b[0m         )\n\u001b[1;32m    616\u001b[0m         \u001b[39mfor\u001b[39;00m line \u001b[39min\u001b[39;00m parse_stream(result\u001b[39m.\u001b[39miter_lines())\n\u001b[1;32m    617\u001b[0m     ), \u001b[39mTrue\u001b[39;00m\n\u001b[1;32m    618\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    619\u001b[0m     \u001b[39mreturn\u001b[39;00m (\n\u001b[0;32m--> 620\u001b[0m         \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_interpret_response_line(\n\u001b[1;32m    621\u001b[0m             result\u001b[39m.\u001b[39;49mcontent\u001b[39m.\u001b[39;49mdecode(\u001b[39m\"\u001b[39;49m\u001b[39mutf-8\u001b[39;49m\u001b[39m\"\u001b[39;49m),\n\u001b[1;32m    622\u001b[0m             result\u001b[39m.\u001b[39;49mstatus_code,\n\u001b[1;32m    623\u001b[0m             result\u001b[39m.\u001b[39;49mheaders,\n\u001b[1;32m    624\u001b[0m             stream\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m    625\u001b[0m         ),\n\u001b[1;32m    626\u001b[0m         \u001b[39mFalse\u001b[39;00m,\n\u001b[1;32m    627\u001b[0m     )\n",
      "File \u001b[0;32m~/Documents/GitHub/openai-cookbook/apps/web-crawl-q-and-a/env/lib/python3.9/site-packages/openai/api_requestor.py:680\u001b[0m, in \u001b[0;36mAPIRequestor._interpret_response_line\u001b[0;34m(self, rbody, rcode, rheaders, stream)\u001b[0m\n\u001b[1;32m    678\u001b[0m stream_error \u001b[39m=\u001b[39m stream \u001b[39mand\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39merror\u001b[39m\u001b[39m\"\u001b[39m \u001b[39min\u001b[39;00m resp\u001b[39m.\u001b[39mdata\n\u001b[1;32m    679\u001b[0m \u001b[39mif\u001b[39;00m stream_error \u001b[39mor\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39m200\u001b[39m \u001b[39m<\u001b[39m\u001b[39m=\u001b[39m rcode \u001b[39m<\u001b[39m \u001b[39m300\u001b[39m:\n\u001b[0;32m--> 680\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mhandle_error_response(\n\u001b[1;32m    681\u001b[0m         rbody, rcode, resp\u001b[39m.\u001b[39mdata, rheaders, stream_error\u001b[39m=\u001b[39mstream_error\n\u001b[1;32m    682\u001b[0m     )\n\u001b[1;32m    683\u001b[0m \u001b[39mreturn\u001b[39;00m resp\n",
      "\u001b[0;31mRateLimitError\u001b[0m: Rate limit reached for default-global-with-image-limits in organization org-QkIqVS4vwPzxNzDo6pfbr06g on requests per min. Limit: 60 / min. Please try again in 1s. Contact support@openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method."
     ]
    }
   ],
   "source": [
    "from openai.embeddings_utils import distances_from_embeddings\n",
    "\n",
    "df['embeddings'] = df.text.apply(lambda x: openai.Embedding.create(input=x, engine='text-embedding-ada-002')['data'][0]['embedding'])\n",
    "\n",
    "df.to_csv('processed/embeddings.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from openai.embeddings_utils import distances_from_embeddings, cosine_similarity\n",
    "\n",
    "df=pd.read_csv('processed/embeddings.csv', index_col=0)\n",
    "df['embeddings'] = df['embeddings'].apply(eval).apply(np.array)\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_context(\n",
    "    question, df, max_len=1800, size=\"ada\"\n",
    "):\n",
    "    \"\"\"\n",
    "    Create a context for a question by finding the most similar context from the dataframe\n",
    "    \"\"\"\n",
    "\n",
    "    # Get the embeddings for the question\n",
    "    q_embeddings = openai.Embedding.create(input=question, engine='text-embedding-ada-002')['data'][0]['embedding']\n",
    "\n",
    "    # Get the distances from the embeddings\n",
    "    df['distances'] = distances_from_embeddings(q_embeddings, df['embeddings'].values, distance_metric='cosine')\n",
    "\n",
    "\n",
    "    returns = []\n",
    "    cur_len = 0\n",
    "\n",
    "    # Sort by distance and add the text to the context until the context is too long\n",
    "    for i, row in df.sort_values('distances', ascending=True).iterrows():\n",
    "        \n",
    "        # Add the length of the text to the current length\n",
    "        cur_len += row['n_tokens'] + 4\n",
    "        \n",
    "        # If the context is too long, break\n",
    "        if cur_len > max_len:\n",
    "            break\n",
    "        \n",
    "        # Else add it to the text that is being returned\n",
    "        returns.append(row[\"text\"])\n",
    "\n",
    "    # Return the context\n",
    "    return \"\\n\\n###\\n\\n\".join(returns)\n",
    "\n",
    "def answer_question(\n",
    "    df,\n",
    "    model=\"text-davinci-003\",\n",
    "    question=\"Am I allowed to publish model outputs to Twitter, without a human review?\",\n",
    "    max_len=1800,\n",
    "    size=\"ada\",\n",
    "    debug=False,\n",
    "    max_tokens=150,\n",
    "    stop_sequence=None\n",
    "):\n",
    "    \"\"\"\n",
    "    Answer a question based on the most similar context from the dataframe texts\n",
    "    \"\"\"\n",
    "    context = create_context(\n",
    "        question,\n",
    "        df,\n",
    "        max_len=max_len,\n",
    "        size=size,\n",
    "    )\n",
    "    # If debug, print the raw model response\n",
    "    if debug:\n",
    "        print(\"Context:\\n\" + context)\n",
    "        print(\"\\n\\n\")\n",
    "\n",
    "    try:\n",
    "        # Create a completions using the question and context\n",
    "        response = openai.Completion.create(\n",
    "            prompt=f\"Answer the question based on the context below, and if the question can't be answered based on the context, say \\\"I don't know\\\"\\n\\nContext: {context}\\n\\n---\\n\\nQuestion: {question}\\nAnswer:\",\n",
    "            temperature=0,\n",
    "            max_tokens=max_tokens,\n",
    "            top_p=1,\n",
    "            frequency_penalty=0,\n",
    "            presence_penalty=0,\n",
    "            stop=stop_sequence,\n",
    "            model=model,\n",
    "        )\n",
    "        return response[\"choices\"][0][\"text\"].strip()\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        return \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"I don't know.\""
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer_question(df, question=\"What day is it?\", debug=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'On their turn, PCs or enemies may do ONE of the following, plus move about 60’: Make an Attack against a target, Run about 60’, Activate Gear, Reload a weapon, Attempt a Combat Maneuver, Attempt to remove a Fatal Wound from a target, Talking, dropping an item, or retrieving one from their Quick Inventory.'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer_question(df, question=\"What are the actions PCs can take in combat?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Roll under Attack, modified by the target’s Defense, to hit. Example: Miguel, a level 3 Street Samurai wielding his family heirloom katanas, tries to hit a Rent-A-Cop in light armor. Miguel’s Attack is 11 plus his Template bonus of 4, or 15.'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer_question(df, question=\"How do you Attack?\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "05f34a34d73b71652304030c1097be3a5720ea2447153dd6542d145a26b73181"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}