llm_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "# os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch, numpy as np\n",
    "from minai.core import *\n",
    "from datasets import load_dataset\n",
    "from torch.utils.data import DataLoader\n",
    "from torch import nn, tensor\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The model was loaded with use_flash_attention_2=True, which is deprecated and may be removed in a future release. Please use `attn_implementation=\"flash_attention_2\"` instead.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3983fccc4a974a2d9d6fc0fd89e94913",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model_id = 'meta-llama/Llama-2-7b-hf'\n",
    "# model_id = 'gpt2-medium'\n",
    "m = AutoModelForCausalLM.from_pretrained(\n",
    "    model_id,\n",
    "    device_map=0,\n",
    "    use_flash_attention_2=True,\n",
    "    trust_remote_code=True,\n",
    "    low_cpu_mem_usage=True,\n",
    "    torch_dtype=torch.bfloat16,\n",
    "    use_cache=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
    "tokenizer.pad_token = tokenizer.eos_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = load_dataset(\"knowrohit07/know_sql\", revision='f33425d13f9e8aab1b46fa945326e9356d6d5726', split=\"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_text(x):\n",
    "    x['text'] = 'Context: ' + x['context'] + '\\nQuestion: ' + x['question'] + '\\nAnswer: ' + x['answer']\n",
    "    # tokenize here?\n",
    "    return x\n",
    "\n",
    "def tok(x):\n",
    "    x = tokenizer(x['text'], padding='longest')\n",
    "    x['labels'] = deepcopy(x['input_ids'])\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lf = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This dataset is for training a model to generate SQL.\n",
    "\n",
    "In the dataset, every item presents as context the schema defining a table and an English question to translate into a SELECT query. The answer is the appropriate SELECT query."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'question': \"Which head's name has the substring 'Ha'? List the id and name.\",\n",
       " 'context': 'CREATE TABLE head (head_id VARCHAR, name VARCHAR)',\n",
       " 'answer': \"SELECT head_id, name FROM head WHERE name LIKE '%Ha%'\"}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(iter(dataset))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now preprocess to create a single field `text`, collecting this information into a single string formatted of Context, Question, Answer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'question': \"Which head's name has the substring 'Ha'? List the id and name.\",\n",
       " 'context': 'CREATE TABLE head (head_id VARCHAR, name VARCHAR)',\n",
       " 'answer': \"SELECT head_id, name FROM head WHERE name LIKE '%Ha%'\",\n",
       " 'text': \"Context: CREATE TABLE head (head_id VARCHAR, name VARCHAR)\\nQuestion: Which head's name has the substring 'Ha'? List the id and name.\\nAnswer: SELECT head_id, name FROM head WHERE name LIKE '%Ha%'\"}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_text(next(iter(dataset)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We shuffle, filter to only small items, and split into train/eval sets.\n",
    "\n",
    "(It is not strictly necessary to shuffle now, since the DataLoader will shuffle the train set for us later.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = dataset.shuffle(42).map(to_text).filter(lambda x:len(x['text'])<380)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['question', 'context', 'answer', 'text'],\n",
       "    num_rows: 76119\n",
       "})"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset = dataset.select(range(0, len(dataset)-200)) # Subset for testing\n",
    "eval_dataset = dataset.select(range(len(dataset)-200, len(dataset))) # And only 200 eval samples\n",
    "train_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reorder by length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Collate function for DataLoaders\n",
    "def collate_fn(examples, debug=False):\n",
    "    batch_size = len(examples)\n",
    "    input_ids = tokenizer([e['text'] for e in examples], return_tensors='pt', padding=True)['input_ids']\n",
    "    batch = (input_ids[:, :-1], input_ids[:, 1:]) # input_ids, labels\n",
    "    # For HF style: batch = {'input_ids': input_ids[:, :-1], 'labels': input_ids[:, 1:]}\n",
    "    return batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataloader = DataLoader(\n",
    "    train_dataset,\n",
    "    batch_size=batch_size,\n",
    "    collate_fn=collate_fn,\n",
    "    shuffle=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_dataloader = DataLoader(\n",
    "    eval_dataset,\n",
    "    batch_size=batch_size,\n",
    "    collate_fn=collate_fn,\n",
    "    shuffle=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([64, 115]), torch.Size([64, 115]))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xb, yb = next(iter(train_dataloader))\n",
    "xb.shape, yb.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that the first size dimension above was the batch_size we set, 64.\n",
    "\n",
    "But the second size dimension is likely to be _less_ than the max permitted sequence length we filtered to, of 280. Why? Because the dimensions of a batch will be determined by the maxi existing sequence length _in that batch_.\n",
    "\n",
    "Because we have set `shuffle=True`, you will get a different initial batch with different max seq length, and therefore different dimensions, every time you re-evaluate the above cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<s> Context: CREATE TABLE table_name_74 (week VARCHAR, attendance INTEGER)\\nQuestion: How many weeks had an attendance of over 68,000?\\nAnswer: SELECT COUNT(week) FROM table_name_74 WHERE attendance > 68 OFFSET 000</s></s></s></s></s></s></s></s></s></'"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(xb[0])[:250]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Context: CREATE TABLE table_name_74 (week VARCHAR, attendance INTEGER)\\nQuestion: How many weeks had an attendance of over 68,000?\\nAnswer: SELECT COUNT(week) FROM table_name_74 WHERE attendance > 68 OFFSET 000</s></s></s></s></s></s></s></s></s></s></'"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(yb[0])[:250]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Two points to note:\n",
    "\n",
    "- the tokenizer introduced a `<s>` beginning-of-sequence token, to help the model recognize the beginning of the input sequence, in the input `xb[0]`.\n",
    "- the label `yb[0]` is equal to `xb[0]`, except by not having the beginning of sequence token, by having one more token at the end. That last token is the next token to predict.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dls = DataLoaders(train_dataloader, eval_dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(x, y):\n",
    "    return torch.nn.functional.cross_entropy(x.view(-1, x.shape[-1]), y.view(-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from peft import get_peft_config, get_peft_model, LoraConfig, TaskType\n",
    "\n",
    "# peft_config = LoraConfig(\n",
    "#     task_type=TaskType.CAUSAL_LM, inference_mode=False, r=32, lora_alpha=16, lora_dropout=0.1,\n",
    "#     target_modules=[l+\"_proj\" for l in [\"k\", 'v', \"q\", \"o\", \"gate\", \"up\", \"down\"]]\n",
    "# )\n",
    "# m = get_peft_model(m, peft_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prog = ProgressCB(plot=True)\n",
    "cbs = [DeviceCB(), MetricsCB()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Just freeze embeddings for small memory decrease\n",
    "m.model.embed_tokens.weight.requires_grad_(False);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`requires_grad == False` tells PyTorch not to track gradient values. Gradient values are used to update weights in training. So setting requires_grad to False turns off training of those weights, the weights used to define initial embedding layer which maps a token value to an embedding vector. Training fewer weights requires less memory.\n",
    "\n",
    "But why can turn off _these_ weights? Embedding layers are relatively expensive in terms of parameter count.\n",
    "\n",
    "But why can we get away with it? Empirically, it turns out (✨) that freezing them often has only a minimal impact on downstream task performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(m.model.layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Or freeze first n layers for larger decrease (in this case, 24). (Can freeze up to len(m.model.layers)-1)\n",
    "n_freeze = 24\n",
    "for param in m.parameters(): param.requires_grad = False\n",
    "for param in m.lm_head.parameters(): param.requires_grad = True\n",
    "for param in m.model.layers[n_freeze:].parameters(): param.requires_grad = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "optim = partial(torch.optim.Adam, betas=(0.9,0.99), eps=1e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 1e-3\n",
    "sz = len(dls.train)//50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`epoch_sz` controls how many batches are seen before validation metrics are reported. It lets us define a pseudo \"epoch\" which is smaller than a true epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cb_trn = TrainCB(preds_nm='logits')\n",
    "learn = MomentumLearner(m, dls, loss_func=loss_fn, lr=lr, cbs=cbs, preds_nm='logits', epoch_sz=sz, mom=0.9)\n",
    "# learn = TrainLearner(m, dls, loss_func=loss_fn, lr=lr, cbs=cbs, preds_nm='logits', epoch_sz=sz)\n",
    "# learn = Learner(m, dls, loss_func=loss_fn, lr=lr, cbs=cbs+[cb_trn], epoch_sz=sz) #, opt_func=optim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m.gradient_checkpointing_enable()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NB lr_find does *not* reset model, so recreate it afterwards\n",
    "# learn.lr_find(max_mult=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train for 1 \"epoch\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>loss</th>\n",
       "      <th>epoch</th>\n",
       "      <th>train</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2.084</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1.052</td>\n",
       "      <td>0</td>\n",
       "      <td>eval</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/algal/miniconda3/envs/mbert/lib/python3.11/site-packages/torch/utils/checkpoint.py:87: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfcAAAFfCAYAAABTOoWkAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAMrdJREFUeJzt3Xl8VOWhPvDnzJptZrJnsieEnYSwBJBFQItYVMSFimtBLa0t2iql12JvK1Ivsep1K4qKtwo/F7QtuCtFhYiyJSg7JCwJ2ck+k3UmM3N+fyQZCBDIJDM5M2ee7+czH5LJyczDkPDMe95z3iOIoiiCiIiIZEMhdQAiIiJyL5Y7ERGRzLDciYiIZIblTkREJDMsdyIiIplhuRMREckMy52IiEhmVAP9hA6HA+Xl5dDpdBAEYaCfnoiIyGeJoojGxkbExcVBoeh5fD7g5V5eXo7ExMSBfloiIiLZKCkpQUJCQo9fH/By1+l0ADqC6fX6gX56IiIin2U2m5GYmOjs0p4MeLl37YrX6/UsdyIioj643LQ2D6gjIiKSGZY7ERGRzLDciYiIZGbA59yJiEje7HY72tvbpY7hk9RqNZRKZb8fh+VORERuIYoiKisr0dDQIHUUnxYaGgqj0divtWBY7kRE5BZdxR4dHY2goCAuVOYiURTR0tKCqqoqAEBsbGyfH4vlTkRE/Wa3253FHhERIXUcnxUYGAgAqKqqQnR0dJ930fOAOiIi6reuOfagoCCJk/i+rtewP8ctsNyJiMhtuCu+/9zxGrLciYiIZMbny/274zW4c+0u/OWjQ1JHISIi8go+X+7tdgd2nKzFtwXVUkchIiI/l5KSghdeeEHqGL5/tPy4pDAIAlBU24LqRguidFqpIxERkQ+ZOXMmxowZ45ZSzs3NRXBwcP9D9ZPPj9wNQWoMi+m49N3e03USpyEiIrkRRRE2m61X20ZFRXnFGQM+X+4AkJUSBgDILaqXOAkREXURRREtVtuA30RR7HXGRYsWIScnBy+++CIEQYAgCHjrrbcgCAI2b96MrKwsaLVabN++HSdPnsS8efMQExODkJAQTJgwAV999VW3xzt/t7wgCHjjjTdw8803IygoCEOGDMHHH3/srpe4Rz6/Wx4AspLD8fauYuQVceROROQtWtvtGPmXzQP+vEdWXosgTe/q7cUXX0RBQQHS09OxcuVKAMDhw4cBAP/1X/+FZ599FoMGDUJoaChKS0tx3XXX4cknn0RAQADWrVuHuXPnIj8/H0lJST0+xxNPPIGnn34azzzzDP7+97/jrrvuwunTpxEeHt7/v2wPZDVyP1RuRou1d7tOiIiIDAYDNBoNgoKCYDQaYTQanavCrVy5Etdccw3S0tIQERGBzMxM/OpXv0JGRgaGDBmCJ598EoMGDbrsSHzRokW44447MHjwYKxatQrNzc3Ys2ePR/9eshi5x4cGItYQgApTG/aVNGBKWqTUkYiI/F6gWokjK6+V5HndISsrq9vnzc3NeOKJJ/Dpp5+ivLwcNpsNra2tKC4uvuTjjB492vlxcHAwdDqdc/14T5FFuQuCgKyUcHyyvxx5RfUsdyIiLyAIQq93j3uj8496/8Mf/oDNmzfj2WefxeDBgxEYGIj58+fDarVe8nHUanW3zwVBgMPhcHvec7m8W76srAx33303IiIiEBQUhDFjxmDv3r2eyOaSrOSug+o4705ERL2n0Whgt9svu9327duxaNEi3HzzzcjIyIDRaERRUZHnA/aBS2+p6uvrMXXqVFx11VX44osvEB0djZMnTyI0NNRD8Xqva979x+IG2B0ilAqub0xERJeXkpKC3bt3o6ioCCEhIT2OqgcPHoyNGzdi7ty5EAQBf/7znz0+Au8rl0buf/vb35CYmIg333wTEydOREpKCn7yk58gLS3NU/l6bbhRjxCtCk0WG45VmqWOQ0REPmLZsmVQKpUYOXIkoqKiepxDf/755xEWFoYpU6Zg7ty5uPbaazFu3LgBTts7gujCCYEjR47Etddei9LSUuTk5CA+Ph6/+c1vsHjx4h6/x2KxwGKxOD83m81ITEyEyWSCXq/vX/rz/Pwfe/BtQTWeuHEUFk5JcetjExFRz9ra2lBYWIjU1FQEBARIHcenXeq1NJvNMBgMl+1Ql0bup06dwpo1azBkyBBs3rwZDzzwAH77299i/fr1PX5PdnY2DAaD85aYmOjKU7pkAufdiYiIXCt3h8OBcePGYdWqVRg7dix+9atfYfHixVizZk2P37N8+XKYTCbnraSkpN+hezI+5Wy5u7JCERERkZy4VO6xsbEYOXJkt/tGjBhxyXP8tFot9Hp9t5unjEkMhUoh4IzZgtL6Vo89DxERkTdzqdynTp2K/Pz8bvcVFBQgOTnZraH6Kkijwqh4AwBg72muM09ERP7JpXJ/5JFHsGvXLqxatQonTpzAu+++i9dffx1LlizxVD6Xcd6diEg63npqmC9xx2vo0nnuEyZMwKZNm7B8+XKsXLkSqampeOGFF3DXXXf1O4i7ZKWE4Y3vCpHHK8QREQ0YjUYDhUKB8vJyREVFQaPRQBC43ogrRFGE1WpFdXU1FAoFNBpNnx/L5XUBb7jhBtxwww19fkJPG5/ccZWd/DONMLW0wxCkvsx3EBFRfykUCqSmpqKiogLl5eVSx/FpQUFBSEpKgkLR92u7+e6ivz2I0mmRGhmMwppm7C2uw9XDY6SORETkFzQaDZKSkmCz2Xq1nCtdSKlUQqVS9Xuvh+zKHehYZ76wphl5RfUsdyKiASQIAtRq9QUXS6GBJYvruZ9vQkrHrnnOuxMRkT+SZbl3LWazr7QBFht3DRERkX+RZbkPigxGeLAGVpsDh8pMUschIiIaULIsd0EQnNd35655IiLyN7Isd+DsvHsuy52IiPyMbMs9q3Pefe/pOjgcvIgMERH5D9mW+6g4A7QqBepb2nGqpknqOERERANGtuWuUSkwJjEUAHfNExGRf5FtuQM8352IiPyTrMu9a9497zSvEEdERP5D1uU+LjkMggCcrm1BlblN6jhEREQDQtblrg9QY1iMDgCQd5q75omIyD/IutwBzrsTEZH/kX25c96diIj8jezLvWvkfrjcjGaLTeI0REREnif7co8LDUScIQB2h4h9JQ1SxyEiIvI42Zc7AGQ515nnrnkiIpI/vyj3Cc515nlQHRERyZ9flHvXyP2H0/Ww2R0SpyEiIvIsvyj3oTE66AJUaLbacayyUeo4REREHuUX5a5UCBiX1LFrnvPuREQkd35R7sDZeXcuZkNERHLnN+XeNe+ed7oOoihKnIaIiMhz/KbcMxNCoVYKOGO2oLS+Veo4REREHuM35R6oUWJUnAEA592JiEje/KbcgbPz7rmcdyciIhnzq3Lvmnffy4vIEBGRjPlXuSd3jNwLzjShocUqcRoiIiLP8KtyjwjRYlBUMAAuRUtERPLlV+UOnB29c96diIjkyv/Kvet8dx4xT0REMuV35T6hs9wPlJrQ1m6XOA0REZH7+V25p0QEITJEA6vdgUNlJqnjEBERuZ3flbsgCBjPeXciIpIxvyt34Oyuec67ExGRHPlluTsXsymuh8PBi8gQEZG8+GW5j4rTI0CtQENLO05WN0kdh4iIyK38stzVSgXGJnLenYiI5Mkvyx0AsjovIsN5dyIikhs/LveOefdcXkSGiIhkxm/LfVxSKBQCUFLXijPmNqnjEBERuY1L5b5ixQoIgtDtZjQaPZXNo3QBagw36gEAeZx3JyIiGXF55D5q1ChUVFQ4bwcPHvRErgExIaXroDrumiciIvlQufwNKpXPjtbPNz4lHOt2nkYe592JiEhGXB65Hz9+HHFxcUhNTcXtt9+OU6dOXXJ7i8UCs9nc7eYtukbuR8rNaLLYJE5DRETkHi6V+6RJk7B+/Xps3rwZa9euRWVlJaZMmYLa2toevyc7OxsGg8F5S0xM7Hdod4k1BCI+NBAOEdhX3CB1HCIiIrdwqdznzJmDW2+9FRkZGZg1axY+++wzAMC6det6/J7ly5fDZDI5byUlJf1L7GacdyciIrnp16lwwcHByMjIwPHjx3vcRqvVQq/Xd7t5k/FdF5HhvDsREclEv8rdYrHg6NGjiI2NdVeeAdc1cv+xuAE2u0PiNERERP3nUrkvW7YMOTk5KCwsxO7duzF//nyYzWYsXLjQU/k8bmi0DroAFVqsdhytaJQ6DhERUb+5VO6lpaW44447MGzYMNxyyy3QaDTYtWsXkpOTPZXP4xQKAVnJnHcnIiL5cOk89w0bNngqh6SyUsKxNb8aeafrcN+0VKnjEBER9Yvfri1/rrMj93qIoihxGiIiov5huQPITAyFWimgutGC4roWqeMQERH1C8sdQIBaiYx4AwBeRIaIiHwfy73TBJ7vTkREMsFy7zT+nHl3IiIiX8Zy79RV7ieqmlDfbJU4DRERUd+x3DtFhGiRFhUMANh7mqN3IiLyXSz3c3TNu+dy3p2IiHwYy/0cWV0H1XHenYiIfBjL/Rxdi9kcKG1AW7td4jRERER9w3I/R3JEEKJ1WrTbRfxQzNE7ERH5Jpb7OQRBwKRBEQCA3ac4705ERL6J5X6eSakd8+67C2slTkJERNQ3LPfzXDGoo9x/LG6AxcZ5dyIi8j0s9/OkRYUgMkQDi82B/SUmqeMQERG5jOV+HkEQMLFr1/wp7ponIiLfw3K/iEmpnQfVFfKgOiIi8j0s94uY1Dnvvvd0PdrtDonTEBERuYblfhFDo3UIDVKjtd2OA6WcdyciIt/Ccr8IhULAxBSeEkdERL6J5d4DLmZDRES+iuXeg67FbPKK6mDjvDsREfkQlnsPRsTqoQtQodlqx+Fys9RxiIiIeo3l3gMl592JiMhHsdwvoeuUOM67ExGRL2G5X0LXYjZ7iupgd4gSpyEiIuodlvsljIrTI0SrQmObDUcrOO9ORES+geV+CSqlAuOTwwBwKVoiIvIdLPfLODvvzoPqiIjIN7DcL+PceXcH592JiMgHsNwvY3SCAYFqJRpa2lFQ1Sh1HCIiostiuV+G+tx5d54SR0REPoDl3gtdS9FyMRsiIvIFLPde6LqIzJ7COogi592JiMi7sdx7ITPRAK1KgZomK05WN0kdh4iI6JJY7r2gVSkxNikUALCL8+5EROTlWO691HVKHBezISIib8dy76VzF7PhvDsREXkzlnsvjUsKg0apQFWjBUW1LVLHISIi6hHLvZcC1EpkJhoAcClaIiLybix3F3DenYiIfAHL3QWcdyciIl/AcnfB+OQwqBQCyk1tKK1vlToOERHRRfWr3LOzsyEIAh5++GE3xfFuQRoVMhI65t13cd6diIi8VJ/LPTc3F6+//jpGjx7tzjxej/PuRETk7fpU7k1NTbjrrruwdu1ahIWFXXJbi8UCs9nc7ebLnPPuvIgMERF5qT6V+5IlS3D99ddj1qxZl902OzsbBoPBeUtMTOzLU3qNrOQwKASgpK4V5Q2cdyciIu/jcrlv2LABP/zwA7Kzs3u1/fLly2EymZy3kpISl0N6E12AGunxnee7c/ROREReyKVyLykpwe9+9zu8/fbbCAgI6NX3aLVa6PX6bjdf57y+Oy8iQ0REXsilct+7dy+qqqowfvx4qFQqqFQq5OTk4KWXXoJKpYLdbvdUTq/Cg+qIiMibqVzZ+Cc/+QkOHjzY7b57770Xw4cPx6OPPgqlUunWcN5qQmo4BAEorGlGlbkN0fre7cUgIiIaCC6Vu06nQ3p6erf7goODERERccH9cmYIVGOEUY8jFWbsKqzDjZlxUkciIiJy4gp1fXTuUrRERETexKWR+8Vs27bNDTF8z6TUCLz5fRHn3YmIyOtw5N5HEzuPmD9R1YSaJovEaYiIiM5iufdReLAGw2J0AIA9HL0TEZEXYbn3A+fdiYjIG7Hc+4HnuxMRkTdiufdD17z7scpG1DdbJU5DRETUgeXeD1E6LdKiggEAe4o4eiciIu/Acu+nSYM6d81znXkiIvISLPd+cl5EhleIIyIiL8Fy76crOkfuRyrMMLW2S5yGiIiI5d5vMfoApEQEQRSBPM67ExGRF2C5uwFPiSMiIm/CcncDLmZDRETehOXuBl1HzB8qN6PJYpM4DRER+TuWuxvEhwYiISwQdofIeXciIpIcy91NOO9ORETeguXuJldw3p2IiLwEy91Nus53P1BqQouV8+5ERCQdlrubJIQFIs4QAJtDxA+nG6SOQ0REfozl7iaCIJxdZ55L0RIRkYRY7m7kXGeeF5EhIiIJsdzdqGvkvq+kAW3tdonTEBGRv2K5u1FKRBCidVpY7Q78WNwgdRwiIvJTLHc34rw7ERF5A5a7m3HenYiIpMZyd7OuxWx+KK6HxcZ5dyIiGngsdzdLiwpBZIgGFpsDB0pNUschIiI/xHJ3M0EQMDGVS9ESEZF0WO4ewIvIEBGRlFjuHjCpc9597+l6tNsdEqchIiJ/w3L3gKHROoQGqdFiteNgGefdiYhoYLHcPUChEDAxhafEERGRNFjuHsLFbIiISCosdw/pWswmr6geNs67ExHRAGK5e8iIWD30ASo0WWxY9GYuSupapI5ERER+guXuIUqFgJXz0qFVKfDdiRpc+8K3WLejCA6HKHU0IiKSOZa7B900Nh5fPjwdE1PD0WK14/GPD2PB6ztxqrpJ6mheTRT5BoiIqD9Y7h6WGhmMDYuvwF/njUKwRonconrMeXE7Xss5ybn485Q1tGLe6u9w0ys70GrluvxERH3Fch8ACoWAeyanYPMj03HlkEhYbA5kf3EMt67ZgfzKRqnjeYXTtc247dWd2F9qwv6SBqzfWSR1JCIin8VyH0AJYUFYf99EPD1/NHQBKuwvNeGGv2/HS18f9+uV7E5UNeG213airKEVugAVAGBNzkk0trVLnIyIyDex3AeYIAi4LSsRXy2dgVkjYtBuF/HclgLcuPp7HPLD1eyOVpix4LWdOGO2YGhMCLY8MgNpUcFoaGnH/31XKHU8IiKfxHKXSIw+AGt/Ph4v3TEWYUFqHK0wY97L3+PpL4+hrd0/5psPlDbg9td3obbZilFxemz45WQYDQFYes0wAMAb2wtR32yVOCURke9huUtIEATcmBmHLUtn4IbRsbA7RLyy7SSuf2k79p6ulzqeR+UV1eGutbtham3H2KRQvLv4CoQHawAAc9KNGBmrR5PFhldzTkqclIjI97hU7mvWrMHo0aOh1+uh1+sxefJkfPHFF57K5jciQ7RYfec4vHr3eETptDhZ3Yz5r+7Ayk+OoMVqkzqe2+04UYN7/m8PGi02TEoNx/+7fxIMgWrn1xUKAX+4tmP0vm5nEarMbVJFJSLySS6Ve0JCAp566ink5eUhLy8PV199NebNm4fDhw97Kp9f+Wm6EV89MgPzxydAFIF/fF+In76wHTtO1kgdzW225lfh3rdy0dpux5VDIvHWvRMRolVdsN3MYVEYnxyGtnYHVm89IUFSIiLfJYj9XDEkPDwczzzzDO6///5ebW82m2EwGGAymaDX6/vz1LK2Lb8Kj208iHJTx6j1rklJ+OOc4dAFqC/znd7ry0OVeOi9H9BuFzFrRAxevmsstCplj9vvPFmLO9buglop4Jvfz0RieNAApiUi8j697dA+z7nb7XZs2LABzc3NmDx5co/bWSwWmM3mbje6vJnDorH5kem4+4okAMA7u4sx+/lvsfbbU6htskicznUf7SvDknc7iv36jFisuXvcJYsdACanRWDa4Ei020W89PXxAUpKROT7XB65Hzx4EJMnT0ZbWxtCQkLw7rvv4rrrrutx+xUrVuCJJ5644H6O3Htv58laPPrvAyjuvPiMWilg9kgjbp+YiKlpkVAoBIkTXtoHuSV4dOMBiCJwy7h4PH3raKiUvXtfua+kATe9/D0UAvCfR2ZgcHSIh9MSEXmv3o7cXS53q9WK4uJiNDQ04N///jfeeOMN5OTkYOTIkRfd3mKxwGI5O9I0m81ITExkubuo1WrHph/L8H5uMfaXnj0fPiEsEAuyEvGzrEQYDQESJry49TuL8JePOo7JuGtSEv46L93lNyO/WJeHr46ewfWjY/HyneM8EZOIyCd4rNzPN2vWLKSlpeG1115zazDq2ZFyMzbkFmPTj2VobOs4ml4hAFcNi8btE5Nw1bCoXo+MPem1nJPI/uIYAOC+qan48w0jIAiu72U4WmHGdS9thygCn/12GkbFGdwdlYjIJ3h8zr2LKIrdRubkeSPj9Fg5Lx17HpuF527LxMSUcDhE4OtjVVi8Pg9TnvoGz2w+huJaaa4hL4oiXviqwFnsD141uM/FDgAjYvWYOzoOAPDcfwrclpOISK5cGrk/9thjmDNnDhITE9HY2IgNGzbgqaeewpdffolrrrmmV4/BkbtnnKhqwgd5JfjX3lLUnbOq27TBkVgwIRGzR8Vc9gA2dxBFEX/7Mt+5+Myy2UPx4NVD+v24hTXNmPVcDuwOERt/MwXjksL6/ZhERL7GI7vl77//fnz99deoqKiAwWDA6NGj8eijj/a62F0JRn1jtTmw5cgZbMgtxncnatD1rxserMEtY+Nx+8REDI7WeeS5HQ4RKz89grd2FAEA/vv6EfjFlYPc9viP/usA3s8rwZS0CLy7+Aq3PS4Rka8YsDl3V7HcB05JXQs+yCvBB3klOGM+O3WSlRyGBRMSMSrOgEidBuFBmn7P0dsdIv606SA25JYAAJ68KR13X5Hcr8c8X1lDK656Zhusdgfe+cUkTB0c6dbHJyLydix3crLZHcgpqMZ7e0qwNb8Kdkf3f3JBAEID1YgM0SIiRIPIEG3Hx8EaROo6/owI0SKq8+vB560oZ7M7sOyf+/HhvnIoBODp+ZmYPz7BI3+XFR8fxls7ijAmMRSbfjOlz/P4RES+iOVOF3XG3IZ/7S3F5wcrcMbchrpmKxwu/gQEqpWICOkqfA3qW9qx93Q9VAoBL9w+Bjd0HvzmCVWNbZj+9Fa0tTvwxs+zMGtkjMeei4jI27DcqVfsDhH1LVbUNllR02RBTZPF+XFtkxW1zRZUN1lR2/m1tnbHRR9Ho1Rg9Z1jMXuU0eOZn/riGF7NOYnhRh0+/+2VXr+IDxGRu/S2Qy+8Ygf5FaVCcO6GH4ZLH2gniiJarHbUNllR3WRBbZMFtc1W1LdYMX1IFNLjB+b88wdmDMI7u07jWGUjPjtYgbmZnttTQETki1ju1GuCICBYq0KwVoWkCOku4hIapMHi6YPw3JYCPL+lAHPSjV6xaA8Rkbfg/4jkk+6blorwYA1O1TRj4w9lUschIvIqLHfySSFaFX49Iw0A8OLXx2Gx2SVORETkPVju5LPumZyMGL0WZQ2t2LCnROo4REReg+VOPitArXQubbt66wm0Wjl6JyICWO7k4xZkJSIhLBDVjRas21kkdRwiIq/AciefplEp8PCsoQCAV3NOwtzWLnEiIiLpsdzJ5908Nh6Do0PQ0NKON7YXSh2HiEhyLHfyeUqFgKXXdIze/2/7qW6XvCUi8kcsd5KFn44yYlScHs1Wu/Na8kRE/orlTrKgUAhYNnsYAGDdjiKcMbdJnIiISDosd5KNmcOiMD45DBabA6u/OSF1HCIiybDcSTYE4ezofUNuMUrqWiROREQkDZY7ycrktAhMGxyJdruIF746LnUcIiJJsNxJdpZd2zF63/RjKU5UNUqchoho4LHcSXbGJIbimpExcIjA81s4eici/8NyJ1n6/eyhEATgs4MV+GgfLwlLRP6F5U6yNNyox50TkwAAv9uwD69sOwFRFCVORUQ0MFjuJFt/nZeOX0xLBQA8/WU+Htt0CDa7Q+JURESex3In2VIoBPz3DSPx+NyREATgvT3FWLw+D80Wm9TRiIg8iuVOsnfv1FS8evd4BKgV2JpfjQWv70QVV7AjIhljuZNfuHaUEe8tvgIRwRocKjPj5ld2oOAMT5MjInliuZPfGJsUho2/mYLUyGCUNbTi1jU7sONkjdSxiIjcjuVOfiU5Ihgbfz0FWclhaGyzYeE/9uDDH3mqHBHJC8ud/E5YsAZv/2ISrs+IRbtdxMPv78PLW3mqHBHJB8ud/FKAWom/3zEWv5w+CADwzOZ8PLbpIE+VIyJZYLmT31IoBDx23QisnDcKCgF4b08J7l+XhyaeKkdEPo7lTn7v55NT8No9WQhQK5BTUI3bXt2JMzxVjoh8GMudCMA1I2Pw/i8nIzJEgyMVZtz88vfIr+SpckTkm1juRJ0yE0Ox6TdTMSgqGOWmNsxfswM7TvBUOSLyPSx3onMkhgdh46+nYGJKOBotNix8cw82/lAqdSwiIpew3InOExqkwfr7J2JuZhza7SKWfrAfL319nKfKEZHPYLkTXUSAWokXF4zBAzPSAADPbSnAo/8+gHaeKkdEPoDlTtQDhULAH+cMx5M3pUMhAB/klSL782NSxyIiuiyWO9Fl3H1FMlbfOQ4A8I/vC7HzZK3EiYiILo3lTtQL12XE4o6JiQCAZf/cj8a2dokTERH1jOVO1Et/un4kEsMDUdbQiv/57KjUcYiIesRyJ+qlEK0Kz87PhCAAG3JL8M2xM1JHIiK6KJY7kQsmDYrA/VNTAQCP/vsg6putEiciIrqQS+WenZ2NCRMmQKfTITo6GjfddBPy8/M9lY3IKy27dhgGR4egutGCP390SOo4REQXcKncc3JysGTJEuzatQtbtmyBzWbD7Nmz0dzc7Kl8RF4nQK3Ec7dlQqkQ8OmBCnyyv1zqSERE3QhiP5bdqq6uRnR0NHJycjB9+vRefY/ZbIbBYIDJZIJer+/rUxNJ7rktBXjp6+MIDVLjPw9PR7Q+QOpIRCRzve3Qfs25m0wmAEB4eHiP21gsFpjN5m43Ijl46OrBSI/Xo6GlHY/++wCXpyUir9HnchdFEUuXLsW0adOQnp7e43bZ2dkwGAzOW2JiYl+fksirqJUKPHfbGGhUCmzNr8b7uSVSRyIiAtCPcn/wwQdx4MABvPfee5fcbvny5TCZTM5bSQn/AyT5GBqjw7LZQwEAf/30CErqWiRORETUx3J/6KGH8PHHH2Pr1q1ISEi45LZarRZ6vb7bjUhO7p82CBNTwtFstWPZP/fD4eDueSKSlkvlLooiHnzwQWzcuBHffPMNUlNTPZWLyGcoFQKe/VkmgjRK7C6sw5s7iqSORER+zqVyX7JkCd5++228++670Ol0qKysRGVlJVpbWz2Vj8gnJEUE4U/XjwAAPP3lMZyoapI4ERH5M5fKfc2aNTCZTJg5cyZiY2Odt/fff99T+Yh8xp0TkzBjaBQsNgd+/8E+2HjtdyKSiMu75S92W7RokYfiEfkOQRDwt1tHQx+gwv5SE17ZdlLqSETkp7i2PJEbGQ0BWDmv49TQl74+jkNlJokTEZE/YrkTudm8MXGYk26EzSFi6Qf70NZulzoSEfkZljuRmwmCgCdvSkdkiAYFZ5rw/JYCqSMRkZ9huRN5QESIFtm3jAYAvL79FHKL6iRORET+hOVO5CHXjIzB/PEJEEXg9x/sR7PFJnUkIvITLHciD/rL3JGIDw1EcV0LVn1+VOo4ROQnWO5EHqQPUOOZ+R2759/ZXYycgmqJExGRP2C5E3nYlMGRWDQlBQDw6L8OwNTSLm0gIpI9ljvRAHj0p8MxKDIYleY2rPjksNRxiEjmWO5EAyBQo8Szt2VCIQCbfizDl4cqpI5ERDLGcicaIOOSwvDAjDQAwGObDqGqsU3iREQkVyx3ogH0u1lDMNyoQ12zFdOf3opFb+7BW98XoqimWepoRCQjgiiK4kA+odlshsFggMlkgl6vH8inJvIKBWca8cv1eSiqbel2f3JEEGYOjcLMYdG4YlAEAjVKiRISkbfqbYey3IkkIIoiCs40YVt+FbblVyPvdB3a7Wd/FTUqBSalhmPmsGjMGBqFtKhgCIIgYWIi8gYsdyIf0mSxYceJGmwrqEZOfjXKGlq7fT0hLBAzh0VhxtBoTEmLQLBWJVFSIpISy53IR4miiJPVTdiWX41t+dXYU1gHq93h/LpGqcCE1DDMHBqNGcOiMCQ6hKN6Ij/BcieSiRarDTtP1naUfUEVSuq6j+rjDAGY0Tmqnzo4AroAtURJicjTWO5EMiSKIgprmrEtvxo5BdXYdaoWFtvZUb1KIWB8chhmDovGzGFRGG7UcVRPJCMsdyI/0Gq1Y1dhLXI6y77wvFPqYvRazOg8An/q4EgYAjmqJ/JlLHciP3S6trlzrr4KO0/Voq397KheqRAwLinUeQT+qDg9R/VEPoblTuTn2trt2FNY55yrP1XdfVQfpdNi+pAozBwWhSuHRCI0SCNRUiLqLZY7EXVTUtfiPNVux8katFjtzq8pBGBMYseofk66EUNidBImJaKesNyJqEcWmx15RfXYll+FnIJqFJxpcn5NIQCPzBqK31w1GEoFd9sTeROWOxH1WllDK3Lyq7H5cCVyCqoBAFMHR+D5BWMQrQuQOB0RdWG5E1Gf/GtvKf784SG0ttsRGaLBCwvGYtqQSKljERF636G8KhwRdTN/fAI+eWgahht1qGmy4p5/7Mazm/NhO2eVPCLybix3IrrA4OgQfLhkKu6clARRBFZvPYE71u5Chan18t9MRJJjuRPRRQWolVh1cwb+fsdYhGhVyC2qx3Uvbsc3x85IHY2ILoPlTkSXNDczDp8+NA0Z8QbUt7Tjvrfy8D+fHYHVxt30RN6K5U5El5USGYx//XoyFk1JAQCs3V6I217biZK6FmmDEdFFsdyJqFe0KiVW3DgKr90zHvoAFfaVNOC6l7bjy0MVUkcbcKIoosrchp0na/HenmJ8ffQM7I4BPfGI6JJ4KhwRuay0vgUPvfcjfixuAAAsnJyM5deNQIBa6dbnEUURJ6qasLuwDscqzTAEqmE0BCJWHwCjIQCxhgCEB2s8tkZ+W7sdhTXNOFXdjFPVTThV04yT1U0orG5Go8XWbduEsEAsmpKC2yYkQs/L7pKH8Dx3IvKodrsDz27Ox2vfngIAjIrTY/Wd45AaGdznx7TZHThSYcaewjrsKaxDblEd6lvaL/k9GpUCxnPK3mgI6Cz/QMR23hcRou1xtT1RFFFpbnMW+MnqjgI/Vd2MclMrevofUiEAieFBSI4IxoHSBjR05gzWKPGzrEQsmpKClH68FkQXw3InogGxNb8Kv/9gP+qarQjWKLHqlgzMGxPfq+9ta7djf0kDcovqsLuwDj+crkfzOWveA0CAWoGxiWHITAxFi9WGClMbKk1tqDC1oabJ0qvnUSkExOgDEKPXItYQiCidFrXNVpyqbkJhTXO3dfbPpw9QIS06BIMiQzAoKhhpUcEYFBWC5IggaFUdeyparXZ8uK8M//iuEMerOpbyFQTgJ8Ojcd/UVExOi/DpK/C1tdvRarUjNEjt038POWC5E9GAqTS14bfv/Yg9RXUAgNsnJOLxuaMQqOm+m76xrR17T9cjt6hjZL6/xATreYvj6AJUmJASjomp4ZiQEo6MeAM0qosfHmS1OXDG3IZKc0fZn+ks/Upzq/NNwBlzGy43Ha5UCEgKD3IW96DIzj+jghHhwm5/URTx3YkavPl9Eb45VuW8f7hRh/umpuLGMXFun7roba4Wqx2m1vZuN/N5f5pa22Fus12wXdeZEYMigzE3Mw43jolDWlTIgP89iOVORAPMZnfgpa+P4+9bT0AUgWExOqy6JQPVjW3YU1iPPUW1OFJuvqBoo3RaTDynzIcZdW69YI3N7kBNkxUVplbniP9MYxvCgjTOEk8KD+rxDURfnaxuwrodRfhnXila2zv2DIQHa3DXpCTcc0UyovXuX7O/ytyGg2UmHCoz41C5CSermtDQWd42Nx/wlx6vx7zMeNyQGYtYQ6BbH5t6xnInIkl8f6IGv9uwr8dd5knhQZiQEo5JqeGYkBqOlIggWe/qNbW04/28YqzbcRplDR0r/KmVAm4YHYf7pqYiI8Hg8mOKoohyUxsOlZlwuMzUUejlZlQ3XnqaQq0UYAhUQx+ohj5ADUNg95s+UHXOx+dsE6SGAOCro2fw0b5ybD9e4zw7QBCAiSnhuHFMHK5Lj0VYsMblvw/1HsudiCRT3WjBH/61H98WVGNwdAgmpoZjYmoEJqaEw2jwz6vM2ewO/OfIGfzju0Lkna533j8hJQz3TU3FNSNjoFJeuPdAFEWU1LXiUHlniZeZcLjcjLpm6wXbKgQgLSoEGfEGjIo3YIRRh4gQrbO0A9VKt7yRqm2y4PNDlfhkX7lzKgboOLZh+tAozBsTh1kjYhCsVfX7uag7ljsRSc5md1y0sPzdgdIGvPl9ET7ZX+7cXR4fGoiFU5IxfWgU8isbcbjcjEOdZW5us13wGCqFgCExOqTH6ZEeb0B6vAEjYnUI0gxsoZY1tOLT/eX4aF85jlSYnfcHqpWYNTIGN2bGYcbQKLdPe/grljsRkZc7Y27D27tO453dxRcdiXfRKBUYZtQhPb6zyOMMGGbUSXJw3qWcqGrEx/vK8fH+chTVnl290BCoxpx0I24cE4dJqRFuPabC37DciYh8RFu7HR/tK8NbO06jqKYZw4w6ZMQbkB6vx6g4A4bG6Hxq5CuKIg6UmvDx/nJ8sr8cVeccCxCt0+KG0XG4angU0uMMnKN3EcudiMgHiaIoqwMM7Q4Ruwtr8fG+cnx+sOKCKYb40ECkx+udxwmkxxkQpdNKlNb7sdyJiMirWGx2fFtQg88OlGNfSUO3XffnMuoDnHstMjqPJ4jRa2X1pqevPFbu3377LZ555hns3bsXFRUV2LRpE2666Sa3ByMiInkzt7XjcJkZh8s7Dhw8WGbCqZrmiy75GxmicR5v0HXsQXxooN8Vfm871OXDKpubm5GZmYl7770Xt956a79CEhGR/9IHqDE5LQKT0yKc9zVbbDhaYXYuxnO43ITjVU2oabJiW341tuVXO7cNC1KfU/LuySQIAoLUSgRplAjUqDr/VCL4nI+DNOd8Xa1EkFYJjVLhVW80XC73OXPmYM6cOZ7IQkREfi5Yq0JWSjiyUsKd97Va7ThWacahcjMOlZpwqNyEgjONqG9px/bjNRKmPUup6HhTEKjp/sbgZ+MTcPvEpAHP4/ETIi0WCyyWs0dKms3mS2xNRETUXaBGibFJYRibFOa8z2Kzo6CyCYfKTajt5QWEesPmENHaeaGcZosdre02tFjtaLF23Ndi7f5517UR7A4RjRbbBZcCvnJIpNuyucLj5Z6dnY0nnnjC009DRER+RKtSIiPB0Kfle93JZnegxflmoKP4W9u7yt8m2QV2PF7uy5cvx9KlS52fm81mJCYmevppiYiIPE6lVECvVEAfoJY6SjceL3etVgutlucsEhERDRTfWfKIiIiIesXlkXtTUxNOnDjh/LywsBD79u1DeHg4kpIG/ohAIiIi6s7lcs/Ly8NVV13l/LxrPn3hwoV466233BaMiIiI+sblcp85cyYGeMVaIiIicgHn3ImIiGSG5U5ERCQzLHciIiKZYbkTERHJDMudiIhIZjy+Qt35uo605wVkiIiIXNPVnZc7a23Ay72xsREAuL48ERFRHzU2NsJg6PmiOYI4wCetOxwOlJeXQ6fTue3C9l0XoykpKYFer3fLY1Lv8fWXFl9/afH1l5a/vf6iKKKxsRFxcXFQKHqeWR/wkbtCoUBCQoJHHluv1/vFP6634usvLb7+0uLrLy1/ev0vNWLvwgPqiIiIZIblTkREJDOyKHetVovHH3+c142XCF9/afH1lxZff2nx9b+4AT+gjoiIiDxLFiN3IiIiOovlTkREJDMsdyIiIplhuRMREckMy52IiEhmfL7cX3nlFaSmpiIgIADjx4/H9u3bpY7kF1asWAFBELrdjEaj1LFk7dtvv8XcuXMRFxcHQRDw4Ycfdvu6KIpYsWIF4uLiEBgYiJkzZ+Lw4cPShJWhy73+ixYtuuB34oorrpAmrMxkZ2djwoQJ0Ol0iI6Oxk033YT8/Pxu2/DnvzufLvf3338fDz/8MP70pz/hxx9/xJVXXok5c+aguLhY6mh+YdSoUaioqHDeDh48KHUkWWtubkZmZiZWr1590a8//fTTeO6557B69Wrk5ubCaDTimmuucV6sifrncq8/APz0pz/t9jvx+eefD2BC+crJycGSJUuwa9cubNmyBTabDbNnz0Zzc7NzG/78n0f0YRMnThQfeOCBbvcNHz5c/OMf/yhRIv/x+OOPi5mZmVLH8FsAxE2bNjk/dzgcotFoFJ966innfW1tbaLBYBBfffVVCRLK2/mvvyiK4sKFC8V58+ZJksffVFVViQDEnJwcURT5838xPjtyt1qt2Lt3L2bPnt3t/tmzZ2PHjh0SpfIvx48fR1xcHFJTU3H77bfj1KlTUkfyW4WFhaisrOz2+6DVajFjxgz+Pgygbdu2ITo6GkOHDsXixYtRVVUldSRZMplMAIDw8HAA/Pm/GJ8t95qaGtjtdsTExHS7PyYmBpWVlRKl8h+TJk3C+vXrsXnzZqxduxaVlZWYMmUKamtrpY7ml7p+5vn7IJ05c+bgnXfewTfffIP//d//RW5uLq6++mpYLBapo8mKKIpYunQppk2bhvT0dAD8+b+YAb/kq7udf014URTddp146tmcOXOcH2dkZGDy5MlIS0vDunXrsHTpUgmT+Tf+PkhnwYIFzo/T09ORlZWF5ORkfPbZZ7jlllskTCYvDz74IA4cOIDvvvvugq/x5/8snx25R0ZGQqlUXvCurKqq6oJ3b+R5wcHByMjIwPHjx6WO4pe6zlTg74P3iI2NRXJyMn8n3Oihhx7Cxx9/jK1btyIhIcF5P3/+L+Sz5a7RaDB+/Hhs2bKl2/1btmzBlClTJErlvywWC44ePYrY2Fipo/il1NRUGI3Gbr8PVqsVOTk5/H2QSG1tLUpKSvg74QaiKOLBBx/Exo0b8c033yA1NbXb1/nzfyGf3i2/dOlS3HPPPcjKysLkyZPx+uuvo7i4GA888IDU0WRv2bJlmDt3LpKSklBVVYUnn3wSZrMZCxculDqabDU1NeHEiRPOzwsLC7Fv3z6Eh4cjKSkJDz/8MFatWoUhQ4ZgyJAhWLVqFYKCgnDnnXdKmFo+LvX6h4eHY8WKFbj11lsRGxuLoqIiPPbYY4iMjMTNN98sYWp5WLJkCd5991189NFH0Ol0zhG6wWBAYGAgBEHgz//5JD1W3w1efvllMTk5WdRoNOK4ceOcp0aQZy1YsECMjY0V1Wq1GBcXJ95yyy3i4cOHpY4la1u3bhUBXHBbuHChKIodpwM9/vjjotFoFLVarTh9+nTx4MGD0oaWkUu9/i0tLeLs2bPFqKgoUa1Wi0lJSeLChQvF4uJiqWPLwsVedwDim2++6dyGP//d8XruREREMuOzc+5ERER0cSx3IiIimWG5ExERyQzLnYiISGZY7kRERDLDciciIpIZljsREZHMsNyJiIhkhuVOREQkMyx3IiIimWG5ExERycz/Byj2Cg9lH7ClAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 600x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit(1, cbs=prog)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# learn.model.save_pretrained('models/sql_1ep_636')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16.106410496"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#SGD\n",
    "torch.cuda.max_memory_allocated()/1_000_000_000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16.106410496"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#adam\n",
    "torch.cuda.max_memory_allocated()/1_000_000_000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = \"Context:\" + eval_dataset[0]['context'] + \"\\nQuestion:\" + eval_dataset[0]['question'] + \"\\nAnswer:\"\n",
    "tokenized_prompt = tokenizer(prompt, return_tensors='pt')['input_ids'].cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with torch.inference_mode():\n",
    "    output = m.generate(tokenized_prompt, max_new_tokens=90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context:CREATE TABLE table_name_95 (tournament VARCHAR, score VARCHAR, outcome VARCHAR, surface VARCHAR)\n",
      "Question:Which tournament has an Outcome of runner-up, a Surface of hard, and a Score of 6–4, 6–2?\n",
      "Answer:Answer:\n"
     ]
    }
   ],
   "source": [
    "print(prompt + tokenizer.decode(output[0][len(tokenized_prompt[0]):], skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To produce better inference results, it would be better to use an instruct model and to ensure the dataset is in the format expected by the model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## fin -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}