REINFORCE(Policy Gradients).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import torch\n",
    "import gym\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Constants\n",
    "GAMMA = 0.9\n",
    "\n",
    "\n",
    "class PolicyNetwork(nn.Module):\n",
    "    def __init__(self, num_inputs, num_actions, hidden_size, learning_rate=3e-4):\n",
    "        super(PolicyNetwork, self).__init__()\n",
    "        self.num_actions = num_actions\n",
    "        self.linear1 = nn.Linear(num_inputs, hidden_size)\n",
    "        self.linear2 = nn.Linear(hidden_size, num_actions)\n",
    "        self.optimizer = optim.Adam(self.parameters(), lr=learning_rate)\n",
    "\n",
    "    def forward(self, state):\n",
    "        x = F.relu(self.linear1(state))\n",
    "        x = F.softmax(self.linear2(x), dim=1)\n",
    "        return x\n",
    "\n",
    "    def get_action(self, state):\n",
    "        state = torch.from_numpy(state).float().unsqueeze(0)\n",
    "        probs = self.forward(Variable(state))\n",
    "        highest_prob_action = np.random.choice(self.num_actions, p=np.squeeze(probs.detach().numpy()))\n",
    "        log_prob = torch.log(probs.squeeze(0)[highest_prob_action])\n",
    "        return highest_prob_action, log_prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_policy(policy_network, rewards, log_probs):\n",
    "    discounted_rewards = []\n",
    "\n",
    "    for t in range(len(rewards)):\n",
    "        Gt = 0\n",
    "        pw = 0\n",
    "        for r in rewards[t:]:\n",
    "            Gt = Gt + GAMMA ** pw * r\n",
    "            pw = pw + 1\n",
    "        discounted_rewards.append(Gt)\n",
    "\n",
    "    discounted_rewards = torch.tensor(discounted_rewards)\n",
    "    discounted_rewards = (discounted_rewards - discounted_rewards.mean()) / (\n",
    "                discounted_rewards.std() + 1e-9)  # normalize discounted rewards\n",
    "\n",
    "    policy_gradient = []\n",
    "    for log_prob, Gt in zip(log_probs, discounted_rewards):\n",
    "        policy_gradient.append(-log_prob * Gt)\n",
    "\n",
    "    policy_network.optimizer.zero_grad()\n",
    "    policy_gradient = torch.stack(policy_gradient).sum()\n",
    "    policy_gradient.backward()\n",
    "    policy_network.optimizer.step()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main():\n",
    "    env = gym.make('CartPole-v0')\n",
    "    policy_net = PolicyNetwork(env.observation_space.shape[0], env.action_space.n, 128)\n",
    "\n",
    "    max_episode_num = 200\n",
    "    max_steps = 10000\n",
    "    numsteps = []\n",
    "    avg_numsteps = []\n",
    "    all_rewards = []\n",
    "\n",
    "    for episode in range(max_episode_num):\n",
    "        state = env.reset()\n",
    "        log_probs = []\n",
    "        rewards = []\n",
    "        #print(state)\n",
    "\n",
    "        for steps in range(max_steps):\n",
    "            env.render()\n",
    "            action, log_prob = policy_net.get_action(state)\n",
    "            #print(action, reward)\n",
    "            new_state, reward, done, _ = env.step(action)\n",
    "            log_probs.append(log_prob)\n",
    "            rewards.append(reward)\n",
    "\n",
    "            if done:\n",
    "                update_policy(policy_net, rewards, log_probs)\n",
    "                numsteps.append(steps)\n",
    "                avg_numsteps.append(np.mean(numsteps[-10:]))\n",
    "                all_rewards.append(np.sum(rewards))\n",
    "                if episode % 1 == 0:\n",
    "                    sys.stdout.write(\"episode: {}, total reward: {}, average_reward: {}, length: {}\\n\".format(episode,\n",
    "                        np.round(np.sum(rewards), decimals=3),  np.round(np.mean(all_rewards[-10:]), decimals=3), steps))\n",
    "                break\n",
    "\n",
    "            state = new_state\n",
    "\n",
    "    plt.plot(numsteps)\n",
    "    plt.plot(avg_numsteps)\n",
    "    plt.xlabel('Episode')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode: 0, total reward: 13.0, average_reward: 13.0, length: 12\n",
      "episode: 1, total reward: 10.0, average_reward: 11.5, length: 9\n",
      "episode: 2, total reward: 11.0, average_reward: 11.333, length: 10\n",
      "episode: 3, total reward: 21.0, average_reward: 13.75, length: 20\n",
      "episode: 4, total reward: 14.0, average_reward: 13.8, length: 13\n",
      "episode: 5, total reward: 13.0, average_reward: 13.667, length: 12\n",
      "episode: 6, total reward: 13.0, average_reward: 13.571, length: 12\n",
      "episode: 7, total reward: 12.0, average_reward: 13.375, length: 11\n",
      "episode: 8, total reward: 11.0, average_reward: 13.111, length: 10\n",
      "episode: 9, total reward: 14.0, average_reward: 13.2, length: 13\n",
      "episode: 10, total reward: 15.0, average_reward: 13.4, length: 14\n",
      "episode: 11, total reward: 25.0, average_reward: 14.9, length: 24\n",
      "episode: 12, total reward: 12.0, average_reward: 15.0, length: 11\n",
      "episode: 13, total reward: 14.0, average_reward: 14.3, length: 13\n",
      "episode: 14, total reward: 37.0, average_reward: 16.6, length: 36\n",
      "episode: 15, total reward: 10.0, average_reward: 16.3, length: 9\n",
      "episode: 16, total reward: 13.0, average_reward: 16.3, length: 12\n",
      "episode: 17, total reward: 31.0, average_reward: 18.2, length: 30\n",
      "episode: 18, total reward: 26.0, average_reward: 19.7, length: 25\n",
      "episode: 19, total reward: 22.0, average_reward: 20.5, length: 21\n",
      "episode: 20, total reward: 14.0, average_reward: 20.4, length: 13\n",
      "episode: 21, total reward: 21.0, average_reward: 20.0, length: 20\n",
      "episode: 22, total reward: 24.0, average_reward: 21.2, length: 23\n",
      "episode: 23, total reward: 25.0, average_reward: 22.3, length: 24\n",
      "episode: 24, total reward: 11.0, average_reward: 19.7, length: 10\n",
      "episode: 25, total reward: 18.0, average_reward: 20.5, length: 17\n",
      "episode: 26, total reward: 13.0, average_reward: 20.5, length: 12\n",
      "episode: 27, total reward: 17.0, average_reward: 19.1, length: 16\n",
      "episode: 28, total reward: 15.0, average_reward: 18.0, length: 14\n",
      "episode: 29, total reward: 15.0, average_reward: 17.3, length: 14\n",
      "episode: 30, total reward: 14.0, average_reward: 17.3, length: 13\n",
      "episode: 31, total reward: 14.0, average_reward: 16.6, length: 13\n",
      "episode: 32, total reward: 45.0, average_reward: 18.7, length: 44\n",
      "episode: 33, total reward: 12.0, average_reward: 17.4, length: 11\n",
      "episode: 34, total reward: 12.0, average_reward: 17.5, length: 11\n",
      "episode: 35, total reward: 33.0, average_reward: 19.0, length: 32\n",
      "episode: 36, total reward: 11.0, average_reward: 18.8, length: 10\n",
      "episode: 37, total reward: 27.0, average_reward: 19.8, length: 26\n",
      "episode: 38, total reward: 44.0, average_reward: 22.7, length: 43\n",
      "episode: 39, total reward: 26.0, average_reward: 23.8, length: 25\n",
      "episode: 40, total reward: 33.0, average_reward: 25.7, length: 32\n",
      "episode: 41, total reward: 14.0, average_reward: 25.7, length: 13\n",
      "episode: 42, total reward: 21.0, average_reward: 23.3, length: 20\n",
      "episode: 43, total reward: 29.0, average_reward: 25.0, length: 28\n",
      "episode: 44, total reward: 12.0, average_reward: 25.0, length: 11\n",
      "episode: 45, total reward: 11.0, average_reward: 22.8, length: 10\n",
      "episode: 46, total reward: 11.0, average_reward: 22.8, length: 10\n",
      "episode: 47, total reward: 23.0, average_reward: 22.4, length: 22\n",
      "episode: 48, total reward: 10.0, average_reward: 19.0, length: 9\n",
      "episode: 49, total reward: 16.0, average_reward: 18.0, length: 15\n",
      "episode: 50, total reward: 14.0, average_reward: 16.1, length: 13\n",
      "episode: 51, total reward: 10.0, average_reward: 15.7, length: 9\n",
      "episode: 52, total reward: 18.0, average_reward: 15.4, length: 17\n",
      "episode: 53, total reward: 17.0, average_reward: 14.2, length: 16\n",
      "episode: 54, total reward: 42.0, average_reward: 17.2, length: 41\n",
      "episode: 55, total reward: 9.0, average_reward: 17.0, length: 8\n",
      "episode: 56, total reward: 30.0, average_reward: 18.9, length: 29\n",
      "episode: 57, total reward: 11.0, average_reward: 17.7, length: 10\n",
      "episode: 58, total reward: 23.0, average_reward: 19.0, length: 22\n",
      "episode: 59, total reward: 11.0, average_reward: 18.5, length: 10\n",
      "episode: 60, total reward: 26.0, average_reward: 19.7, length: 25\n",
      "episode: 61, total reward: 14.0, average_reward: 20.1, length: 13\n",
      "episode: 62, total reward: 28.0, average_reward: 21.1, length: 27\n",
      "episode: 63, total reward: 11.0, average_reward: 20.5, length: 10\n",
      "episode: 64, total reward: 41.0, average_reward: 20.4, length: 40\n",
      "episode: 65, total reward: 11.0, average_reward: 20.6, length: 10\n",
      "episode: 66, total reward: 14.0, average_reward: 19.0, length: 13\n",
      "episode: 67, total reward: 14.0, average_reward: 19.3, length: 13\n",
      "episode: 68, total reward: 11.0, average_reward: 18.1, length: 10\n",
      "episode: 69, total reward: 13.0, average_reward: 18.3, length: 12\n",
      "episode: 70, total reward: 26.0, average_reward: 18.3, length: 25\n",
      "episode: 71, total reward: 31.0, average_reward: 20.0, length: 30\n",
      "episode: 72, total reward: 19.0, average_reward: 19.1, length: 18\n",
      "episode: 73, total reward: 15.0, average_reward: 19.5, length: 14\n",
      "episode: 74, total reward: 44.0, average_reward: 19.8, length: 43\n",
      "episode: 75, total reward: 22.0, average_reward: 20.9, length: 21\n",
      "episode: 76, total reward: 13.0, average_reward: 20.8, length: 12\n",
      "episode: 77, total reward: 19.0, average_reward: 21.3, length: 18\n",
      "episode: 78, total reward: 11.0, average_reward: 21.3, length: 10\n",
      "episode: 79, total reward: 20.0, average_reward: 22.0, length: 19\n",
      "episode: 80, total reward: 46.0, average_reward: 24.0, length: 45\n",
      "episode: 81, total reward: 28.0, average_reward: 23.7, length: 27\n",
      "episode: 82, total reward: 17.0, average_reward: 23.5, length: 16\n",
      "episode: 83, total reward: 17.0, average_reward: 23.7, length: 16\n",
      "episode: 84, total reward: 10.0, average_reward: 20.3, length: 9\n",
      "episode: 85, total reward: 10.0, average_reward: 19.1, length: 9\n",
      "episode: 86, total reward: 15.0, average_reward: 19.3, length: 14\n",
      "episode: 87, total reward: 35.0, average_reward: 20.9, length: 34\n",
      "episode: 88, total reward: 18.0, average_reward: 21.6, length: 17\n",
      "episode: 89, total reward: 13.0, average_reward: 20.9, length: 12\n",
      "episode: 90, total reward: 11.0, average_reward: 17.4, length: 10\n",
      "episode: 91, total reward: 46.0, average_reward: 19.2, length: 45\n",
      "episode: 92, total reward: 15.0, average_reward: 19.0, length: 14\n",
      "episode: 93, total reward: 34.0, average_reward: 20.7, length: 33\n",
      "episode: 94, total reward: 39.0, average_reward: 23.6, length: 38\n",
      "episode: 95, total reward: 20.0, average_reward: 24.6, length: 19\n",
      "episode: 96, total reward: 10.0, average_reward: 24.1, length: 9\n",
      "episode: 97, total reward: 17.0, average_reward: 22.3, length: 16\n",
      "episode: 98, total reward: 27.0, average_reward: 23.2, length: 26\n",
      "episode: 99, total reward: 17.0, average_reward: 23.6, length: 16\n",
      "episode: 100, total reward: 15.0, average_reward: 24.0, length: 14\n",
      "episode: 101, total reward: 28.0, average_reward: 22.2, length: 27\n",
      "episode: 102, total reward: 28.0, average_reward: 23.5, length: 27\n",
      "episode: 103, total reward: 21.0, average_reward: 22.2, length: 20\n",
      "episode: 104, total reward: 11.0, average_reward: 19.4, length: 10\n",
      "episode: 105, total reward: 19.0, average_reward: 19.3, length: 18\n",
      "episode: 106, total reward: 30.0, average_reward: 21.3, length: 29\n",
      "episode: 107, total reward: 20.0, average_reward: 21.6, length: 19\n",
      "episode: 108, total reward: 32.0, average_reward: 22.1, length: 31\n",
      "episode: 109, total reward: 12.0, average_reward: 21.6, length: 11\n",
      "episode: 110, total reward: 31.0, average_reward: 23.2, length: 30\n",
      "episode: 111, total reward: 20.0, average_reward: 22.4, length: 19\n",
      "episode: 112, total reward: 19.0, average_reward: 21.5, length: 18\n",
      "episode: 113, total reward: 14.0, average_reward: 20.8, length: 13\n",
      "episode: 114, total reward: 21.0, average_reward: 21.8, length: 20\n",
      "episode: 115, total reward: 26.0, average_reward: 22.5, length: 25\n",
      "episode: 116, total reward: 12.0, average_reward: 20.7, length: 11\n",
      "episode: 117, total reward: 38.0, average_reward: 22.5, length: 37\n",
      "episode: 118, total reward: 21.0, average_reward: 21.4, length: 20\n",
      "episode: 119, total reward: 24.0, average_reward: 22.6, length: 23\n",
      "episode: 120, total reward: 17.0, average_reward: 21.2, length: 16\n",
      "episode: 121, total reward: 11.0, average_reward: 20.3, length: 10\n",
      "episode: 122, total reward: 14.0, average_reward: 19.8, length: 13\n",
      "episode: 123, total reward: 22.0, average_reward: 20.6, length: 21\n",
      "episode: 124, total reward: 16.0, average_reward: 20.1, length: 15\n",
      "episode: 125, total reward: 16.0, average_reward: 19.1, length: 15\n",
      "episode: 126, total reward: 36.0, average_reward: 21.5, length: 35\n",
      "episode: 127, total reward: 10.0, average_reward: 18.7, length: 9\n",
      "episode: 128, total reward: 19.0, average_reward: 18.5, length: 18\n",
      "episode: 129, total reward: 62.0, average_reward: 22.3, length: 61\n",
      "episode: 130, total reward: 37.0, average_reward: 24.3, length: 36\n",
      "episode: 131, total reward: 17.0, average_reward: 24.9, length: 16\n",
      "episode: 132, total reward: 22.0, average_reward: 25.7, length: 21\n",
      "episode: 133, total reward: 51.0, average_reward: 28.6, length: 50\n",
      "episode: 134, total reward: 25.0, average_reward: 29.5, length: 24\n",
      "episode: 135, total reward: 43.0, average_reward: 32.2, length: 42\n",
      "episode: 136, total reward: 16.0, average_reward: 30.2, length: 15\n",
      "episode: 137, total reward: 22.0, average_reward: 31.4, length: 21\n",
      "episode: 138, total reward: 14.0, average_reward: 30.9, length: 13\n",
      "episode: 139, total reward: 25.0, average_reward: 27.2, length: 24\n",
      "episode: 140, total reward: 33.0, average_reward: 26.8, length: 32\n",
      "episode: 141, total reward: 19.0, average_reward: 27.0, length: 18\n",
      "episode: 142, total reward: 31.0, average_reward: 27.9, length: 30\n",
      "episode: 143, total reward: 11.0, average_reward: 23.9, length: 10\n",
      "episode: 144, total reward: 26.0, average_reward: 24.0, length: 25\n",
      "episode: 145, total reward: 14.0, average_reward: 21.1, length: 13\n",
      "episode: 146, total reward: 29.0, average_reward: 22.4, length: 28\n",
      "episode: 147, total reward: 18.0, average_reward: 22.0, length: 17\n",
      "episode: 148, total reward: 39.0, average_reward: 24.5, length: 38\n",
      "episode: 149, total reward: 30.0, average_reward: 25.0, length: 29\n",
      "episode: 150, total reward: 12.0, average_reward: 22.9, length: 11\n",
      "episode: 151, total reward: 17.0, average_reward: 22.7, length: 16\n",
      "episode: 152, total reward: 22.0, average_reward: 21.8, length: 21\n",
      "episode: 153, total reward: 10.0, average_reward: 21.7, length: 9\n",
      "episode: 154, total reward: 40.0, average_reward: 23.1, length: 39\n",
      "episode: 155, total reward: 19.0, average_reward: 23.6, length: 18\n",
      "episode: 156, total reward: 25.0, average_reward: 23.2, length: 24\n",
      "episode: 157, total reward: 25.0, average_reward: 23.9, length: 24\n",
      "episode: 158, total reward: 16.0, average_reward: 21.6, length: 15\n",
      "episode: 159, total reward: 20.0, average_reward: 20.6, length: 19\n",
      "episode: 160, total reward: 17.0, average_reward: 21.1, length: 16\n",
      "episode: 161, total reward: 27.0, average_reward: 22.1, length: 26\n",
      "episode: 162, total reward: 15.0, average_reward: 21.4, length: 14\n",
      "episode: 163, total reward: 14.0, average_reward: 21.8, length: 13\n",
      "episode: 164, total reward: 12.0, average_reward: 19.0, length: 11\n",
      "episode: 165, total reward: 22.0, average_reward: 19.3, length: 21\n",
      "episode: 166, total reward: 14.0, average_reward: 18.2, length: 13\n",
      "episode: 167, total reward: 28.0, average_reward: 18.5, length: 27\n",
      "episode: 168, total reward: 12.0, average_reward: 18.1, length: 11\n",
      "episode: 169, total reward: 22.0, average_reward: 18.3, length: 21\n",
      "episode: 170, total reward: 15.0, average_reward: 18.1, length: 14\n",
      "episode: 171, total reward: 42.0, average_reward: 19.6, length: 41\n",
      "episode: 172, total reward: 17.0, average_reward: 19.8, length: 16\n",
      "episode: 173, total reward: 11.0, average_reward: 19.5, length: 10\n",
      "episode: 174, total reward: 12.0, average_reward: 19.5, length: 11\n",
      "episode: 175, total reward: 12.0, average_reward: 18.5, length: 11\n",
      "episode: 176, total reward: 36.0, average_reward: 20.7, length: 35\n",
      "episode: 177, total reward: 17.0, average_reward: 19.6, length: 16\n",
      "episode: 178, total reward: 16.0, average_reward: 20.0, length: 15\n",
      "episode: 179, total reward: 17.0, average_reward: 19.5, length: 16\n",
      "episode: 180, total reward: 28.0, average_reward: 20.8, length: 27\n",
      "episode: 181, total reward: 19.0, average_reward: 18.5, length: 18\n",
      "episode: 182, total reward: 11.0, average_reward: 17.9, length: 10\n",
      "episode: 183, total reward: 10.0, average_reward: 17.8, length: 9\n",
      "episode: 184, total reward: 14.0, average_reward: 18.0, length: 13\n",
      "episode: 185, total reward: 16.0, average_reward: 18.4, length: 15\n",
      "episode: 186, total reward: 21.0, average_reward: 16.9, length: 20\n",
      "episode: 187, total reward: 17.0, average_reward: 16.9, length: 16\n",
      "episode: 188, total reward: 75.0, average_reward: 22.8, length: 74\n",
      "episode: 189, total reward: 21.0, average_reward: 23.2, length: 20\n",
      "episode: 190, total reward: 53.0, average_reward: 25.7, length: 52\n",
      "episode: 191, total reward: 21.0, average_reward: 25.9, length: 20\n",
      "episode: 192, total reward: 12.0, average_reward: 26.0, length: 11\n",
      "episode: 193, total reward: 37.0, average_reward: 28.7, length: 36\n",
      "episode: 194, total reward: 45.0, average_reward: 31.8, length: 44\n",
      "episode: 195, total reward: 20.0, average_reward: 32.2, length: 19\n",
      "episode: 196, total reward: 13.0, average_reward: 31.4, length: 12\n",
      "episode: 197, total reward: 13.0, average_reward: 31.0, length: 12\n",
      "episode: 198, total reward: 28.0, average_reward: 26.3, length: 27\n",
      "episode: 199, total reward: 21.0, average_reward: 26.3, length: 20\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEGCAYAAAB8Ys7jAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAB+eUlEQVR4nO2dd5gcxZ3+P9XdM7M5arUSyhKSyEIgcjAm2IBtwD4bZ+N0zjY+R3zYv/PZ5zvbd05n+5wDNhjjAAYcMCCiCQIRhAAhFFCWdhU3T+ju+v3RXR0m7WzUrlTv8+hZbe9Md3V19bfeer+hhJQSDQ0NDY3JB+NgN0BDQ0NDY3jQBlxDQ0NjkkIbcA0NDY1JCm3ANTQ0NCYptAHX0NDQmKSwxvNiU6ZMkXPnzh3PS2poaGhMejzxxBN7pJRt+cfH1YDPnTuXlStXjuclNTQ0NCY9hBCbix3XEoqGhobGJIU24BoaGhqTFNqAa2hoaExSaAOuoaGhMUmhDbiGhobGJIU24BoaGhqTFNqAa2hoaExSaAOuoaGhUQZ3PLuTvb2Zg92MotAGXENDQ6MEBrIOH7j+SW55avvBbkpRaAOuoaGhUQK26wKQddyD3JLi0AZcQ0NDowR8+81E3bhMG3ANDQ2NEnB9y+24E9OCawOuoaGhUQLKgLsTlIJrA66hoaFRAop4T1ACrg24hoaGRikEDHyCWnBtwDU0NDRKQEsoGhoaGpMUing7k9WACyEWCyGejvzrFkJ8XAjRIoS4Swixzv/ZPB4N1tDQ0BgvKOlkgtrvwQ24lHKtlPJEKeWJwMlAP3ALcA2wXEq5EFju/66hoaFxyOBQCyO8ANggpdwMXA5c5x+/DrhiFNuloaGhcdARRqEcGgb8TcCN/v/bpZQ7AfyfU4t9QQjxPiHESiHEyt27dw+/pRoaGhrjDGW4J6j9rtyACyGSwGXA74dyASnlj6WUy6SUy9ra2obaPg0NDY2DBqWBHwoSyiXAk1LKDv/3DiHEdAD/Z+doN05DQ0PjYOJQklDeTCifANwGXOX//yrg1tFqlIaGhsZEwCERBy6EqAEuAm6OHP4qcJEQYp3/t6+OfvM0NDQ0Dh7CTMyD3JASsCr5kJSyH2jNO7YXLypFQ0ND45CEMtyTNpFHQ0ND43DFISGhaGhoaByOOGTCCDU0NDQONwS1UA6BMEINDQ2NwwpaQtHQ0NCYpFCJPNqAa2hoaEwyBIk8EzSMUBtwDQ0NjRKQqhqhZuAaGhoakwtOEIWiDbiGhobGpILe1FhDQ0NjkuJQ29BBQ0ND47CB1GGEGhoaGpMTjh99og24hoaGxiTDRK9GqA24hoaGRgloCUVDQ0NjkuJQ2pFHQ0ND47CCE6TSH+SGlIA24BoaGholoMMINTQ0NCYplHKiMzE1NDQ0JhlcXQtFQ0NDY3Ii0MB1GKGGhsZoIZ1z+OmDG7GdCWpZDhHIQyEKRQjRJIT4gxDiBSHEGiHEGUKIFiHEXUKIdf7P5rFurIaGhodHN+7lP/6yhme2dx3sphzSOFR25PkOcIeU8ihgCbAGuAZYLqVcCCz3f9fQ0BgH2I5nULK2ZuBjiUlfjVAI0QCcC/wMQEqZlVIeAC4HrvM/dh1wxdg0UUNDIx/KqaYMucbYwAlS6SdmP1fCwOcDu4FfCCGeEkL8VAhRC7RLKXcC+D+nFvuyEOJ9QoiVQoiVu3fvHrWGa2gczlAGJTdRvWuHCA6FVHoLOAn4gZRyKdDHEOQSKeWPpZTLpJTL2trahtlMDQ2NKBQzzGkJZUyhJsrJHEa4DdgmpVzh//4HPIPeIYSYDuD/7BybJmpoaORDhbfZE3Rpf6hg0m9qLKXcBWwVQiz2D10APA/cBlzlH7sKuHVMWqihoVEAtaTP6TDCMYU7wffEtCr83EeBG4QQSWAj8C484/87IcR7gC3AG8amiRoaGvlQdls7MccWEz0TsyIDLqV8GlhW5E8XjGprNDQ0KkLgxNQMfEwx6cMINTQ0Jh4CJ+ZEtSyHCMJU+onZz9qAa2hMQgROTM3AxxSHQhihhobGBIOrE3nGBYp463rgGhoaowZHJ/KMC8IolIPckBLQBlxDYxIiMOD2BLUshwhcV0soGhoao4xAQtEMfEwRSCjagGtoaIwW7CCMcGIalkMFYTnZg9yQEtAGXENjEsLVUSjjgjCVfmJacG3ANTQmIZTd1ok8Y4tDZUMHDQ2NCQSdyDM+CJ2YE7MeijbgGhqTEFpCGR9E58cJaL+1AdfQmIzQO/KMD6LSyUSUUbQB19CYhFAMPKsZ+JgiarQnYiihNuAaGpMQYS2UiWdUDiVEDfgEtN/agGtoTEY4OpFnXBDVwCdiPRRtwDU0JiFcncgzLpBaA9fQ0BhtOHpLtXFBlHVPxMWONuAaGpMQeku18UFUNXGl5MbHtrBhd+/Ba1AetAHX0JiEcHU52XFBfhjhtbes5uYntx3EFsWhDbiGxiSEjgMfH0Rlb9uVuDIsJDYRoA24hsYkhN7UeHwQ1cCzttfXzgSaNCvalV4IsQnoARzAllIuE0K0ADcBc4FNwJVSyv1j00wNDY0otBNzfBCVUFTS1GRl4C+XUp4opVzm/34NsFxKuRBY7v+uoaExDggSeSaQMTkUEZVQ1GQ5keLBRyKhXA5c5///OuCKEbdGQ0OjIpTa1NhxJf+7fB1dA7mD0axDDlEGrravm0jJU5UacAncKYR4QgjxPv9Yu5RyJ4D/c2qxLwoh3ieEWCmEWLl79+6Rt1hDQyPcEzNPQtmwu5dv3vUiD7yo37XRQEwDdxxgYjmOK9LAgbOklDuEEFOBu4QQL1R6ASnlj4EfAyxbtmzi3LmGxiRGqQ0dlKNNa+Ojg6hakrEnqYQipdzh/+wEbgFOBTqEENMB/J+dY9VIDQ2NOMpJKMWOawwP0VR6NTlOJL/DoAZcCFErhKhX/wdeATwL3AZc5X/sKuDWsWqkhoZGHE6JRB6lz+oEn9FBTAP3J8WJxMArkVDagVuEEOrzv5FS3iGEeBz4nRDiPcAW4A1j10wNDY0oSjFwZWQ0Ax8dRLsxZOATZ3Ic1IBLKTcCS4oc3wtcMBaN0tDQKI9oGKGUEp9glXRuagwPMsbAJ6kGrqGhMbEQNSLRkrK5CZhsMplxKCXyaGhoTBBEDUt0SR86MTUDHw1E1ZLsZI1C0dDQmFgozcD1Rg+jCaeIhDKR/AvagGtoTELEDXgRBj6BHG2TGcXCCDUD19DQGBGizDDKCJXhnkgscTIjaqvVRDmRQjS1AdfQmISIStxRBm5rCWVU4UqJZXgRPpqBa2hojApcN8q6izDwCcQSJzNcV2IqAz4BY+y1AdfQmISISygRBq53qx9VuBISpmcmdRy4hkYZZGyHr93xAv1Ze9jn2NWV5rvL18WcT4ciXFeS9A1LtqiEohn4aMCVEsuMSygTaXWjDbjGhMHqbV384L4NrHhp37DPcfeaDr5x14t09mRGsWUTD46UpBLe6xt3Yuo48NGEK8Ey/IlSa+AaGqWhmKR6UYaDwyWV3HElKcsE4ozQDiIlJo6Rmcxw3YgTU2diHr5YsXEvNz+57WA3Y0JDMcmRGHD1ck0kljQWcF1JylLarGbgY4WYhFJGA//DE9t4bAQrx+FCG/Bxwg0rtvDtu9cd7GZMaCgmmRmBAXeDRJZD24BHJZRiYYQTKVJiMsOVMnBilqsH/t9/f4HfrNg8rm0DbcDHDbbrHvKscKTIaQZeMRwXqpSEEjHWjqsllNGEKwnCCMtFoXQP2Aelz7UBHyfYjhzUe71pTx8/uG/DOLVofLGza4Bv3/1i2eiQUEJxhn0d5zDJRHRLMPCcllBGFdFEnrAWSuE2dgM5h9wIiMdwoQ34OMFx5aCs8C+rd/K1O16gNzP8MLqJirvXdPLtu9fR0V06OmQ0JJRAA55AoV5jASeigUeX9HpLtdFFJRJKdzpX9Ph4QBvwcYLtykEfcCbnMc+RSAgTFUF0RBlmOBoSyuGigbuRKJQYA5+A9TomM1yXiBOz+NjqHvAM+MGIfNIGfJzguBJnEFakmGdmBBLCREUgj5QZ5MrIl/vMoNc5XDRwKalKFEahRBm460q+eedadnWlD0obDwVUUgulyzfgB2PVow34OCHnuIMzcHvkcdATFWGKdxkG7o6cgR8uEkIsDjzGwMN+3tmd5n/vWc/yFzoOShsPBXgGvDCVPurL6U7bsb+PJ7QBHydUooEr5n1IGnC1tLfLOTG1Bl4pXBmJA49p4KFOqyS5Q30yG0u4koJUeoizcMXAdRTKIQxPAy9vVDK5kRuwwfCTBzaydlfPmJ2/FJRhLS+heJ8Zyf07h4kG7riRKJRIf4Vx4G7Qj4diVqqUku/fu55Ne/rG/Dr5USgQH1/dgYQygRm4EMIUQjwlhPiz/3uLEOIuIcQ6/2fz2DVz8sNxJa6MlwHNR6iBj81AkFLylb+u4fZVO8bk/OWgJq/yEsropdIP5m+YzJDSG0tVxVLpI9UIyyWeTHb0Zmz+++9r+duzu8b0Oo4rsczCcM0oAw+iUCa4Bn41sCby+zXAcinlQmC5/7tGCQTOtTJx0GOtgSt9tFwbxgoVaeD24Cy90uscikZLQd1aqogTM1oPvFzq92RHdKUxlvCKWXkMPEqs7KISygRl4EKImcCrgJ9GDl8OXOf//zrgilFt2SEGu4KXKdDAx2hQjrSe8b1rO7n16e3D+m4lZU6DOPDcyBN5DkWjpaDuLVUkEzOaSp89hCWUXAUrutGAVwslHgcOeQx8YOI7Mb8NfAaItrBdSrkTwP85tdgXhRDvE0KsFEKs3L1790jaOqlRiTY71gx8pHUyfvXwJn54/8ZhfVfdf7aMEzNXQajh4Nfxfh7KTkzXX0FZpsAQpSQUN5RQDkE5KSAEYzxRSwmJvGqEEO/zCS2hCCFeDXRKKZ8YzgWklD+WUi6TUi5ra2sbzikOCdgVaLNjHQeeC9jp8Iyb7cphfzdIMKkkDnxEGviha7QU1GRoCoFlGnmGJSQKahwdikk9Ybjo2N6bE9lSLao8xhm4SuSZgAYcOAu4TAixCfgtcL4Q4nqgQwgxHcD/2TlmrTwEEDLw0gNurDMx7RKZZJXCqSCbtNx3YTAJZeRRKIdDIo/yYZiGIGGIPAklnMAyB4GBSyn5zt3r2Li7d0yvExKCsb23qIQSRbRPgyiUiaiBSyk/J6WcKaWcC7wJuEdK+TbgNuAq/2NXAbeOWSsPAVSiP2fH3Ik5Mn3YduSwjUGuAvkmNwoMXMkLh7QT0783w2fgRffEdKMSyvgZlt6MzbfufpE7nhvb6JDxiveXEhJ+HHix60MkkWeSFbP6KnCREGIdcJH/u0YJDEkDH2Mn5nCN20hK4ippo5I48BGVk3XG58U+mAgkFEOQMEVMB1bGWkpIq0SecZzMgpVWGV/HaCBg4GN8nWgmZhRRKXHSJPJIKe+TUr7a//9eKeUFUsqF/s99Y9PEQwOVLO2VZqkSeg5GG8rBqSAZqRRyFUgouQqM/GAolkr/xye2ce8Lh47CpyQUwxAkTCPG/KLPti87upmY6ZzDl25/nh7faVcMlYSLjgZCJ+YYa+CRHXli1/fvU0o5ORJ5NEaGihh4bmIz8NwIJBSnkjBClYk5gjDCYpPUd5av4/pHx3+3lLGCslmeE1PEnmdUE+73yxKPlpF7dnsXP3/oJR7dWJqrVRIuOhqwx8FZLaVERuLAY9f3rzuQc7BdSU3SxJXj73vRBnycYFcQATLWmZhq0A03kmQkTswwE7P09+1RYOD5GriUko7u9JhNigcDqp9MAxKGUTJDcLQZuJK2ejPlGPjYkpDgOuMglamok2IGXPWzkk9a65LA+MeCawM+TgiNZ/G/u64clV3ZyyHnjIy1jEQDD2qhlLk3ZdxHFIWSN0l1D9hkbHdM68sMBQ+v38ONj20Z0TkCBm4YHgOPPM8o2+7zGfhoGTk1PnvSpTcccSp4zqOBcDyNHeMN4+2LRKH411dJPK21qdjx8YI24OMAKUPmWuplijKWMYsDD4zb8I3wcBlGRZmYoxIHrq7j/ezoSY/4nKOJm1Zu5bvLR7a5dRhGiKeBl2Dgamen0Qq1U+cpZ8Bz4yShhHLg2F0nGq4p8ki46meVxNNa6zHw8dbBtQEfB0TtZSnjGXVcjlkceKTU6LC+7wxeEnewa1caB15u78xKrqPa2dE9sQy47cgRywvq3lQYYTwKJaKBBxLK6Nx7bggMfKzjs0eaVVwJ1BAUwuvr2PX9cdbVny+haAY+Lsg5Lv9++3Ps7S29R+NooViqcz4yTsi6xzqRZyRG2M4rZj/Ua5cb4PGtwYbpLPW/pvpZ7UYzUTTwnOOOOMrILUjkiad4Vye8GimhhHIIauDjUAtF9bMhBKZvwJUcns/AW3wJRWvg44T1nb384qFNPLxh75hfK2owK2HgY6XXZke47AxKtQ7DIAylHvhgnysHJ89Z3NnjTdATpaCT7Uoyo8TATSEKJBTbCbda6x9tJ2YFDDyYqMd4xTMeUo0a5qYIJZSggJj/R9XHDdWWd1wz8PHBeGho4bWKL3GjiBrticvABw+FLP1dlXhRLg48PO9wQwnzmf5Ek1BUkanhSkQQkVAMQdIygs12wXs2hQx8dCWU3nIGvIKJejQwHht3qGsIQVAPRZXwzffp1CY9Az7edWcOewM+HppVRQw84rgcq8E/Ghr4cL8/FCcmjISBxyepiWbAh7vr0PWPbuah9XuAiITiM/DovdmOS5Uy4NlRdmLalTDwsZc2oucvdW/b9vfzX39dU3YDlcEgIxKK0sDVNnb5RdOqk4WlfccDh7EBH3sniEJcAy8+sMeDgY88CsWXJ4bRZ9GdYkp+JiqhDLMPnLw48I7uzIjON9oI6r0M0cD9373r+cMT24B4Kn3KypNQXBkY8P6MSqUf3TDC7jKZmOOVSh+sBkv0431rd/OjBzay05/AhwP1mhiCkhKKYtxq1aM18HHCeEooQ9HAhRjLMMKRxYFXUlFxsO+WM1w51w1elGEb8ICB+xq4/wKPVHceLSiZaKiOzKwThnC6sVR6EWfgbqiBKwY+WiRFTb4qPLEYxktCCaoulnifRrMwmmmIUEIJGHicAGoDPs4YqTEbCuw8jbIYlNGuS1ljqIHHQ+yGAilluGoZxvcrqwcuAy1xMIlh9bYuvn/v+qLnUD9dVwZOzJHqzqMF9QyGOklnbacgGcwUngau+lRKL8xTLefVYxrtKJSyEsoo7GtaCYLNP0pcZzQKoyn5RUQllBIaeHBcJ/KMD0JjNPYzpl0JA/cHWkNVYsyiUEZyz9FmD2eQVlQP3HGp8Y3PYH3w59U7+O+/ry0wylHn1r7+LLYraatXIV4TwYAPz7DknLCQWOjEJKaBq+eiNjsOrznKTsyMXXIyHPdaKIMkxo2MgXs/oxq46ttovHvSNEgU2fh4PHAYG/DxdGJWEAfuD7T6KmvMnZjDCwMM2zQcDTwI+yq3pZorqUt5DHywF0+dJ//5KQ3ccWUQAz6rudo75wSQUZRmOtRJOuu4wb1GnZheFEr8uVYl4wZ8tMa4eiaOKxkoESVUyUQ9GrDzJIyCvwfb8w1fjgzjwMP4b+VfiGrwlimCmuHaiTlOGFcJJcbASzgx/ReioSoxZuVkRyKBlKq3USkqqwfuUpMyB/2c93e/9G6eFBHV6Tv9NPpZLTXedyaAI3M4USiOK2OlfKNOzGSEgasxXcDARzmMEEqHEubc4hPraGMwpp9zhjdRRhH1NRRq4OGqxzJEUDNcM/BxwrhKKEOIAx9LBj6SHXkqkYHKfr/CcrI1SgMfJA5cMfB8oxzV+VWhoan1BydLrhiGU+8lf/OCaD3wqAaunkt1Mv5aj14iT3ie7hIGvJKJejRgD7KCDnetH/69K9NgFNPAIyuNhGkENcN1Kv04YTwllCFp4NWJCbknZixde0RhhOWjUAIJZVAGXpxhRbXJbDApJrzvTAAGHkShDMGJqe5RGSXlXFNx4K70no8ay/kMfLQmrhgDLxGJMhrOw4raMkhE1Kg4MSMSSn4YoZNnwJOmMuyagY8LcoGeNx5hhIUauJSSr/7tBdZ39gDhC11f5UWhbNnbz5duf35EiQj5yI1AA49+ZziDtJIC/B4D9yWUQV489fcCAx7RwFXooJoUJkJJ2SAKZQgyWb7cF5NQ/CV91nEjDDxfQine5y929PDVv71QcXRO1ICX2pVn/HbkCQlYsfaPZhihIQollKgGb5kiKDmrNfBxQiWJJaN2rcg1giI4AzY/vH8Dd6/xtvpSL3RdyiJjO9z5/C5+/tBLQTnU0WzHcKIS7JgBHz4DL8WsVcndSsMIQwZeWgNXL2+ljtHxQOhcq7wt+Rp31LAE0Q92GCeuHG3BNUs8r7+t3sUP799QUg4p1Q4orYGPlwHPFXmniv19ZE5M76dhFGZiBhOIKz0JxTfw4+0oP2wN+EgLOw0FThHjF1w/kpmXtAxSlrc1096+rHd8FI3OiDTwQV6YcnBcGZTmLPViq34JnJgVMvACDTySSh8Y8KrKZJnxQBiFUrlhyd8KL4gDjzDwjOOEUSgRA36s2AROtuh5VUZlpXVnco5LY7UnR5WKBR+MGY8Wir1TUYwuAw+jUFLFolD8vUmT5GjY92xYh3YccNgacOUQOlhRKGFKta+J5lxSphE4ScaihkeQSj+ccrCRiW6o7KqSMrHqMxWHERbRwN3YRBEa8NoxYOBb9w1P4rIjz7tSBHHeagKObOiQjDjP1DNSmZgz2M3tyWt5o3FP0Xaq7cDSFbYl68hg44KeEhp41LCO5YQZHY/FrjMam4O4RWqhJEyvMmG+Bp7q28Hvkl/i4offBM/+cdjXHCoGNeBCiCohxGNCiFVCiOeEEP/uH28RQtwlhFjn/2we++aOHsbXiVmogednJmZsh1QidIZ0+jU8RlO3HUkmZiXlACr5bqlqhOo5qCiUQZ2YSgOPGJ/oxOS4kqzjYBqCKqUTj2Jf3vNCJz9/6CV2DaHWRnRnpiFJKHljVRljQ0Q0cNsNzq3Suo8zXsIQknON1UVDP9Vu6ukKVwM526VZGfBBNPBoe8cCg1X4HI3t+cIolLAaoWl4cklUgk2YgoZbr2KB2MGBmjnwt89C/75hX3coqISBZ4DzpZRLgBOBi4UQpwPXAMullAuB5f7vkwa5II5zfMMInTzveC7ijEtZZvBCBgx8FFnMSMrBxl6YYbLOhClK3o+aXKp99jhoGGGwggk/l7+sztouSdOIOPpGr8aMqgM9lNVItN+GxcCLxIFHMwBVPysJ5VhjEwCnGWuw+7vhF5fCih8F51USykC2sn7JOt5mETVJc1AJBUpP1qOBeFRU4XWGWzQsiigDFz4DVzHfUV9LjZHD6HyWnzuXcOexX4P0AfjjeyHdNexrV4pBDbj00Ov/mvD/SeBy4Dr/+HXAFWPRwLFCIKFEXqpHNuzlB/dtGPVrldPAo1JAyjIKDfgovgTqmlIy5KV/7B4qYFbfuHMtq7Ye8D4fqdg2mAae8PtgsOJTmSIMPF+qytqeXyE5Bgx8wC8UlbVdOrvTXHvL6kHPH+23oWngcbnPiRgWtWKLMnAloRwjNgPQIPpJ3P0F2PwQ/O0z8ND/AtDlx8mncw5b9vbzxdueKzsuPLlAUF9lDerEVJ8fK0Rrx+eKtDk/dn44iEso3jHTEB4Dj9Sdn+VuR0iXF92ZdFYfCa/6Brx0P/zk/DE34hVp4EIIUwjxNNAJ3CWlXAG0Syl3Avg/p5b47vuEECuFECt37949Ss0eOcLQrHCQ3bZqB9+6+8VRd74UiwMPveRKE3V8J6b3SFRkwGganfjWW0Nk0dFU+kFWLX0Zm+/es56/Pbsrdq3qpOegLR414J0zYRik8mpcF0MxDbyAgTsy1qfZUVzSKwaesV0e3rCXG1ZsYX1nb9nvRGWMoTzXgiiUEmGE6vla/jL/WGMzqxInApB85tfQfjwsvhSW/zvYmYiE4nLv2k5++fCmspJQ1vb03rqURU+JbdXGTQMfhOmHRGk0olBCCcUyBKYpYr6sue5WANbJGd57ffI74XU/gb3rYdvjw75+JajIgEspHSnlicBM4FQhxHGVXkBK+WMp5TIp5bK2trZhNnP0USwuOWM7ZG03cO6MFoox8EIN3CWVMANjozCaLGYkkSRD0TZVBcB0TtWjjuvbxe5Jtc0y/foeFYYRZksZcCcioZiVRbYMBf25UEJR5x3IlQ/Hi4+1yttSEIUSKXNajIFbhqDN7GG62MezNaewzp3hnei098Mxl4Nrw4EtMQlFTUjl+ijnuCQsg/qqREkJJTo2xjJsc7C8hFJRSkOBYuAiIqEEDDwSBz7L3gLCZJuYHrZlxsnez+4dw75+JRhSFIqU8gBwH3Ax0CGEmA7g/+wc7caNJbKqGJJbOODUJgCjhajBCmZuO9+AOzEJJb9No9KOvEH/f/etZ8XGyvYEHYrxV/JPf1CPWkVHlK5zoga+ZXqMeTADp5bGUSkivkrwdn9PjaKEsnZXD//51zVIKQPdOGu7gdzTlynP9qKscSgGXH3WLuPEjGrglmlwvLEFgF01i1nuLsWpbYfjXw/N8wBw9m4MjHDGdgJJqBxhyDpepFR9lVXSgDuxaKXxcWIWu07AwEehnKwpBH6wj8fADRGLQplhb4HWBbhmMnxP6qd7Pw+2ARdCtAkhmvz/VwMXAi8AtwFX+R+7Crh1jNo4JigmoaiHPZTIgkpQjIFnijFwK2SLQZtGlYHHDdz/3buBv6zeWdl3K6ioqBAa8HwG7he9L/JSqZcwYQyNgZeWUFyytuPF5wYGfGROzDuf28WPH9hI94AdTE7RcEV1rBRie34ORUIJkkZKOzE9Bu4Gx4/2HZj76hfzP/aVbHvL/ZCohpb53uc7w1rq6ZwTk4RKtt/2klaqE2ZJx+d4aeCx8VjMgI+KE9P7GQ0jtEwDyzBiAQFH5DZD22ISRlgZEisJtVMPvgEHpgP3CiGeAR7H08D/DHwVuEgIsQ64yP990qC4hKIY+Oga8KIaeLDEUxq4F4Wi4sDz2zQayN96KxtZ/g+GWCbmIC+F6r+BvF3Rw11Lir1wIXusxIBHo3cUVN8mLSNI5Ik5MUdoUPr8++nN2qHk4DhBW4fGwIfgxLRD57NXlTBkhsUYeMKAc3mKrW4bVDdjY5FL1Hknq50CyTrsPaGzfiDrxCShku1wXBKWIJUwy0QTjY8GHmPgxSSU0QgjjCXyRDTwCAMXdoa23HZoOwrLFPHJpOGIMTfg1mAfkFI+AywtcnwvcMFYNGo8EEoohQy8c4wYuCEKU/hLxYHntwnghhWbaa1NcvFx04fVjvzY2dwQDHi0BvjgDNyToEIG7kehJEtvO5ULJBTPKA1m4DJlGLi3T6QMslujOvFIoGSGvowd6PtZewgMfJgaeNQQ5hw3UuaUgIFnbBcsz8g0bb2b2fI5/p9zVZDEFFxbCGieh9z3UnDOtO3GJKEoDvRn+a+/vsC/XXYM2UjhppI74RSRJMcCtuNiCI8lF3VijmIijxACv1psoIGrMdxub8fEhbajSJhGXI9vmAH7Nw37+pXg8M3ELJLUol6UsdLAU5YZGMKiTsxBNPCfPfgSv1+5bdjtsPMmKykr3ysyX18uh0BCyRVn4MUz5xR79MoJlGNv3vZu4cQXtlEZcDNk4KYRFNsfqUEJGHgmysDdINKhb5B46vz+rxT5K6dglW6EETYqE9PCZtqK/2CzmMlvnAsC2SrGDFvmYnVtCn71JJTiO9iv3LSfm1ZuZfW2rlhcfakJyM6bbMYKtiuD8TRmqfT+V00jKqHEGfhsx/M10LaYhGnE+6/hCOjePuzrV4LD1oDbQRhQMSfm2DDwqoQRRBDkh4aVNOCRl6BrIDeiZWn0Je7PDS1McSjapsoiVYy1QAMvk/psmaIsw1PnU5GesUzMSD/bkThw4cdLj3Rj4/4IA1eMNWeHq5j+Mpv9ep8dJgOPfNYuwcCztleN8ASxkWTXJn6VeiM2VlAcLMYMW+ZT1bsVA+/YQEQDzw+7U1ma/VmHnOMGYZmlVkjOEMbJSGA7svyKbhS2dotKKNFMTDMShbLA3YSDCVMWYZkifr2G6V5ST7Zv2G0YDIetAQ9roRTqkmOlgStmCJEwuEgceMoqDCNUL6+Uku50bkS79USNf6klcz7++MQ2bnlq25BS6XflOzHzolCKJVcoB18ikFDKa7EK0XvK18AzvgFXxwZPtHG5+rdP8ZafPMrX73ih4O9K4+7LY+DKWAzGwKNyXaUFpNQ1gnM4MnRi5mngOUdytB99si5xNBAWB4ux1OZ5GG6Oaezz2+JCuptXGI8H0qKCqpPSnc7hSm/CSJXpy7iEMvIolIzt8Lmbnyl4J23XDbc3K5pKX9yJeaA/y6d/v4q+QSZbKB5GaBkCywwZ+CL5Erur54GViiX4AJ6EAvz9kacqudVh4bA14GE1wrEPIwy02YRBQRy4HQ60KAP3mGP4uXTOe0GHupt5FLYjg7KXA7nKDPj1KzbzmxVbKk6ll1IWODGdPAZerviQZXhRDukyBi7a5mIMPGWZgZM22p+DsbGdXWlufXoHKzft5+cPvVTwd8XAezNRycENJpv+wSQUJ2R0wyknC57hikehhPKQ47osFltxk3XstdoBAgYeu/cWL5RwjtFBTdLr63d0/ZAfJ7+F2b01dm013lTSj4rqyTpu0YS3kRQ9K4YNnX3c+NhWHtkQD3e1HRlxihcZTyXCCB/ftJ/fP7GNZ7YNniGpbs8QBGGEpp9Kr86/WG6is2YRgC+hRBn4EQA8+MSqQa81XBy2BjyUUAp1yd29mWEVfCp9LcUujYJqhLbrvQhKQkn5YYTNNYmYlKCSi0biVbcdNxj0gRNukJcsnXMD4xCep3TfdA/YZGwv5TqoF5JXZKncktcyBTUps2xER8yAF9XADU/f9ys8AoPKMhBOakdOrSOdcwsmEdWm3nQuNgFW6sRUk1Rt0hrWhg7eOSSulAjhMcNohE3OkSw2tmJPORrL3zmmuAbuhRLOFR1MqUvR2vM8F2SXA5Dq3hy7tmLgB/qVAffkKCmLT+RD2ZSjEigJJ5+45Fw3lOSKaeAlEnnUMypVjCuK6ERpxGqh+JmYvZ1MFfvZXRca8Fif+Ay8Kt0x6LWGi8PWgBcrJ5v1PduOK9nbN3os3HY8g5ZfQ0H99Gone7WGVRhhU3UyJiWorLmRvBQ5Vwb1jCstxpTJOWQiWX5QPpW+I7KR8EDOwXVl8PlymqWaUBOmQW3SKrnrOcQnnXgmppJqvD7sz9qBRlxMQvnLMzu5/tHQYKkVwxFNVUBotBTUy39gIBcpheAG7RksjFAZmlp/045KEW13zvEYuKnKmxoRDdxxOUpswW07moS/0lLlefOjI/rMRq5O3MIVcjlv2vE1MnhVBlN9cSe5msT2+32RssKSx8XIhOPKsr6OKHZ1pfnsH54p2xfq+vnXsp1wLBctZhXUXY//LZiEhyChGHmZmKb/HsudzwCwt/4ogEIN3E/mqc+OXY7j4WvAi0gomZzL9MZqIHTEjQYcV3pLr4h2lokwBDWAU5GQt6aahKc1OqPHwHOOGxi3SjXwAWXAK5RQlHwyr7U2aG+wV2O5Ja9i4IZi4KVfsDgDjxpw72fSCiepmAaed93fPr6F6x7eFLtXgGmNngHf3x/fCEFp3Ht6w7HhMXDl6KuMgdekykfZ5CNfvnKkxPANtOHLKDnHJdG/i0bRj5x6bLDJbk1+GCGAYfJ/s/4HV1h8YuC7NNr7+DwfwpYG1XkGXGn1Bwa8voju/1hs7ORcWfY5R/HQ+j3ctHIrGzpLO/kCA563YslFJJRy5WTz+zlk4JUYcO+nEF7tdfDrzPjvsesb8AOKgRt5Ekqyhi7qaLb3DGsXrEpw+BrwIuVkM47LzGbPgO/qGsWtzFyJZRiYRhENPKKhpiwDw1+iNdUkSESW/d2jIqGEL1e6Qg08nXNidTbU/ZSC6re5UzwD3p+1CzTwYok8QTErn4F7k0bxtsXjqQtT6ZUjOObELCKh9GXs2EShDLiaxPMNuIoy2d0TGvBoLP2gTky/3XWpoUkomTwG7kYYOBCMk6aedd6BqccEK49SRm4N8/iXpu/wxZav84Gpv+a23KnslK3U5BtwJeH1RzXw0hKJ47qRcNHyMmSfb0zL1ZBREk7h1nnhdfITeaQMHb2FEkrlDFzGqhFGGbj3Hsudz7DVbcOtbgIoTOQBOmnhAvNJnN+/B3atHvSaQ8Xha8DzJBQpvbjh2S01wOim0wcMPFpDwS5mwL0BmbQMmmuSsWV/sP3VCJyYuYgGPlBiaQrw60c28ZdnvBT7dM5bIShjaojyGrgqZDW31etHFX4G5TXwoBCTKQJD319CRinNwEMNXCHKwIstp6MvcjpPQumKSChSyqA9UQOetUMJpVQY4e2rdvDrRzcHE4yaoCpFPIzQiwNXYW3q3rJOaMCNacdiGYKUZQRMPL/gU/dADrOmmZfqlnIgI8g5kq2yjdqBHezqSvOZP6wiYzvBRH9AOTEHqS0zFA1c9X0556+6fv6uQdHr5CfylCuopSaN7ko08EjRMLXisUzB/Nx63tj7a4xN9/O8nBMEBiRMo0CPv4MzyMgEYudTkO4e9JpDxeFrwPMSadRDn9lcg2kIdnYNjNq1bNcNUnDznac5RwbLVKUtvu/c+bz1iJ1cN/BRrt34NnjiusCYjIiBuzIioRQPswL41SOb+cMTW72iTTmHTC5aa9osq4F3DeRIWUawc8tALtyrsawGHolCUVUL+0toytmArYu8CI0wCkUhWcaJ2Zux6cs6AdMKJJQGxcDDlzydc4OohD29ITOP10Ip3t7rH93M9Y9sDia+2pQ5gkQeLw48Yr+D6Ie27mfZKVuwaluCkgTKuORPul0DORqrE1QnTA74K41tso369HYe2biH363cxobOvsBwqtVI0hSRFU7h/dpDkFDU6qfchhIh0Sh0YlYpB22e0Yxv4ZfHwP0xNRQJJUyllyy6+918cecHuXLgJtyaNm5yzgtWOwlTFEwm3829lvOz32T16+6BuWcNes2h4rA14PlhhGqAVCcNpjdWsXXfKBpwR2KZcQYexoHHJRSAj1+4iOP3/JVpbidJdwCeviFWH3w49co9Z2L4cpVL5EnbXmJHsGmCE4auVSXMol5/hYGsQ03SDFl01imshVIsDjyya0+tH7vcV0JTDjYrTsWZrKoeF60nU04D7/PlHXUOZSwUA49KKNG2dPaEq7NMBVEonT0Z0na4ElFOzEqfY9yJKYMVXXCPpsH07lUcufde/uKcFoQXegzc30U9n4GnczRUJahKGMEG2ltlG3XZPeTS/X5/2MF7oQhEtLZMMTJh+45yQ1RiwB3/OuUYeHFnZDyMUBb8DfxwzRIMvNSGFFHISBy4KWCm2EPjtnu5v/FyXl9/PXuveoB73JOCVY4XXhgfj2rM7e8b3RLVCoeNAf/d41v5wxOhvhfsbuLKQD4B72WY1VzDtv39Rc/z/I5uPv37VUMKMww18DCDK6jF4oThalHmyMb7eCa5lEeqz4Odq+jtG+BD5q3clPwS7g1XDnnPvVygD/saeBknZjrnMpBzYjp5LsJ6nTISykDOoTphUp3wWXTWLqgHPlg52cEYeLABcpVFqTBChVTUgBfRwCFcyism2FSTpCphBMw0vy3R5XxUAiumgau4+HTOCdpXm7JwS4Thlbtf8CUUGTfgtabDG3b8N93Jdr4rrwQ8Y5I0jSAaJd/IdQ/YNNYkqEqYQfu3Sa9ev9mz3e8PNwwjVBKKIZi34ddcZf696HN0XJeEXyUx67h8/k+rufJHj/DJ360q2O1nKBJK1Geg9hatLhGFko1MlAVOzICBD25Q8zc1Pl5sBGBF4yV0ydqY3wYKNfDopJPvTxktHDYG/IYVm7np8S3B78Uq84EXyjerpZqt+wsZuJSSL972HL9/YtuQsjWLauD+9aQMDUdgePa9BPs38VzVUtZai8BO07hvFR+zbma26MRcfyesXz6k+w/3S/QllDJx4Omcw0DWiRmq/qwT7AdYzvAM5ByqIgx8IOtEilmFWYP5COLADUFtsjIGXp9KFN3QIToRqperXqR5df8twRZXGdsJMygz4dZi4K0UmqqTsTBC1Zao4VRtidbdyL83VTdlIBv6EVRoX6VyWNZxUT7LnOs5MZVTDSn5TPZ7TM9u5m9zPkPW8HwPVyw9gnefPS9k4LFJwJuga5NWsCIDj4EDpHq8ZJ7+rB3EYXt9K5m/6n9Y9OR/8Cnrd2SK7MpjOzLYaKJ7wOb6R7ewausB/vjktoKNUvqGYsDLTNT5fR71NeQcGZs4AgZegRNTndb0wwhPMDYijQQdVfOxXRlbNXo/jZhDNZpHMNqbxCgcNga8N2MHRkvN4MqY2U6cgc9srmF3T6YgkeMf6/fw2CaP+ZbLFMyHx8B973WRGg09/mAK6qBsvBeANTUns8bwQpTO7fw1VSLHZ3P/jJuoha2PDun+82WMAd84O64sWE1kbJf+rBO7x/6sg2V6S/NyG0Gns06w8a36XnDtCnbkSZhGEPpWSpKIM/BiGnihhPKarhv4YPpn8KsrYGB/bAmt9NCBnDdJJS2DpppETANXbWn1tX3wxkouEgeu7jcKldWbjoRTquzISnXwrO2GNU3yJZSHvsOFufu4uemdrG04PdC8zz+qnfeeMz/ixAyfsXLG1qbMPAPu7YpY3b896I/oGDhJrGPGcz+iv3kx9WKA5N7nCtpqu17N8IRlBCTnhJmNQKHRDFc+5aJQCp2YarxafqGyfElPSXRKiiv2fCrTwJWE4oURniA2kplyDFhVXlnfiN8GlAYetiUdmXQ0Ax8h+jJOWIAoX5N1w2Vw0jKY1eI5sbblsfDv3L0u+H++V7wcbMctycAh1OMC5rjhXmiYwb6qOWxzp0DNFI7vX0G3rOYR91iy00+GLUMz4LkgyUUx43AAR9vi+lX8BnJOTJvsy9gFMlAxBBJKMQZerh6469IkejFvfi9Tt98NyJLMTL2Q9XnheEU1cNOA3k7O3X8Lz4sjoeNZuPUjsaSbvnQWHv4uJ2/+edDG5ppkTEJRn5/akAqONdYkgkxMRYjzJ53OyObU4dLeu8Zdz+/im3e9WPQe8++3OhnKBY70GbiU8NhPeDqxlJtr3xz4WqJQiT4xA55RBtwKSAxAJ03YIsHxO37P9xPfJtPfE5sglxre+N/+sm8CULfrsYK2qrGeMAU7/ZDS4xozvNJ4rCDyozIJRWng4Weicp5lGAUSihrrqpRu1IAr1l+JAQ/CCA2BieR44yVyU5cEJCafgVt55WSjY3N/v2bgI4JXwzkeARKNkc1GDXiztwzdGtHB+7M2T2zZz9LZTUB5x0s+bFdimQZmhL1GXww1kFMJAzK9sPF+mP9yEpZJ1pXB/noPuCeQw6K//RToeG5IO16re86XUPLbkok45OIM3A7rQJRxTg3kHKqTZqhjRzTwID64WAKII7nEfAKe/QNH3PEebkp+mSPWXg83vR3+9tnYZwMnZkkNPBKFYhnwj29jyhzXGlfDKe+FdXfR19cDQAN9zF/+Prjz87xsx09pTaQh08u8xL5A91X3ATC1vio41lidCDbFaKhKAIXZmNFwVGU8lGH5xUOb+PEDG0r2Zdg3biAr5VxPEjANAftfgu5tPFFzJlnXW1WaRvyVNoMolIgR8++lJmkGzwRAYrCx+jia09t4lfkYdXuejhGVE4yXyNUdgTttCZvdqTR2FDHg/mrzaGMzLz/wR1Jkef+OL/Cj5LfJ7o3XWalEQikW7hpN+vKSmIpHoRQLZwwZeCUauPfTEDDF3kGD6CfXviQoJxvNHgbPPxBtS5SBd2kDPnxIKenLhhJKYMAjrCYaCTLTN+Db9oUGfH1nL1LC0lnNwNCqyTn+oC7JwJUBtwx46teQ6YKT3xk63mYuA+AfwjPkPVNPBuSQdrwONfCQGSsUqy2Szrmxz/RnHY/xRLJJi2Eg61AVlVByoYSiJJhSYYRnms9BbRv9F/wn09nLSc99BdbcBk9cFxZnJsLAq3xnoKPkoHg0D/gG/PlbWdt0DuuddjjyQnAymFv+wUKxjVuTn6d15wOw9O2YOJxtPAd//TSf3/Y+0n1h3G7AwOsjDLzaY+CuneX81Bq8VUOc2UULoynWpwz4us7eojVX8pG13UB+8hi4b5hfehCAtdVLvWQrv2RDFOr3qGFRDLwmTwNPmgbfnPY/fP2o33n9e2BtbJwfLzaSnbqElGXwmHsUzXtXxp4LeGN9fuY5vpf+PJ+Rv+DB1MeZ1uMlsJg7VsY+G0ShVKSBRxzHEYd3wSYKhGO9mFQVTeQZLAooWvVx1sAaANzpS4NNjdU4tgInZpzcpGMMXEsow8ZAzsGNOAvVYA52SXfjDHxqfYqkZcQklLW7PMa2ZFZjcM5KYfuMKSo/RF+owIALFx75Psw5C2adEsYuH3M5/2AJq+vOBKCrZQkIE9beAfs3c+vT2/nVI5vKtiFk4HENHDyD+IP7NnD38x15gy7ixMvYgQw0mIRSk/TK4gqhJJQoYypeFdB2XE4Vz8HcczDP+AAvy36L3yz7PVzydbAHoCfcmkrF2talPNYbbPrrXydaU73G7YXubXTUHeP15ZyzwKqmdtOd/CjxTepEmuWn/Rxe/S0GjFpe6T4Iz/6RGqeHMzP/CF7ykIF7BjzhJxzlHJcr5HK+lf5//JPxYAEDjzq7e9I2QoQrEWUgBnu5c44MGbijnJjASw9AXTv7quaENVLynKxCxPMPovdSm8fAG2sS5FzJXprYIxto7FkX9G0Dfcw3dpFrX0LSMnhMHkUqewA2/yN2vZTdy3s2fYYuo5H/l7uKOgbYv+hK0jJBTeeTsc8GEkpFYYSRlVZEurDydGeIR6GoPoveu/B38hmseqR69okdj3Hx9u+zT9Yh247CNAwcJ+LEjCT5RPV4NfnUJE0toYwEgbMk58R2c4mGIUXrkRiGYGZTdUxCWdfZS9IyWNReDwxNA3f8RJ7BNPC6jX+Frq1w1seBMHbZbV3E2zOfpaq+1bu2UQPTl8DjP4HvnEDDXZ/k94+upxzyHXxRDTxru/zioZe4bdWOGBvcF4uDdkINvFwYoe/EFEIEm98qXVQIZcALv98wsJl29sO8c0iaBqZhss2aDVO9utbsDe8vysAhNOBu3j0CNPV439tfd6RXAtVKwbxzOGL9Tcw3dvHp3PvYXHs8mAlWp07iHPthcDIMJJq40rgnGDsqRLCtwZNQqhOmt0mE7XIRKwD4fOJ67J545bnOnjRN9DCV/fRmcjQb6aC40dFiM681HiwompUPj4GH/gOvmBWw6UGYe44ntdkuOd+BmI/8OtXKcNWkrJi/QElCadvhBXcWrX3rg/FwnPESAPKIk0haBg84J5C16uG618Dv3xnUXj3ffYhqt5dv1H2aXzmv5Cz3R+Re/b+slvNo2Pt0cC0pZSSRZ3AnZlRPVvdiGt4Gw/mp9NGEKdV/4I2P/qxDa603CQ8WieJKSJGl6Y9vImdW8+bs57GshBcu6MrYPq4QOrUV1Lic1lAV86eMJg4LAx5lRZlIqFd1JJMrjELxjs1oro4l86zd1cORbXU09m/hGutG0tnKH0jOUQw8DMHLFtPAdzwCqUZYeJHXFssgZ7v0ZGykhLa6lH8PDrz+5/D6X8AZH+HlfX/jX3q/WbYN6nohA49LKANZL3knqtvt643GQdteMlKRJWsUAzknuEZN0qTfz8QM041F0dDFOd1PeP+Zey5CeOy2P+tA65He8T2hAzlaUyR6b9FsUYWGbs9JeKB+UVgC9ciLEEjud07gPvfEoP+fTJzkfemIpbx45HtYZrxI79ZnvfvPOggBbXVJ/94sUobETO/jdGMNq+pfRg1p5j3xn7H7OtDVxR+TX+RXya/Sm7H5svVTTrrjClJk+UriZ/xP4of07olrw/nIOtEoFM+JOUduh94OmHduUOvccd0CBu71eXzS7Isw8GhfKUkok3NZK2fTNrCRebl13F/1CT5k3gqAmH4iScuggxZ+d8af4IyPwHO3wPPe318lH2BPajabUou9fqptpKE6yZPuQlq7ngfbk5RUhUsDd5BEHl/SK1LzRm3+kU8ogqJhyfgEr67T7juiB9PBXSlpE12IXB//mHYVa+VszMiWatF9XMGLRlEbT0fbPq2xatBJerg4LAx4NGRsIOtEJJSIEzOIA/e6ZFZLTYyBv9jRw+Jp9bQ88hU+YN1O1b41FV/f8ZlRfiamWgEobdTavQbaj0GFNCQtbxswVchKRUBkcq5XlP+418Erv8JdyQtZ6pQvlJNv3PI18P6c42XeldDt+rJOQSRNMaR9JyZ4E6Tq72i9iGJOzEXdK+ikFVoXAN7yty9jeyU5E7WwN3T2qReyNoinjsYqxxl47YG1kGokWzs9uFeOuZztLafx7/Y7ABEwwUfMk8mIFJz6PvYf+U8AiBduB7wJrDph+rKN5Gp5Pd/cdDnvHfgFlnBZMeMqfuK8iplbb4dtTwTXf+ve77LA2MlRxlbq+jZzrniKZHoPX7J+yUnGekwhqXvxllhfPLu9i8//aXWwosjabmxnHdeVHOl6jJiZy4I+jfZzFKrM6bW3rObZ7V2hBp6ygjGoJKGAgctZJGWGz/JL5rCLs83n2OxOxapvDfq3y2iCi76EnHose/90DVtfeJyTWcPqKZeQ9M/bWpckZRk8wyJMmQO/gl9fxuZC4wmeSH2AWX2rPQbv1wrpSee4+rdPsbc3E9YDz3ka/2f+sIoXO3q9+zIMX9KLj6d8CSUo9+tPXNP8VdRgkSiOK2nFCxRIJ5v9a4rgmmriSAYauPI3eNeLGvD8kMzRwqAGXAgxSwhxrxBijRDiOSHE1f7xFiHEXUKIdf7P5lFv3SghulQayBUvrhSNAwdor/dmTdtx6U7n2NmV5pT6PaTW/w2A+v2FMbClENPAIzVYgpTxjI1lgOhcA1OPCb6X8l9MlQSgGHg+g13HLJrpLpudGW5r5t1fVALqzXgp5fmx3yrFGvza2r6EUkwCUfcULfNZk7D8aoRusMysTpiFjGvN7Rzf+w/+nnh5MHkFDFwIz6jvDRm42ly3yhK82VyOu9tj2dEt1RRqDqyFqUcHFfRyjgv17fzx2O+zUR5BU02CXt+gbXeauGb+zXDiW6htnc4mtx2z03vOfVmHmqRFnZnly9YveHPuZhxhcZm8l21yCukpx/MD+zL6k1Pg758DKZHrl/Mq5x4eTL0MgDf0XE89A7hWDW+07qOPKta4s5m26dZYd9y9poPrH90SSFhZ22Ua+6mn35NQpOQIucv7cPM8X2qTdA/kAlkpCssw2NWd5oYVW7jnhc6iDFxJQl5msMtadxYApxhrWV51EV/PXcl37NcFGzqAP5EaJjtO/wKtuZ3M+u2FALww9ZJAymmtTSKEYF3Cq5nNNi9ypW8gyzXWjTSLXv7lwNfgTx+Cr82FzjU8u73b2xlp8/6AaGRslx0H0vxu5TbuXuPJVGpFmL91W+jEjEso/UEoaGUGXEqYIjwDPpBoAcJ64K4MiYMy3MphHJTnyIUSChTWlx8NVMLAbeCTUsqjgdOBDwshjgGuAZZLKRcCy/3fJySiJUOj1fGixXCiceAADdXei9CdtlnX4TkwX7bnt2Cl6JMpmruGwsCLaOC2DJZ4vRmbOdZ+L/qkPTTgqi3KkLbVRySUCDa43vZZ7NtYsg35se/xzZK98w9kHdLR9N++rL/xrQxkIG9XoeIGPJrJCB4D78865CISSn2V5a2IpISHv+eFCd76YTalFnND1ZuDc9WmrDATc8rCmAae86Mt5m/5A/+V+Bkz/vx2SHcVCSOUpPathfZjCiro9WVsUpZBY3UiVlRJpOoAb0ekF+Rsqv2VVn/W5vjEVo69+QLebt3NbTWv45uLb+ReZwk/sy+huTZJH9U8MveDsHUFPP5TnLu/zDY5hdvmfYGtbhsX2g9gY7LnZV8BYEX9K/mNcz7NPS/GSo12D3jt2etLWDnH5Z83fJjPW9d7Eoorme52QN00SNb4zm6Hju407Q1hmKOCZYjAmdo9kAs18GQYB16TtAImn845vChn4uI9s0daLuf/nCu42T2XhBFuEq36clPDKbw5ey1PzHkvX7LfTn/1EYGRb/VJR7q6nZ3JufDU9eC6mM/9gSONHfzYvYxWuQ9W/QakA1seDZ5H90Au5sRURExVg0yUSCwLwgjzJLZ8Bj64Bi5pFd6qoD/V6vdlWCBMtS1M5PEzQ+2wzRDWl1c11UcTgxpwKeVOKeWT/v97gDXADOBy4Dr/Y9cBV4x668rgS7c/zxObSzPOKKIp2elcREKJODGz+Qbcj+vtHsjxYkcvrXRxxKZbYclbWC3n09pduQFX6cWmGamF4rjBEq83Y3OM5WW/MfXY4HtqQOzxB2xgwPMcqC/a07zrdJZOCrHzEnmiUAy/P+vEwsb292X4aeJ/+F3yS1SRwTJF2UQexazVxFjjSyhOJMHkDFZx8v6/wV8/BXde6yXWTD2WH7ddC2aY5RgwcPB08ANbAv00a7scZW5n8dNfYbU7l0TfDvjzv8R2pQc4gr2Y2W6YekxBAabejE19lUWdkmrw5R+/fxqrk6xxZ1PbtwWyffRlHF7v3IGZ6eLKzBf4w5QPYqeaeFfus/zCuYSapEXSNFjZfKkXqvi3z2Dteor/tV/LjCmN3OcuAWC1WEz22Cu5NvduNhzzEe4yzsIWSXj4uwXPY29vBteVtLp7ac5s50RjvRcHLiXTnJ3QPBcInd0d3ZniBtwUQZ32roEcfRk70I+rI/4KFbaasV3SpNgqjmCNO5vu5uO980TKqkZry3R0p3nEPZb7ZryPn9uXBBM9hJmrdSmL2xvfAp3Pw0Pfpu2x/+I5dw431L6TTxjXwDtuhVQDdDwbvK9dA7nACGZsNziuDLhl+GGE+XHgbgkG7o+nWVWeNFpKA885Lp/8nSfVTMEz4JlEE6B2qFerWO980UQe7/pKQokz8LEoaDUkDVwIMRdYCqwA2qWUO8Ez8sDUEt95nxBipRBi5e7du0fYXA9Z2+XnD73EHc/uqujzpSSU6AYD6iErfa85kWOpWEdv9352Hhjgzda9CDeLOP2DrBXzaet7EZwcbH4E1t8NfXtKXt9L5CmsB64GWE/a5ijh12lRUReEk4mqsa2SSPIllPW5Vmxp4EQcffnIT+SJQi3t+vMY+ILexznffJpTjbV8O/F/tMl9LMi+wJv6bygq16SzcWmqJmDgrsdSejv55N5/42M934THfwqnfxg++iS8+2/sMKfHIihqk1YYU916JEjXqxHj38t7uBVpJHlX9rNsP+a98OwfsTL7vX7zz3OUv0M77ccGz1X1XW/GpjZlUZuyYlFKqu0N1RZr5GwEEjrX0J+1OdFZjTP7TB6TR1OTMGNSTdIyqEmZ9GdduOKHUNvGQP0cbnbOYWZzDff7Bvxx6ySmN9VinvoeXnHqsYiaVu5vvRKeuSnQzpM9m3mP+Vf29vSRdVxONDz9f4HYgcwN4LiSac6uwIAnTBEUIGuPZIoqJEwj2EWoO+0xcHWfVZHVknJ2KsP0KfkxPpS7mpa6quAeFVKWERhXlaykJAmVSg+eBg7eymu5eRa0HQ3L/x3sLJ/KfYApDdUst0+A+edB+7Gw69ngeXSn7UBCkdJbEUK4I5J6p/LDUnO2y1KxjvMeeRcN9AV/78vYvMxYxWvvfhnHiE0lJZTNe/v445Pb+Ptzu2gV3chkHRecMJerL1iIECJ4h6L3C2E4oZpQVD/Om1LL2UdOKfrujRQVn1EIUQf8Efi4lLLiyuRSyh9LKZdJKZe1tbUNp40FUIypUk0ptutKEQnFKynqdbYapCeu/Cy3pP6NY391HOes/Qpvt+6G+S+HtkWsMxaQkFn443vgFxfD9f8Ef3hXyes7qhqh8NirlDKWHt2bybGQrdAwE/zdPaJtUeVL2xsKGXjOcRlwTLbIqcjd5Qx4YZaigso4TMccLZJ3Z3/DdtnKV3Nv4mLzcX629+1cs+3DXJW5EZ74RcF5BgokFCuoB26ZAlb8EFPafDzxBXjPXfDKrwSad34aeE3KCisABpEo3grDzmU5R66ka85F7KGRfa1eglNT/2Z/6zqv384xViOtKph+YsE2YH0Zm9qkx8BVUsdAxAGbskzWibnedXetJjXQwQxnO8Z8T8+u8Q2eQtIy/EnHgbo2eP8DPHj2r7CxmN5Yxf3uEr5jv5a/p16JaQi+dPlxzGmtpbE6wc21b4TaqfC3z0C2n/fu+hJfSFzP8Y/8C9lsmhMNTz6yhEtT30sYTpYWd0+wu7yKnAJKSihq0dQ9YNOftYPVXyqQUMxg0wvFHB9Pz+IlOZ2WWm81mn+/qi/V9oMqVV4VswKCkL36KouejISL/xOmL+Hhs3/BGjmHtroU/X54L+3HQcdz9PpSQ/dAjrTtBpPvPt+AK0kxYOB5K0LbdXmN+Qgtux/jrebycMONrMPbzTsRSM4wnitpwFXyVW/GplV0QW0bJ8xs4l8uWuTfi9cfyslv5TFwZcAzthcVtLC9nuvfexpLZ4++m7AiAy6ESOAZ7xuklDf7hzuEENP9v08Hxm7nzjyoGbrS4PjeSBjhQCQzUEkoOTfPifn8bbRs+TvX2Rexef6bOWXvrbSzD057PwAbE75Bef5WOOZyOPV9XlLFgeLhYOGGDl53u1KlR3svUTrncqTcFNO/g7bgvSBJ06CxRiWuxDMkATbK6Rj7SqdlR4tFKQ1PBSyEDNwmk3MwcbjGupETxXp+wj/xQ+cyLst8mV82fIjfTvs0LxgLYM3tBdcIDbiAWz/CiemVXiq9I6kXA/D4T3mh6WUszx4Ps04NjDf4urYRZeBmKH21HQXJ+uCac3uepJ4++uZf6rW/ejYADf1bgkgZkFxgPIk791xPJ87TwHszNnU+A+/L2GRsb8MGZcAB9ienezH3Hc+yuP8pAMz551KVMKhOFjLw6qQZ1vuon0aH9F7Y6Y1V5LD4lv0G+qymWJ811yTpSFtw8X/B9pXw/VNZkFvHn53Tmdu5nMTyL7JEbCCd8M7V2vsiLbldGMiQgVthPxaXUMJ2dg3kfIds6LwEb7JN+eGI+T6W5hqPRRcYcCeUUCDU7r2CYF6bQgaeoCeTY13dKXyi6X/ZmvAmnyn1SaT0pa32YyHbg9ntvUd7+7I4rqTJH/fKcIfFrARVRo5ULl5SIutIlhlrAXi39Tdat90Nv30rNTsf4+XG0wCcYm0oqYFHt1NspRtq48RTOYrViiARKWblXT+MQolGRI0FKolCEcDPgDVSymiw8W3AVf7/rwJuzf/uWKE3YOCVOQWiDDydc4IOro6EEWYcb/9EYWfgr58mN/V4vmS/g0eO+hzfnPLv3Fp9BSx8BQC7kzPJiCqoboFXfRNO/5B38tW/K3p9xwk3NQbPiLgyDHNq4wCz3W0eA4kgysCbahIkTS+7MZpWrJaYL8npWF0bC1KbFexIzKqKFVYTiApTdKW3bP1m4gd8wPozv7HP554q756fkQtY3ngFK1tfwz3iDNjxVMGEpdrS0rsOnvo1r9/xddxsP7P6n+WbA1+AdBdPzX4nvVm7oDZ02i9Dq1CdNEMGnqyBpW/14o17Ojiu50HSpMjNPQ+AA6npIEyaBrYEOx8tFNuZbeyGRRd758uLf+/LONSmTOpSJr0Zp8ABC1BXlWBHagF0PMexuWfoN+qh/XiuvmARl584I2bQUqbBkplN3P/i7mDFdMB/wZUTC+JGEKC5NuExueNfD6/5X+jaxj+MU/hI7qM81vIaqlb9khONDWydfhEDJJnav442e6f/ZcXAw3MWM+DR9PrudI7+TMjAg5j9hBnscJTvY2nxdexk5DypPA0cQk3ZimngHgOvS3nO6zuf7+Dmp7azcrMnd7X58kx/1oFpntZe37U2dt7Gat+A90bfd8mMp77Nt7e9iV/vewvc8TlYd7cXppjp5Rixmf5pp9Imujh75cfghT9z7sPvxBSS3PSTWSrWldTAOyIbdkwR3VA7Jfb3er/v8hl4tP4PeLHrxXxOo4lKpoezgLcD5wshnvb/XQp8FbhICLEOuMj/fVwQSCgV1tgtKaHkOTFTpuHp2b27cM77Ag4mXQM57pXL+NPUD4PhfT6ZTPKb1o/Aldd5D7dlHsw+A1bdFGSkRZHzJQRlOINUZj+M8IPWbZ7WuvRtse+lIhp4c02ywPsfPddGOR3DTkP39qJ9oL6TiHjR1Usc9Y4be17kcvNhfmi/hn+130tNdSogyqp40D3iNO/AI9+Hv3wyiNFWRnDqznsBaMx18j33P/nM9qtpcffB639B/5QlSFlY67vHdyoq1Ca9KJSgXsUp/wxuDu76Aif1PsATyWUkq7yNk9OuCc1zaE5voUbkqNl0F682H/HuZ/HFsXtVY6FPaeBJj4Hnyz/gGZ1Nifmw9TEudB5kU/1SMAw+eN4CTp3XEmNXCcvgI+cfSc6R/OA+rz/29+eCdHX17POrBTbVJMNa0SdfBR98iH9xPgYIbq57M0ioERkOtJzIBjGH9v71tAcGfC4Q16ajtVoUosk9+Qw8YXqhoaGE4hT4WFS0Sv6KIxMYcE9yUJKE6dcogbgG3pO2g41SntpyAPAYOPjjeOrRgOCoPXfyZevn2F1euKAy4Pv6wroyx4uXaH/q22yuOpo7ExfAoz+AG/4Jfvwy5m/7E5Zw6Tvtam53TmftEa+F99xNJtHAvc4S5HFvoJ29WL07C/oKQkkIvDBCUcDAVXuyQR+CF7kEoTKQzrlUHWwGLqX8h5RSSClPkFKe6P/7q5Ryr5TyAinlQv9nZSEho4ChMvDejB10bkxCidRCCXYwf+5mqGkltfh8TEPQPZBjf382WEaCZ/jvrroI5p0bXuSEN8KetbAtXrAH4hs6qDZY2Fy55ctca13PW83lPFhzUZDEoqBemI7udCCfpPI25+2PMHAAdq8t2gfRTYPVC/1y8ylmiw4O9Of4pPU73mH+neN2/I6MtPiNeRngMWFlqCz/ZX9JTvfi1Vf8wHNG3vfV4L4AmrcthxnLWN96PqcZa3i86kw+0vwjOO51gZHuSdv89MGN3PLUtuD3hogBr0mZfqytf69TjvSiO565CYHDX2pfGzombRdaj6QlvZW3GXfQ9Ke3c7V1C8/KeYjGGQCxiB/1U0koAzknMOxRCaU2ZXFb9RWw7F2slgtYPe21sT6NSQqmwbwptbxu6QxuWLGF3T0ZDgxkafInXvUiJ/KqBTZVJzjQnwsmKqftGHZnvDasyzTTdYwXWtkz5UQ2ijlMS6+n3dnprQDrpgbXBo8ZqvuMtTNyzd6MTW/aDlZfQFD+N2EagVaujCZ4OnmtH2YYvV+1vZ9acXQXYeCKvddVWdiuZJ2fhLNlXz8pywiMYTrnQLIWWuazpOse3m7dzeV9N/lt8c4RzUt4tfkI0rD49RGf5+upj8DVT8M7/wKJGk5b/21cKbDmnM5Hcx/j3sVfgFmn8NOlN/Nh+2oSc04FYHpP8eQ3xfwFLi1FJJQ6JaH4hlrda5NvI5RdytguqQnAwCccVGp8dOCX/7zNFD8edSAioYSZmB4DrzdzXoGooy9DmAkaqxN0p3N09ecCAwregC+ohXL866GqER7+TsH1bcfljL23cO6aLwLezHyW8RzH7b2Df7b+isDlr81vK/ieck6lc24wAaUSZkyjVEZztTuPXKLBM6hFoBJ5Eqa3T2INab408F98xfoZ9b2b+Kj1J76UuI4z99/KHZzBQNJLXKiyzDDTzC8nm3NcOP8LXhr1krcE0sZA1qGNA1R3Pg2LL+appV/mXdlP8znzU+QSXny1Gvy9GZtfPbKZW57yilT1pu0gNR5CeSe6euI1/wtvupH3t93AxurjAodsxnahZQEtmW2cK5/AaZ7P9fYF/FC8IfhqXcDAneC8tSkrOK42Ko4ueWtTFi85U0m/4utcmfkCu9vPiT+fIhtHvGHZLLK2y/M7uznQn6PZdwCq8+Yz8OaaJLYrg009osv6vb0Zdp7yOd6Z/TTZpgVsNOdR63RzdvYf7ElMD3wIyoC0NxbKJ/nXlNKT5Goiff3R84/kiqUzYvfTHBnvVZZJTarQaZu1Xfb1ZQMHeZBRbAguPX4aH79wYXDfylC/4BeFA++ZKD9UEDL6ss/wp8a38RfnVF7Pcpro4XixHgs7kFAELq8yVzAw6zyyyUZvAm+eC3PPhtM/iCWzrJWzqGnw/AZq9bnfqcZI1iGmHU9WJJnd/2zR/uroTmMZgiZ6MYUMJkqFQAPvz/qbPXj9q7T6A/1hUMBYa+CF0/UkgHqpbVf68byJQT7v0FKbRAhvx5gCCcWvhfIy8SRk+7wUdaChymJfX5aejJ3HwI3CCnKpem+Z/+A3vLodUxYGf3JcybI9t9I+sJ6F4izSOYdXG4+QNes4v/8/qCHDguoZBe2Oapfq+qUYeB/VbFz4bhY//20vHG3mybFzRYvPm4bgNGMNCWzOMZ/FzvwSG4M73FO4xHycm4xXBZNbVcLwWETajqfSH3Wp92/Pei8J464vcOKeAX6UfN674KKLae9u5153KYkDA0zxtVn1rHrSOfb1ZWmuTXqRNDkn9hyjO/q0qoONM6BxBn3LH6IxZQYRFBnbgdYFpNwBTmIN2aM+zud3nsqUqvCZRbNeXVfSl3WCMEIIQ9PiEorJ9v12YJiirNTry0IDPt03oh3daQ70Z2ny2WNowPMYuHrp+7xNhpUjsL7KYm9floxRzX3uUq6yDO5JnccFyWc5pm8Fz9SchBox6trFQgij11Qyxp7ebGA4Ad7/Mm/lt2rrgUi7krC332+7z8BjYYQmBwZygXySNI1gdWOZghNmNnHCzKbg8/V5KyDwJsjocwZgyZu44ZHZdNureFXqMe5KfYa2jV2cnjyKW3tezTuslXRRy0yxh72LLiO5K682zxkfZuDhH/FQ+jiO8n1G0U2na5ImWEk2VR/LOQP3eX6cJi/rlIH9cNvH+OSe7Xx9xpfp2+b7ePI0cDXpdw3kYqubpup4dEo6NzE08AmH6CCoJJRQJW2oNG6VKRWrhWK7vMx9zFsuzTkLgIbqBJv9QRxlJCn/PFv39fOR3zwZht6d9gGwUp4Rj6DW7WHqgKeLvsm8l4GBfl5prmT7tPPZJqfyopxVdKaOMiK1AkjmGfBoJbc1c97iOVbvD90Rriv5xE1P8+jGvYD3MluG4BxjNTkSZGSCl4uV3OMu5SO5j/FPNT/npeSiWDalalvC35QitoXVlCNhwQXwzE3M6LyfKnLkjr4C2o9jVotXVz3nyGAyUoN/d0+W3oxNz0AurMYYZeDBtmqF9SNytksymtKdc2PykzzSLwaWF1cO/h6V/vOqS5mBYVfJITEJJWnRl3ECjbohz4DHGLh/LZVs1dmd5kB/LjDQKgY4YRRq4BD6IdS15k+ppSdtB2QlYRpkzAa+N+0/OMf5EXfN/1xBO9rrizNwdc35U2qDY6q2Sqn7aamNS4Y1STPmxExaBpmcEzj8ZrVUh9EhRuFYjvo3ZjZ7O17VpqzYzk0KvRmHtXI2f3VORSB5YubbOV68xH/a/8Nl5iO83/oLGZnAXnhpwUbCVDfzoxNu4ju8KfAZBXHg/qQN8NdZnyAps/CbN3rRTSt+DD96Gay5jbPcJ3hF237PgQkFEkqVX3ZAyjjJskyD+iorsEmZnDsmsd9RTEoDHl1WV1Iovc+Pe1UGPH+HGNt1yeZyLLVXebHehsrGS7DFN+CNUQZumWRyLg+t38Ofn9nJmp3+g65rg9M/CKtujIXZnSjXIJD0Vs/kdeaDNLz4RxpEPx2zXxV8plh8dnxJmww+F40SiNYV6ZFVcNI7vA2P/cJA+/qz3PzUdu583q8f4UdpnG2sZl31CfxVejXGb3TOBwQb+muoimyJVmWF4XKmIUgYRVLpL/8evPm3/Oy0v3Fp9r+8KolCcERTVeAAVSGUSufevLcP8HRTNSFHX3I1uRbb2DjrRwwZhqA+ZXlGr8Uz4L3UYM0+taD/DN9R15sJjWJcQilk4CrEUGm7DXkrvfyoDPBe7qaaBB3dmZjvpLSEEnd8qWvN842t2pYsaRlYfsLOjlwtqZqGSDu8a08tEoESvea8iAGPauAK0RVFdMWZsgzeedY83n7G3PCafhih2jIueu5iBbWiq6sLjvIkibpUdOemcByr5/PR3Ec5LfN9njv2U1yc/SrvyH6W0+XPeXXmP3hz9lrMmoZQ0ougWzQhzTD5KNhlKmMH4yrTvIiP2Fcj978EN70N/vZpqG6i5/Jf4ErBmen7mV/tF7PLM+AQSoHFVlShBu4Ufa9HE5PSgPdmowZ8cAau9M6qhMlA1i0II8w5kvb0BprlAVjw8uB7DVWJQJuMMvDqpMFAzgmuHd11hfP+FY5YCn/6MNz4ZuRjP2GZeAFbJHnmuGtoFr0sfuxadsoWuqafFbx8qSIzdayudXXUiVkYBw6ePMSRF3g1JTY/BBTuhp3Y8TinyNUsMraztnYZPzTeyP/k3hBkCvakvRohaqCnEmYwCJXxd/xkpLCjjoDFl9DnmH59Cv+eLDNghYoFqoG/yTfgXQO5wGjFolAUA88UYeCOG1xjakPKczo1ziQrkjxhLsGwwtVKFMogqwmjLiqh9BRq4HV+PZaKGHjk/+31VezsStM1EDLw6pISStzxpUI6503xfAY7D3gljRN+4Sb1uWhfDSqh+JOnOieUYOAxAx7RwBMmly05gsuWHBEcU4XW1Nif0xoa8GIlbaOrqwuObg+OhRJK+E57SVYmXkaC6REp2c4D7hKmNdfzrJzPk3IRCcPwa6EUJvIoZpyKxKv3ZUPnbWN1gvvs40l/fB28dzl84CF4/wNsbb+QFe7RLOy8i/k1pQ246v/8HZCaa5LxKBTNwAvRF5NQBmfgKuKgOml6tVD86mXRLdWOHvBLgM4/L/ieKmgFcUZSZXnnUcvezp40Hd1p3v/rlezP4NXqPmIJdK5B/PVTvMG8n47649h9xMv519x7WHny17ks8x8kkslgACTNwkcRzbBTL3o0Aw7iS8+BnAOzTgOr2tsYmdAgWIagWmQwr3s130r/PwA21J/CvkQ733NeixsZCqmESXXCu/fqRDQKJYykKVYPpT9bqPmpDaLVS62Y2KY93suRc2QgXxTTwD//p9V87Y4XYudU1QjBi3vu6E6DYfKzaV/kZ9VXIYQIdpePQmVdKslGZWIC7O4tIqGkvC3bOoN45DhrLaaBgzepbNjdiyvD56b6pVBC8Rl4X1xCmdfmM/DukIEnDBEYh7qIAU9E+qIYAgbeNggDj674auMMPB+phMdsO7rTtNQmY/6BYptKKIMnBJw6r4WapBmXUHJRCcVmelN18Hv03Gq7Q3VfxXZ4yjlh9cukaQTvSE/aDq6nVlNdtuVtWTjNy8Ho6EnzZ/d06npf4hW5e3EQUF2YQanGTb5c1BTZDHuixIFPOPRlwjjWwTRw2y+PWZuMSiguCeFQ/eh3OFZswnYlx6efYntijscmfUQZV3QQVSV8A96nGHiaf6zbw9+f6+CuNR3QMh+uuh0+vAK3/QSaRS87m5dimga/cS5gXfsl7KYpVjOiGAMvFhVQyolpGsJ7CawUzDkTNvoG3DdWn734KK49xQA3x32Jc/iJfSl7axcGhrApFnVgxJyYqh2WYQQvRrGKhNFiUArqhVMvdU3CRIhQQgHY7rPMKEtb0FbHa5fOIOdIfr9yW+ycWdsN+m1aQ1XAAp+uOY0Oc0bQH/mGpDZlxiWR6gSzWmowBDztO/DynZgAOw6kg89HEZ1085Np1AqjOU8DL2Dg1QmEgH15EorSq1VWYNL0JBRl6KMhl8fPaOTtp8/hrCPjzjYFZWRiGniyPANXGrgVKU2Q/9msb8Cn1qdi5yvGwJUBb6+voiph8tmLj+LKZbOC/lbjWO1Pe0TEgDdFyFNbXSpWf7s2ZcXqt4BHCtRnlsxq4u41HTy8YQ/P7egONiUPq43G7Udnd5q/OqfiVLcyO72WPQ3HBZJqsfspiOuvTsRKUxz0TMyJiJ60HTzgwTRwFTZWmzKDLb6yjsvZ1hoS932Z25LX8upVH+TY3GrWVMcjN6KaZ5SRVCe9GGUV/9rRnQn2z3x0w97wBFaK9OU/YbU7l01TLwrYqxqsiUjCw2AaeFNUA88z4EnToNav/Ad4MtCeF6Fre8DoXn5UG2+b77HeG6rfylfst2FZVjDAijmt1P8DBm6UZ+DRWiIKs5rjDNwwBHUpK2CWANv9vovKAlUJk2+98UQuO/EIDvRnY5JN1nFjum9nTxopJY4bXidhGgWrGuWUVCy2ucYLFT32iMYgKSNfAwfY4U8wBRq43y/RUDLwpAzV3NCJqRJn4i+8ZRo01ySDJJXuARtDEDiAoxp4wjQCSU/tBwreePzyFccVRMkoqGvObq0JfBLF4sVTRQhDKQapVoKqAmL0uRfTwNXkrByYV505l3MXtRUYcPW+HhEJiYzeV12VFTj0LcNgSl1hjLjHwL02fOyChfSkbd7zy5U0VFm866x5sXPmS4wd3Rn204D9iRfh3w7Q/i8PFr1/1f/5Y6y5JhFMshnb1Qy8GPoyNk3ViZjHtxSUXl6Xsqjyt/jK2ZJlxnpAcL17EVW5btYb83i86eLYdxXjSpgiqBwI4UBXL1dHdzrYveeRjXtjxibXtIDXZP+TrsajCspQJq3QyAwWhRJl4NloHHjWWxaq3W9uX7WD6zrme3+86wv09XaH99K5BswkuxNH+PcVsuuWPKdVsKtORANX20lBGFcehdoPM4qZviGKMpX6lBVLWA0YeJHNCJprEth+2J9CViVd4RnLnCPZ15f1N47wrmOWkVC6/ElfTYpnLAgCFWPPQRmdnV1pkpZR8DImIst0IaIGPDQ++RJKsQiNltpkEOPcNZCjoTpBQ5VXnnZ9Z09wrahhLLZxQylYpvCSZlJWEM5XjIEXc2KW0nBTlumXsE0zraEq9tyLMXbL9ErXqolJwTC86n4qmqon473P0xsjDDxiwGtTFs01iWDSVKn6e3tDP5Qd2Zno6OkNvOr46QzkHP75nPmB4Y6Wi45ily8JpRKWNzOLwsnI+34JBl6TpDtt+yv/sZdQJmcceNaLy26O6E0lP6sYS5VFdcKgo8uTUJaIddB2FF/Z8W52HjeP21ft4LS6lth31cNurE7GXlD1UFQZzc7uTKBL7+xKs3lvP3P95WpQgySaiekbo6RpxJwt+YjO7uUyMdUu8AM5h1uf3sF9awVvvvBzJB/4Kq/Y+DR/Em+hoepiz4BPWYyQvgPGCo2c5+Q1fMeLGY8DjzDw/F1HohgoMmAV44obnwR0FTLwfIYLBHHU+/uygUHNOXENHDzmZLsSw39OliEK+lRtEqEYuHq+Z8xv5ccPbKQ6YQb1riE04Du6BoqyW3X+/IkiasDzDWH+Cw9ezWxlwLvTORqrEwgh+PDLj+SZbQeY2pCivaEqZhiLTXalcMWJM5jbWosQgsaaBN1pO4j+iCJGGGrDFV8xJC0vGml3b4b2hsElFIBPXLSIJbOaCo7X+FUrIcLAm8I+jEpXdSmTpppkEH+tUvWjdVKyESc3wDWXHEVDdYJ3nT2v4Jz5EsrqbV0cObWOwRBEoRiFDBy8iXg8UuknpQHvzdjMaq6hqSYxaBRKbyRkLNDAbZvjWQezXoe5y9vmLOu4BS+8mmWjHnkIl9mK/Xf0pOnN2CyZ2ciqbV08vGFvYMCdIinsarDGJJQiM7Uylp6BVlEheQbc152TvgHf2+cZsodnvJvz3roM6/cf5qbUl+H5mZ4Bn3MG1m5/0ojIDDVJL6Qrnct61fYS0SiUUANXq4hKNfBZzYqBh32r2KNKLNl+YKCowYV4dtusFo/5uzLquEsFzyC6eXIxBq6iUPb3e5OB+vsp81owDVEg/0QllBkRTVYhUWL1FGPg1fEolGIOvil1Kdbs8lZK3QO5YCK7+sKFsc9F5ZehMPBlc1tYNtcjJ965B0ow8PD8QdRTCQau+k5KT8aqjkwI+TKRwj+fO7/o8epEuHmHel+n1KdImAKBCJ5l1vY2QWmqTgQTocqwjkoodp4Bn9VSw3+97vjYNdW73RWxH139OZ7d0cXHzo/3ezGUikJRKy5Vw1+n0heBFxbozcSDFbRSS6T2rtW8reNrfHPgWhbuf5BGemHmKV4igCvJ5JwCPUvN0tEIFIjrgqYhONCfY2fXAOcsbGNqfYpv3LmW1/3fQ2ze2xfWIIkw8FADFyWNABAkIsRjcs2CKBQloaRzTqDlPrJxLyy8kG8sup61zIP7/gu6t8HUowOWGp1Aqv2iS+r+1AtZnQgzHqP3kO/5h+Ia+PTGqlgdGAjZy/w2j+l0dKepr7JiqxwFxQRVxI/KKFUGRG1y0dmdDmrOqLbmP0+v8qBNVyTBxjtuccLMxgK2pAx4OucWODCjbci/jppUhAjHUCihFGHgdcnguXkSSnHjHGV79anC9lQCNTkU08CjE1510qQqYVBVioHnOW0rYeClUJM0g2zXvkiIZ0NVIli5qPejLmXRXJMM+lH5bmISiitLTiIKIQMPI9pWvLQXKeHMiKRWCipiKn9CVuNKrc61E7MIvFKgnhY2mITy+5XbqEsKjnrwIxzX/SCL3Y28bfu/e3+ceWoQhqSSQ6JQy+amPAYe1QVVAoMrvZC5T71iMUtnN/H8zm6+ceeLAVM1/Q2BIa6BhxJK6aVq9PqqYpyCSg9WDlq1lHzEd6buzZjcWvUa2L/J+0Lb0eEmrFaogXsM3GfdsSiUsBaKaYYlcYsx8GIauGUaXHPxUVx+Yhjdowb/vNaaoO9KlUMI05O9iVgZcvVspioG3p0JN44APnz+kVy5bFbsXLUpi3TOZU9ftuCZXn3BQj748iNjx6JRMcUklEADzxs3U+q8Co6N1YngmVeVYeAttUkO9OfIOS7dabukMzLYOMAQJbXpwaDOXVuEgUeNcsoyY3tm5iPKzNsbUnENvIjOXw4nzW7mgRd309GdDpOskhaN1Ymg39T7UZu0uPKUmVx9obe5Qo0/0UQZeNZ2i+rwUSRMb4xHNfBHNu4lZRmc6EeqlEMQRlgkDhzC6CHtxMyDlJK+rBfX3VQdenyL4fkd3fxl9U6uPaEXo2cHd8/7NFfbHyMhc/RQC1MWYRkiSKXPN6KKreS/7NHBuri9Pvj/rOYarjxlFj+96hTeeeY8bn9mB8/7WZpWpB54XAMvzcCh0IArDVw5SgdyLtV+iOSBAS+rsTZp8uz2LrrTXpLMyvrzocZnFVOPDmSQqBOzJmnFjHZBLRQIdqWH4hq40s/z8c/nzufkOaF/QQ3+KHOrK8IIoTDRRYUMKpabskxaapN0dKdjGvhbT5vDmXlhdeoa2/f3F6yqzls8lbefPid2rDaS7FJMnw8YeN6zS5gGrbWpmPOtVBQKhBv/7u/Legy8xGSmdN9Sq5VKoNh9TRkGbgivnVEHdsFnyzDwYjp/OXz45UfiuJLv37s+lmRVHzHgaiKpTVmcPKeF9/h6thCeI3NPHgMvlleRj4aqRCwK5ZENezllbktF2ZOhhFKCgXdpBl4U/VkHKaE+4TJHdNCW2Uy6SLo1wPfuXUd9lcXrkivAqmJb23nc4yzhxqb3cXPV68DfkiljO7iy8EVUgz3/ZY/qWounRQx4xMP+/nPnU5u0+MwfngG8ZaUynMU18BIG3DRicbApy6vBoKSEgaxNTcKkKmkGzsCLjmnHlfDYxn10D+Soqan1Np1omAFNc4LlZ9IMdeLqhBl5WeJyihqEpl+NEOB3j2/l0u88yOXf+wfPbu8K7quYtpoPpT+21oUJIKU03fwKb+rFiOrMU+tTdHRncKUsKlEoKNlg+4GBWJ+W/HxE1y0mayRLMHCvfanYNUrFgQNM8WWAPb1ZTwMfhIEPxYGZDzU55K+UIDRGVQkTIUTAboshauxba5Mxp2i5Z1AMs1treMOymdz42BY27vHi52tTJg1VVlEJJR9T6pIxJ2Y0jLAcVLVR8Gp7v7CrJxaRVA5BHHiJ2jbjxcAnnROzL2Nj4vDWRy+jNtPJu1PQ/z9fg6Mv8ooqLXg51E5BSsnDG/by6uOmknrxdlh4EVZNA7Cd/+55BafMbeYqvJdChajlv4gpy+TTr1zMeYvzi9mEn1MG3BDxnVeaa5P8xxXH8edndlKVMDhlbkvAEgIDbkUSeUrM+h85/8hYAkawNZgv+agolCCRB49N/unpHbzY2UPXQM5Lcz7nk3DWxyHCopOW4W1iQaGEcvqCVt555lyOn9EYxLZbEQnlxse2kDAN9vVnuev5Do6b0Rjo8YNBvYSttSkaqhLs7EqXNOAJ06AuZQXx/ir2fmokbbzdjwW3HRlMksUQ1bSbShjJKFT9lP6sU1TWKKWBA3zwvAUIIpFLkXIE+VAM/IVd3WRsN6homA9lYIerfwNcsXQGLXXJojp10oobyw+et6Bg9amgPjOlLuWFCI5AAwd411nzuPGxrfzlGW+Thboqi3eeOTeYuAMJpUgJgJbaZFAOGFTxtAoYeLUVVH9c3+nVKT9uRmNF7S2lgTdUWZiGCDRwbcAVpAQh6M3YLBLbqM10Ik//ED98PsGRvSu58MU7EKtuBAQsvIiuE97Deen7+dCuZ6C3A459HdV9Xmfu68tyxnxvprUMEdRhKLbc+XCeLgpx9jK3tZakZdBWlyp4mFcsncEVS8MyscoIRSWUZJkwQoC35S3rgxrYOYe6lBUYzehLM6ulhpbaJFv3DdCdtj32KASYcdYQl1DCwkKphElDVYIvXnas/3sYhaK+2522eeOyWax4aS/rOnuCTYErGbD1EQaumG25ksBNNYkgWqCjOx2L/wWP7b6wq5um6mRZ9lcfYW/5kUWlUJvyNiouJmuYhvDlhsJn9+oTjoj9rgxcKQ0cQr9FVJaLQt3bSBj4cTMaSxqpKAMHeN1JM0ueR41DRVqi70QlxjMfC6fWMaUuxZZ9/SRMQcoyg5opEI7BYvfeWpdibaTOuFcrZ/BJRJEHgK37vDwOlXg2GBQJyb9XIQSN1YlgUwhdDxx49Jf/SsOOBzjmX/9BX8ZhieGVZhWn/jNLFjbwlp+sYH5VinOmbufahVtIPvFTmtbdybeTkO2dAud+Bo6+jOpVu4JznrHA00ctwwg2PS62FC6GqJFqrknQ3pAqGmaWj+iOPBCPQqn02mpABBXWfAZuRDTRKXVJZjZXs21/P90DuQL2aBYx4NVJK1KBsHAlotofnSgWTatnX3+Wtbt6gg0uKnGu1fnGsLU2FbStlAYO+OGioQY+tT6Vl/lYxe6eTGzrsmKIRl5UIqGodu3uyZSUNZKWUdGzKxcHrrIJH/YN+MJSBtwfKw0jMODlEK2oOBjyo4CSlje525FIoKFACMEZC1q5fdWOslmixcZJa12SPX1etq4QXlhwJY7UxuoEazs8w791fz9CwIwKDXipVHrwxqti9NqJCaRFkmOyq8nseoGeTI4lYgO5ZBM0z+PMBVP46PlHMndqI9dtmcJPrDfDx5/hriXf5hWZr9H9oWfg/GvBtAKW0FqbZFG7F8JmmSKYfaeWqKecj+hDaaxJcPUFi/jnc4rHuEYRxIFnHS/GVZQPIyyG6O7qriv90D0r1qbWuhSzmmt4saMH25UF7DHGwItIKPmDLqiFYsbriyxur2dxez2b9vYHtUTmtNQyGM5b3MZ7z57HUdPrg7aVi2uOVnjr6E4XlE1d2F6PK2Hzvv5BDHi0OFilDNz7Tuk0daOiZ6cklPwt1cBjgpYh2H5ggJbaZGDQ86FWa+Umu5FgKGOxWAVERQCGqoErqFVxsUJbKcvEEMW1+ym1KbK2GzhAK5dQEkEUyrb9A7TXV1Vc/lXJWMUmiveePZ+XHzWV1y2dwbFHNBT8fTQxKRi4c/QVuBu/zYHHb6JvwQc40djAwNQlJHzW+clXLAbg3b98nB8/sJG3nzGH5e7J7KnpYEpDaFDUruenL2gNvPiWaQQxuKWWrvlQg0gl2Lz+5NJLzSisSCq9GmDlEnmKIbqNWNoON+JVBDxleXVRZrZU85fVnuaezx6VTpy0RKDBq30R1Tni1wydmDEG3l7Hvv4sjiu5YcVmAE6f38JgmFKX4vOvPibWtnKyQFNNMqg109mdYU5rPB1bXVPK8sajLiahVM7AoXgUCnh9UxEDV8atCGMzDEFLbZLOngyL2utKRpiEO+sMXwMvB/V8KxmLqcCAh5OpiuceLISvFFT8dbEJSu0KVKxvVDbmvr4s9VWJIUgoFj3+Dk1b9/UHlTMrgRqvSavwOm85bTZvOW12xecaCSYFA58zdwGPy8VUv3gr6b4uFomt2NNOKvjcJy5aRNdAjp//4yXWdvQELFtBGV4100NY3rM6YQZp34NBDd5KjYCC6Q+qff3ZiHRSXgMvde3ejB0kBKk4cFAxyCJWdjOfPYZRKGGMd3XSpMYvIZvPwEMJJdTAm2oStNWngj6+49ldHDWtPnDIVYqGIAqljAZeHZFQetIFZVOn1lex0E9/rlRCaayQgStjUo6BVxKyFjgxS3xW6eCLypCI0YhCGQxJ06go/bsYA1c+lOEy8DmtNUxvrCrqqExZZlFpBcK++9NTO7j0Ow+yty9bURRKQ3UCKaEnY7Nt/0DsnRkMpu/gHmrM+2hj0JEghPg58GqgU0p5nH+sBbgJmAtsAq6UUu4fq0bOaa3lenkmp/X8nCPW/xZTSETeno/gOWheeWw7P3vwJRwpC5jxkplNvOusubwm4mBSD3phe12sDkY5GH5qb6XLcIVpDVW87fTZ7O7JcOKsZmBoy1bw7jFpGvzhiW186LwFgL909cOyFRuJOmPy2aOaSBKRMMKapEl1srgGGqsH7rd30dR6hBDMn1IXaJ+nRybGSqH03PoyskBzjRevO5B1ONCfK7pxwRkLWlnX2VvWgA+HgSujUSo78uoLFsY2MyiFhe11vO/c+ZxdouSrlxLeU9aAR+PAxwoJU1Sk2x7ZVsc/nzMv5mgMNq0YpgEXQvCvlx5d9Bm+6dRZJTMkVTr9/96zjtbaJBcd3c7lJxbuMZsPRR729WXZ2TVQsQNT4TOvXMwJRWq7jCcqGQm/BL4H/Cpy7BpguZTyq0KIa/zfPzv6zfOQMA3WNL+cgZ4bOXmtt99kcvayop/9+IWL+PtzXgnI/JehOmnyb685tuDcxT47GKr9rbOGAtMQ/McV8ZoM5crJFsO0xiredOosfrNiS7A1VY1f3hZCNhKNSc83PoEGboX6rWfA/SiUvMkkrAcepsQvmlYX/G3elFrWdfZWlIKcj4ZB4sDBk1CkhHV+Zb5iGxecuaCVXz2yuawBT1lGsKPQUKJQoLSE8qZTK1sqJ0yDf7306JJ/VxNvNK8gH4pslJvsRoqkZVZEJizT4NpXHRM7pnwow3FiKrxmyRFFj5+5YApnLij+HdV3jiv5f685piACqBTUM127qxtXhpUzK8U7z5o3+IfGGIM+KSnlA8C+vMOXA9f5/78OuGJ0m1WI9umzeEfimzww6wN8I/d6qpumFf3c0dMbeNUJ04HKjLIabJXq3wpVCaPiSIZySJjeztmVaHYKH375kZiG4BO/exqISygqvG5GiR1NILznZF4UitqpvBQDj2rg0b5d1F6PEHDavKEb8EqjUABe2FXagJ82r7WgLnc+hPDKAgtRuY6s2jWWrBfC57ZoajkDPrYaOHiO0uFGTqhw1uFmiQ4XirQcNa2eS4+bXvH31Nh7ynfAVyqhTiQMd1S2Syl3Akgpdwohppb6oBDifcD7AGbPHr6wv6i9jttWNfIpeRGzZtbwyTIv6ucuOYqp9SlOrGB5ozSshe2Dl5CM4l8uXBRUHBwJLltyBK21ySEN+vaGKr58+XHc80InNUmTk2Y38+z2biAMSatKmEytT9HZkykZhZK0DC44up3O7gzTG6o4b3Eb7z93Pgva4ve1ZFYT7z17HsvmtpCyDN579jwuPT58Ud511lyWzm6qWFeO4owFrbzn7HlFy4wqKLljbRkD3lyb5NpLj+akOc1lr1eXsjDynLHlcMWJM2itTQ7bMVcpXnfSDNrqU2X7MNhXdAwZ+McvXFRQs7tSVCfMYcsnI0HKMvnkRYs4Z1FbxTIowDHTG6hNmtzw6BYgrJw5mTDmUShSyh8DPwZYtmxZYQGNCqEYX2dPhm9eeWLZz85srimQSkpBMd9yS9diqHTpPBiOOaKBY4YRanTlKbO48pSwWJPSr1sjIWizWmro7MkUsMdoLZQZTdV86pWL/e+m+FyRZX5VwgyiRoDY/yFernSoaKhK8IW88+VDMfAXO5QBL+4ofW8FoZy1KWtIJT6H+3yGinIJNgohAx+71zY6poYKz6k3/gYc4KMXDF4CNh+NNQneffY8vnvPekxDlMyAncgYLq3oEEJMB/B/do5ek4pDGfBT57Vw1pFDX6qXgmV6O5VMK7Eh7GSBWva2RDIUZzZXU5s0C9hjGAd+cF62oUJJVSs27iNpGSUjQipBbcoa0fcPJhLjEIUyElQnrRHp3wcD7z17PvVVFtMbq8Z8lTUWGO5IuA24Cviq//PWUWtRCcxpreFD5y3g8hNnjKrG9uZTZ3H2ka3jrtuNNha11/P+c+dz/lGhmvWOM+ZychFJIaqBTwbMbqnhHWfMYXdPhhNmNo3oWX3gZfOByfmszz5yCu9/2fwh+2vGC288ZRbHTJ+YbSuFxpoE//36E4J65JMNIrp/Y9EPCHEjcB4wBegA/g34E/A7YDawBXiDlDLf0VmAZcuWyZUrV46sxRojxneXr+Mbd73IU1+4KLZZs4aGxsSEEOIJKWVB6N2gDFxK+eYSf7pgxK3SOChQceCV1l/R0NCYmJiYYprGmOKS46YjZfFttTQ0NCYPNAU7DDFvSm3RMrkaGhqTC9qAa2hoaExSaAOuoaGhMUmhDbiGhobGJIU24BoaGhqTFNqAa2hoaExSaAOuoaGhMUmhDbiGhobGJIU24BoaGhqTFIPWQhnViwmxG9g8zK9PAfaMYnNGCxO1XTBx26bbNTRM1HbBxG3bodauOVLKtvyD42rARwIhxMpixVwONiZqu2Ditk23a2iYqO2Cidu2w6VdWkLR0NDQmKTQBlxDQ0NjkmIyGfAfH+wGlMBEbRdM3Lbpdg0NE7VdMHHbdli0a9Jo4BoaGhoacUwmBq6hoaGhEYE24BoaGhqTFJPCgAshLhZCrBVCrBdCXHMQ2zFLCHGvEGKNEOI5IcTV/vEvCiG2CyGe9v9dehDatkkIsdq//kr/WIsQ4i4hxDr/Z+EOx2PbpsWRPnlaCNEthPj4weovIcTPhRCdQohnI8dK9pEQ4nP+mFsrhHjlOLfrv4UQLwghnhFC3CKEaPKPzxVCDET67ofj3K6Sz+4g99dNkTZtEkI87R8fz/4qZR/GboxJKSf0P8AENgDzgSSwCjjmILVlOnCS//964EXgGOCLwKcOcj9tAqbkHfs6cI3//2uArx3k57gLmHOw+gs4FzgJeHawPvKf6yogBczzx6A5ju16BWD5//9apF1zo587CP1V9Nkd7P7K+/s3gP93EPqrlH0YszE2GRj4qcB6KeVGKWUW+C1w+cFoiJRyp5TySf//PcAaYMbBaEuFuBy4zv//dcAVB68pXABskFIONxN3xJBSPgDsyztcqo8uB34rpcxIKV8C1uONxXFpl5TyTiml7f/6KDBzLK491HaVwUHtLwUhhACuBG4ci2uXQxn7MGZjbDIY8BnA1sjv25gARlMIMRdYCqzwD33EX+7+fLylCh8SuFMI8YQQ4n3+sXYp5U7wBhcw9SC0S+FNxF+qg91fCqX6aCKNu3cDf4v8Pk8I8ZQQ4n4hxDkHoT3Fnt1E6a9zgA4p5brIsXHvrzz7MGZjbDIYcFHk2EGNfRRC1AF/BD4upewGfgAsAE4EduIt4cYbZ0kpTwIuAT4shDj3ILShKIQQSeAy4Pf+oYnQX4NhQow7IcS1gA3c4B/aCcyWUi4FPgH8RgjRMI5NKvXsJkR/AW8mThTGvb+K2IeSHy1ybEh9NhkM+DZgVuT3mcCOg9QWhBAJvIdzg5TyZgApZYeU0pFSusBPGKOlYzlIKXf4PzuBW/w2dAghpvvtng50jne7fFwCPCml7PDbeND7K4JSfXTQx50Q4irg1cBbpS+a+svtvf7/n8DTTReNV5vKPLuJ0F8W8DrgJnVsvPurmH1gDMfYZDDgjwMLhRDzfCb3JuC2g9EQX1/7GbBGSvnNyPHpkY+9Fng2/7tj3K5aIUS9+j+eA+xZvH66yv/YVcCt49muCGKs6GD3Vx5K9dFtwJuEECkhxDxgIfDYeDVKCHEx8FngMillf+R4mxDC9P8/32/XxnFsV6lnd1D7y8eFwAtSym3qwHj2Vyn7wFiOsfHwzo6Cd/dSPI/uBuDag9iOs/GWOM8AT/v/LgV+Daz2j98GTB/nds3H82avAp5TfQS0AsuBdf7PloPQZzXAXqAxcuyg9BfeJLITyOGxn/eU6yPgWn/MrQUuGed2rcfTR9U4+6H/2X/yn/Eq4EngNePcrpLP7mD2l3/8l8AH8j47nv1Vyj6M2RjTqfQaGhoakxSTQULR0NDQ0CgCbcA1NDQ0Jim0AdfQ0NCYpNAGXENDQ2OSQhtwDQ0NjUkKbcA1JjWEEI6IVzwsW61SCPEBIcQ7RuG6m4QQU0Z6Hg2NkUCHEWpMaggheqWUdQfhupuAZVLKPeN9bQ0NBc3ANQ5J+Az5a0KIx/x/R/rHvyiE+JT//48JIZ73CzP91j/WIoT4k3/sUSHECf7xViHEnX5RpB8RqWMhhHibf42nhRA/Upl/GhpjDW3ANSY7qvMklDdG/tYtpTwV+B7w7SLfvQZYKqU8AfiAf+zfgaf8Y/8K/Mo//m/AP6RXFOk2YDaAEOJo4I14xcROBBzgraN5gxoapWAd7AZoaIwQA77hLIYbIz+/VeTvzwA3CCH+BPzJP3Y2Xvo1Usp7fObdiLeJwOv8438RQuz3P38BcDLwuFcKg2oOXtEwjcMM2oBrHMqQJf6v8Co8w3wZ8AUhxLGUL/FZ7BwCuE5K+bmRNFRDYzjQEorGoYw3Rn4+Ev2DEMIAZkkp7wU+AzQBdcAD+BKIEOI8YI/0ajpHj18CqI0MlgOvF0JM9f/WIoSYM2Z3pKERgWbgGpMd1cLfwNbHHVJKFUqYEkKswCMqb877nglc78sjAviWlPKAEOKLwC+EEM8A/YRlQP8duFEI8SRwP7AFQEr5vBDi83i7IRl4FfI+DBy0reM0Dh/oMEKNQxI6zE/jcICWUDQ0NDQmKTQD19DQ0Jik0AxcQ0NDY5JCG3ANDQ2NSQptwDU0NDQmKbQB19DQ0Jik0AZcQ0NDY5Li/wMd9Hma9f4hngAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PAgent():\n",
    "    def __init__(self,\n",
    "                 gamma,\n",
    "                 model,\n",
    "                 max_episode_num,\n",
    "                 max_steps\n",
    "                 ):\n",
    "        self.GAMMA = gamma\n",
    "        self.model =  model\n",
    "        self.max_episode_num = max_episode_num\n",
    "        self.max_steps = max_steps\n",
    "        self.numsteps = []\n",
    "        self.avg_numsteps = []\n",
    "        self.all_rewards = []\n",
    "    \n",
    "    def update_policy(self, rewards, log_probs):\n",
    "        discounted_rewards = []\n",
    "\n",
    "        for t in range(len(rewards)):\n",
    "            Gt = 0\n",
    "            pw = 0\n",
    "            for r in rewards[t:]:\n",
    "                Gt = Gt + GAMMA ** pw * r\n",
    "                pw = pw + 1\n",
    "            discounted_rewards.append(Gt)\n",
    "\n",
    "        discounted_rewards = torch.tensor(discounted_rewards)\n",
    "        discounted_rewards = (discounted_rewards - discounted_rewards.mean()) / (\n",
    "                    discounted_rewards.std() + 1e-9)  # normalize discounted rewards\n",
    "\n",
    "        policy_gradient = []\n",
    "        for log_prob, Gt in zip(log_probs, discounted_rewards):\n",
    "            policy_gradient.append(-log_prob * Gt)\n",
    "\n",
    "        self.model.optimizer.zero_grad()\n",
    "        policy_gradient = torch.stack(policy_gradient).sum()\n",
    "        policy_gradient.backward()\n",
    "        self.model.optimizer.step()\n",
    "    \n",
    "    def train(self):\n",
    "        env = gym.make('CartPole-v0')\n",
    "    #policy_net = PolicyNetwork(env.observation_space.shape[0], env.action_space.n, 128\n",
    "        for episode in range(self.max_episode_num):\n",
    "            state = env.reset()\n",
    "            log_probs = []\n",
    "            rewards = []\n",
    "        #print(state)\n",
    "\n",
    "            for steps in range(self.max_steps):\n",
    "                env.render()\n",
    "                action, log_prob = self.model.get_action(state)\n",
    "            #print(action, reward)\n",
    "                new_state, reward, done, _ = env.step(action)\n",
    "                log_probs.append(log_prob)\n",
    "                rewards.append(reward)\n",
    "\n",
    "                if done:\n",
    "                    self.update_policy(rewards, log_probs)\n",
    "                    self.numsteps.append(steps)\n",
    "                    self.avg_numsteps.append(np.mean(self.numsteps[-10:]))\n",
    "                    self.all_rewards.append(np.sum(rewards))\n",
    "                    if episode % 1 == 0:\n",
    "                        sys.stdout.write(\"episode: {}, total reward: {}, average_reward: {}, length: {}\\n\".format(episode,\n",
    "                            np.round(np.sum(rewards), decimals=3),  np.round(np.mean(self.all_rewards[-10:]), decimals=3), steps))\n",
    "                    break\n",
    "\n",
    "                state = new_state\n",
    "\n",
    "        plt.plot(numsteps)\n",
    "        plt.plot(avg_numsteps)\n",
    "        plt.xlabel('Episode')\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main_2():\n",
    "    env = gym.make('CartPole-v0')\n",
    "    policy_net = PolicyNetwork(env.observation_space.shape[0], env.action_space.n, 128)\n",
    "    #policy_net = PolicyNetwork(env.observation_space.shape[0], env.action_space.n, 128)\n",
    "    GAMMA = 0.9\n",
    "    max_episode_num = 200\n",
    "    max_steps = 10000\n",
    "    agent = PAgent(GAMMA,policy_net, max_episode_num, max_steps)\n",
    "    agent.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode: 0, total reward: 9.0, average_reward: 9.0, length: 8\n",
      "episode: 1, total reward: 15.0, average_reward: 12.0, length: 14\n",
      "episode: 2, total reward: 14.0, average_reward: 12.667, length: 13\n",
      "episode: 3, total reward: 16.0, average_reward: 13.5, length: 15\n",
      "episode: 4, total reward: 13.0, average_reward: 13.4, length: 12\n",
      "episode: 5, total reward: 24.0, average_reward: 15.167, length: 23\n",
      "episode: 6, total reward: 16.0, average_reward: 15.286, length: 15\n",
      "episode: 7, total reward: 38.0, average_reward: 18.125, length: 37\n",
      "episode: 8, total reward: 10.0, average_reward: 17.222, length: 9\n",
      "episode: 9, total reward: 28.0, average_reward: 18.3, length: 27\n",
      "episode: 10, total reward: 16.0, average_reward: 19.0, length: 15\n",
      "episode: 11, total reward: 50.0, average_reward: 22.5, length: 49\n",
      "episode: 12, total reward: 18.0, average_reward: 22.9, length: 17\n",
      "episode: 13, total reward: 24.0, average_reward: 23.7, length: 23\n",
      "episode: 14, total reward: 33.0, average_reward: 25.7, length: 32\n",
      "episode: 15, total reward: 25.0, average_reward: 25.8, length: 24\n",
      "episode: 16, total reward: 17.0, average_reward: 25.9, length: 16\n",
      "episode: 17, total reward: 16.0, average_reward: 23.7, length: 15\n",
      "episode: 18, total reward: 47.0, average_reward: 27.4, length: 46\n",
      "episode: 19, total reward: 10.0, average_reward: 25.6, length: 9\n",
      "episode: 20, total reward: 17.0, average_reward: 25.7, length: 16\n",
      "episode: 21, total reward: 12.0, average_reward: 21.9, length: 11\n",
      "episode: 22, total reward: 27.0, average_reward: 22.8, length: 26\n",
      "episode: 23, total reward: 10.0, average_reward: 21.4, length: 9\n",
      "episode: 24, total reward: 20.0, average_reward: 20.1, length: 19\n",
      "episode: 25, total reward: 15.0, average_reward: 19.1, length: 14\n",
      "episode: 26, total reward: 24.0, average_reward: 19.8, length: 23\n",
      "episode: 27, total reward: 11.0, average_reward: 19.3, length: 10\n",
      "episode: 28, total reward: 17.0, average_reward: 16.3, length: 16\n",
      "episode: 29, total reward: 12.0, average_reward: 16.5, length: 11\n",
      "episode: 30, total reward: 18.0, average_reward: 16.6, length: 17\n",
      "episode: 31, total reward: 10.0, average_reward: 16.4, length: 9\n",
      "episode: 32, total reward: 41.0, average_reward: 17.8, length: 40\n",
      "episode: 33, total reward: 24.0, average_reward: 19.2, length: 23\n",
      "episode: 34, total reward: 35.0, average_reward: 20.7, length: 34\n",
      "episode: 35, total reward: 10.0, average_reward: 20.2, length: 9\n",
      "episode: 36, total reward: 9.0, average_reward: 18.7, length: 8\n",
      "episode: 37, total reward: 46.0, average_reward: 22.2, length: 45\n",
      "episode: 38, total reward: 61.0, average_reward: 26.6, length: 60\n",
      "episode: 39, total reward: 29.0, average_reward: 28.3, length: 28\n",
      "episode: 40, total reward: 19.0, average_reward: 28.4, length: 18\n",
      "episode: 41, total reward: 68.0, average_reward: 34.2, length: 67\n",
      "episode: 42, total reward: 19.0, average_reward: 32.0, length: 18\n",
      "episode: 43, total reward: 26.0, average_reward: 32.2, length: 25\n",
      "episode: 44, total reward: 18.0, average_reward: 30.5, length: 17\n",
      "episode: 45, total reward: 51.0, average_reward: 34.6, length: 50\n",
      "episode: 46, total reward: 31.0, average_reward: 36.8, length: 30\n",
      "episode: 47, total reward: 19.0, average_reward: 34.1, length: 18\n",
      "episode: 48, total reward: 10.0, average_reward: 29.0, length: 9\n",
      "episode: 49, total reward: 17.0, average_reward: 27.8, length: 16\n",
      "episode: 50, total reward: 39.0, average_reward: 29.8, length: 38\n",
      "episode: 51, total reward: 15.0, average_reward: 24.5, length: 14\n",
      "episode: 52, total reward: 25.0, average_reward: 25.1, length: 24\n",
      "episode: 53, total reward: 22.0, average_reward: 24.7, length: 21\n",
      "episode: 54, total reward: 24.0, average_reward: 25.3, length: 23\n",
      "episode: 55, total reward: 13.0, average_reward: 21.5, length: 12\n",
      "episode: 56, total reward: 24.0, average_reward: 20.8, length: 23\n",
      "episode: 57, total reward: 12.0, average_reward: 20.1, length: 11\n",
      "episode: 58, total reward: 22.0, average_reward: 21.3, length: 21\n",
      "episode: 59, total reward: 39.0, average_reward: 23.5, length: 38\n",
      "episode: 60, total reward: 15.0, average_reward: 21.1, length: 14\n",
      "episode: 61, total reward: 19.0, average_reward: 21.5, length: 18\n",
      "episode: 62, total reward: 12.0, average_reward: 20.2, length: 11\n",
      "episode: 63, total reward: 20.0, average_reward: 20.0, length: 19\n",
      "episode: 64, total reward: 47.0, average_reward: 22.3, length: 46\n",
      "episode: 65, total reward: 15.0, average_reward: 22.5, length: 14\n",
      "episode: 66, total reward: 14.0, average_reward: 21.5, length: 13\n",
      "episode: 67, total reward: 48.0, average_reward: 25.1, length: 47\n",
      "episode: 68, total reward: 14.0, average_reward: 24.3, length: 13\n",
      "episode: 69, total reward: 18.0, average_reward: 22.2, length: 17\n",
      "episode: 70, total reward: 23.0, average_reward: 23.0, length: 22\n",
      "episode: 71, total reward: 11.0, average_reward: 22.2, length: 10\n",
      "episode: 72, total reward: 26.0, average_reward: 23.6, length: 25\n",
      "episode: 73, total reward: 27.0, average_reward: 24.3, length: 26\n",
      "episode: 74, total reward: 17.0, average_reward: 21.3, length: 16\n",
      "episode: 75, total reward: 13.0, average_reward: 21.1, length: 12\n",
      "episode: 76, total reward: 69.0, average_reward: 26.6, length: 68\n",
      "episode: 77, total reward: 13.0, average_reward: 23.1, length: 12\n",
      "episode: 78, total reward: 33.0, average_reward: 25.0, length: 32\n",
      "episode: 79, total reward: 14.0, average_reward: 24.6, length: 13\n",
      "episode: 80, total reward: 39.0, average_reward: 26.2, length: 38\n",
      "episode: 81, total reward: 20.0, average_reward: 27.1, length: 19\n",
      "episode: 82, total reward: 32.0, average_reward: 27.7, length: 31\n",
      "episode: 83, total reward: 12.0, average_reward: 26.2, length: 11\n",
      "episode: 84, total reward: 64.0, average_reward: 30.9, length: 63\n",
      "episode: 85, total reward: 23.0, average_reward: 31.9, length: 22\n",
      "episode: 86, total reward: 19.0, average_reward: 26.9, length: 18\n",
      "episode: 87, total reward: 40.0, average_reward: 29.6, length: 39\n",
      "episode: 88, total reward: 10.0, average_reward: 27.3, length: 9\n",
      "episode: 89, total reward: 21.0, average_reward: 28.0, length: 20\n",
      "episode: 90, total reward: 19.0, average_reward: 26.0, length: 18\n",
      "episode: 91, total reward: 40.0, average_reward: 28.0, length: 39\n",
      "episode: 92, total reward: 57.0, average_reward: 30.5, length: 56\n",
      "episode: 93, total reward: 15.0, average_reward: 30.8, length: 14\n",
      "episode: 94, total reward: 17.0, average_reward: 26.1, length: 16\n",
      "episode: 95, total reward: 37.0, average_reward: 27.5, length: 36\n",
      "episode: 96, total reward: 20.0, average_reward: 27.6, length: 19\n",
      "episode: 97, total reward: 9.0, average_reward: 24.5, length: 8\n",
      "episode: 98, total reward: 19.0, average_reward: 25.4, length: 18\n",
      "episode: 99, total reward: 13.0, average_reward: 24.6, length: 12\n",
      "episode: 100, total reward: 18.0, average_reward: 24.5, length: 17\n",
      "episode: 101, total reward: 18.0, average_reward: 22.3, length: 17\n",
      "episode: 102, total reward: 15.0, average_reward: 18.1, length: 14\n",
      "episode: 103, total reward: 26.0, average_reward: 19.2, length: 25\n",
      "episode: 104, total reward: 21.0, average_reward: 19.6, length: 20\n",
      "episode: 105, total reward: 35.0, average_reward: 19.4, length: 34\n",
      "episode: 106, total reward: 48.0, average_reward: 22.2, length: 47\n",
      "episode: 107, total reward: 16.0, average_reward: 22.9, length: 15\n",
      "episode: 108, total reward: 12.0, average_reward: 22.2, length: 11\n",
      "episode: 109, total reward: 46.0, average_reward: 25.5, length: 45\n",
      "episode: 110, total reward: 31.0, average_reward: 26.8, length: 30\n",
      "episode: 111, total reward: 16.0, average_reward: 26.6, length: 15\n",
      "episode: 112, total reward: 65.0, average_reward: 31.6, length: 64\n",
      "episode: 113, total reward: 14.0, average_reward: 30.4, length: 13\n",
      "episode: 114, total reward: 24.0, average_reward: 30.7, length: 23\n",
      "episode: 115, total reward: 48.0, average_reward: 32.0, length: 47\n",
      "episode: 116, total reward: 49.0, average_reward: 32.1, length: 48\n",
      "episode: 117, total reward: 72.0, average_reward: 37.7, length: 71\n",
      "episode: 118, total reward: 17.0, average_reward: 38.2, length: 16\n",
      "episode: 119, total reward: 75.0, average_reward: 41.1, length: 74\n",
      "episode: 120, total reward: 29.0, average_reward: 40.9, length: 28\n",
      "episode: 121, total reward: 71.0, average_reward: 46.4, length: 70\n",
      "episode: 122, total reward: 22.0, average_reward: 42.1, length: 21\n",
      "episode: 123, total reward: 32.0, average_reward: 43.9, length: 31\n",
      "episode: 124, total reward: 18.0, average_reward: 43.3, length: 17\n",
      "episode: 125, total reward: 36.0, average_reward: 42.1, length: 35\n",
      "episode: 126, total reward: 34.0, average_reward: 40.6, length: 33\n",
      "episode: 127, total reward: 20.0, average_reward: 35.4, length: 19\n",
      "episode: 128, total reward: 23.0, average_reward: 36.0, length: 22\n",
      "episode: 129, total reward: 23.0, average_reward: 30.8, length: 22\n",
      "episode: 130, total reward: 21.0, average_reward: 30.0, length: 20\n",
      "episode: 131, total reward: 38.0, average_reward: 26.7, length: 37\n",
      "episode: 132, total reward: 25.0, average_reward: 27.0, length: 24\n",
      "episode: 133, total reward: 25.0, average_reward: 26.3, length: 24\n",
      "episode: 134, total reward: 43.0, average_reward: 28.8, length: 42\n",
      "episode: 135, total reward: 46.0, average_reward: 29.8, length: 45\n",
      "episode: 136, total reward: 48.0, average_reward: 31.2, length: 47\n",
      "episode: 137, total reward: 26.0, average_reward: 31.8, length: 25\n",
      "episode: 138, total reward: 42.0, average_reward: 33.7, length: 41\n",
      "episode: 139, total reward: 11.0, average_reward: 32.5, length: 10\n",
      "episode: 140, total reward: 38.0, average_reward: 34.2, length: 37\n",
      "episode: 141, total reward: 9.0, average_reward: 31.3, length: 8\n",
      "episode: 142, total reward: 15.0, average_reward: 30.3, length: 14\n",
      "episode: 143, total reward: 15.0, average_reward: 29.3, length: 14\n",
      "episode: 144, total reward: 28.0, average_reward: 27.8, length: 27\n",
      "episode: 145, total reward: 77.0, average_reward: 30.9, length: 76\n",
      "episode: 146, total reward: 60.0, average_reward: 32.1, length: 59\n",
      "episode: 147, total reward: 24.0, average_reward: 31.9, length: 23\n",
      "episode: 148, total reward: 30.0, average_reward: 30.7, length: 29\n",
      "episode: 149, total reward: 20.0, average_reward: 31.6, length: 19\n",
      "episode: 150, total reward: 21.0, average_reward: 29.9, length: 20\n",
      "episode: 151, total reward: 34.0, average_reward: 32.4, length: 33\n",
      "episode: 152, total reward: 14.0, average_reward: 32.3, length: 13\n",
      "episode: 153, total reward: 21.0, average_reward: 32.9, length: 20\n",
      "episode: 154, total reward: 15.0, average_reward: 31.6, length: 14\n",
      "episode: 155, total reward: 15.0, average_reward: 25.4, length: 14\n",
      "episode: 156, total reward: 21.0, average_reward: 21.5, length: 20\n",
      "episode: 157, total reward: 16.0, average_reward: 20.7, length: 15\n",
      "episode: 158, total reward: 13.0, average_reward: 19.0, length: 12\n",
      "episode: 159, total reward: 18.0, average_reward: 18.8, length: 17\n",
      "episode: 160, total reward: 14.0, average_reward: 18.1, length: 13\n",
      "episode: 161, total reward: 16.0, average_reward: 16.3, length: 15\n",
      "episode: 162, total reward: 40.0, average_reward: 18.9, length: 39\n",
      "episode: 163, total reward: 28.0, average_reward: 19.6, length: 27\n",
      "episode: 164, total reward: 24.0, average_reward: 20.5, length: 23\n",
      "episode: 165, total reward: 20.0, average_reward: 21.0, length: 19\n",
      "episode: 166, total reward: 25.0, average_reward: 21.4, length: 24\n",
      "episode: 167, total reward: 20.0, average_reward: 21.8, length: 19\n",
      "episode: 168, total reward: 58.0, average_reward: 26.3, length: 57\n",
      "episode: 169, total reward: 19.0, average_reward: 26.4, length: 18\n",
      "episode: 170, total reward: 12.0, average_reward: 26.2, length: 11\n",
      "episode: 171, total reward: 44.0, average_reward: 29.0, length: 43\n",
      "episode: 172, total reward: 15.0, average_reward: 26.5, length: 14\n",
      "episode: 173, total reward: 14.0, average_reward: 25.1, length: 13\n",
      "episode: 174, total reward: 15.0, average_reward: 24.2, length: 14\n",
      "episode: 175, total reward: 13.0, average_reward: 23.5, length: 12\n",
      "episode: 176, total reward: 52.0, average_reward: 26.2, length: 51\n",
      "episode: 177, total reward: 43.0, average_reward: 28.5, length: 42\n",
      "episode: 178, total reward: 9.0, average_reward: 23.6, length: 8\n",
      "episode: 179, total reward: 23.0, average_reward: 24.0, length: 22\n",
      "episode: 180, total reward: 56.0, average_reward: 28.4, length: 55\n",
      "episode: 181, total reward: 11.0, average_reward: 25.1, length: 10\n",
      "episode: 182, total reward: 24.0, average_reward: 26.0, length: 23\n",
      "episode: 183, total reward: 20.0, average_reward: 26.6, length: 19\n",
      "episode: 184, total reward: 18.0, average_reward: 26.9, length: 17\n",
      "episode: 185, total reward: 21.0, average_reward: 27.7, length: 20\n",
      "episode: 186, total reward: 21.0, average_reward: 24.6, length: 20\n",
      "episode: 187, total reward: 31.0, average_reward: 23.4, length: 30\n",
      "episode: 188, total reward: 14.0, average_reward: 23.9, length: 13\n",
      "episode: 189, total reward: 29.0, average_reward: 24.5, length: 28\n",
      "episode: 190, total reward: 13.0, average_reward: 20.2, length: 12\n",
      "episode: 191, total reward: 19.0, average_reward: 21.0, length: 18\n",
      "episode: 192, total reward: 37.0, average_reward: 22.3, length: 36\n",
      "episode: 193, total reward: 15.0, average_reward: 21.8, length: 14\n",
      "episode: 194, total reward: 36.0, average_reward: 23.6, length: 35\n",
      "episode: 195, total reward: 27.0, average_reward: 24.2, length: 26\n",
      "episode: 196, total reward: 22.0, average_reward: 24.3, length: 21\n",
      "episode: 197, total reward: 43.0, average_reward: 25.5, length: 42\n",
      "episode: 198, total reward: 17.0, average_reward: 25.8, length: 16\n",
      "episode: 199, total reward: 20.0, average_reward: 24.9, length: 19\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'numsteps' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-10-1a35196d5588>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mmain_2\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-3-e559fc00616c>\u001b[0m in \u001b[0;36mmain_2\u001b[1;34m()\u001b[0m\n\u001b[0;32m      7\u001b[0m     \u001b[0mmax_steps\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m10000\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m     \u001b[0magent\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mPAgent\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mGAMMA\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mpolicy_net\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_episode_num\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_steps\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m     \u001b[0magent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-9-61d04b1c775a>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     67\u001b[0m                 \u001b[0mstate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnew_state\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     68\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 69\u001b[1;33m         \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnumsteps\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     70\u001b[0m         \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mavg_numsteps\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     71\u001b[0m         \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mxlabel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Episode'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'numsteps' is not defined"
     ]
    }
   ],
   "source": [
    "main_2()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}