ASL_Gesture_Detection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "946e43b0-f594-480e-bf48-86dbe31753c2",
   "metadata": {},
   "source": [
    "# ASL Gesture Detection using OpenCV \n",
    "## This system detects some basic ASL gestures using OpenCV & Mediapipe."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35d50e85-f110-406f-b4a0-9459066afd7e",
   "metadata": {},
   "source": [
    "### Initial Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f5b1d3f6-649c-43de-b321-d719d606a0d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import mediapipe as mp\n",
    "import time\n",
    "import cv2\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1745825d-b21a-49c1-9aca-73cc55e36ac5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Holistic detetction API (Hands, Face and Pose)\n",
    "mp_holistic = mp.solutions.holistic\n",
    "\n",
    "# Drawing Utilities for keypoint and landmark detection\n",
    "mp_drawings = mp.solutions.drawing_utils"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f06cf59e-dd50-48b1-a5b2-63dbcdf2478a",
   "metadata": {},
   "source": [
    "#### Camera openCV detects the colors as BGR so have to convert to RGB for mediapipe for detect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "aa64f514-da66-4720-b6f4-188d92052e87",
   "metadata": {},
   "outputs": [],
   "source": [
    "def mediapipe_detection(image, model):\n",
    "    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "    image.flags.writeable = False\n",
    "    results = model.process(image)\n",
    "    image.flags.writeable = True\n",
    "    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n",
    "    return image, results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "020addf1-c5b8-46e4-be70-82edca792dab",
   "metadata": {},
   "source": [
    "#### MediaPipe offers drawing utilities to change color, thickness and radius of landmarks and keypoints."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "526b9d76-77a4-4d80-bcba-93e9c8a5ae54",
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_landmarks(image, results):\n",
    "    # Face Landmarks\n",
    "    mp_drawings.draw_landmarks(image, \n",
    "                               results.face_landmarks, \n",
    "                               mp_holistic.FACEMESH_TESSELATION, \n",
    "                               mp_drawings.DrawingSpec(color=(6, 70, 99), thickness=1, circle_radius=1), \n",
    "                               mp_drawings.DrawingSpec(color=(236, 179, 101), thickness=1, circle_radius=1))\n",
    "    # Pose Landmarks\n",
    "    mp_drawings.draw_landmarks(image, \n",
    "                               results.pose_landmarks, \n",
    "                               mp_holistic.POSE_CONNECTIONS, \n",
    "                               mp_drawings.DrawingSpec(color=(6, 70, 99), thickness=1, circle_radius=1), \n",
    "                               mp_drawings.DrawingSpec(color=(236, 179, 101), thickness=1, circle_radius=1))\n",
    "    # Left Hand Landmarks\n",
    "    mp_drawings.draw_landmarks(image, \n",
    "                               results.left_hand_landmarks, \n",
    "                               mp_holistic.HAND_CONNECTIONS, \n",
    "                               mp_drawings.DrawingSpec(color=(6, 70, 99), thickness=1, circle_radius=1), \n",
    "                               mp_drawings.DrawingSpec(color=(236, 179, 101), thickness=1, circle_radius=1))\n",
    "    # Right Hand \n",
    "    mp_drawings.draw_landmarks(image, \n",
    "                               results.right_hand_landmarks, \n",
    "                               mp_holistic.HAND_CONNECTIONS, \n",
    "                               mp_drawings.DrawingSpec(color=(6, 70, 99), thickness=1, circle_radius=1), \n",
    "                               mp_drawings.DrawingSpec(color=(236, 179, 101), thickness=1, circle_radius=1))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b14962a3-7585-42fd-896d-bacd19793c84",
   "metadata": {},
   "source": [
    "## Testing Camera and MediaPipe Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8e6108e4-2cd3-48b3-ae8f-6f2d1f3da2fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "cap = cv2.VideoCapture(0)\n",
    "with mp_holistic.Holistic(min_detection_confidence=0.7, min_tracking_confidence=0.7) as holistic:\n",
    "    while cap.isOpened():\n",
    "        ret, frame = cap.read()\n",
    "\n",
    "        image, results = mediapipe_detection(frame, holistic)\n",
    "        draw_landmarks(image, results)\n",
    "        \n",
    "        cv2.imshow('OpenCV Feed', image)\n",
    "\n",
    "        if cv2.waitKey(10) & 0xFF == ord('q'):\n",
    "            break\n",
    "\n",
    "cap.release()\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aed47baa-ffbd-4b7b-bd8f-e2d852e53a5a",
   "metadata": {},
   "source": [
    "## Keypoint Extraction\n",
    "Mediapipe returns results with X, Y, Z axis values. X & Y being offsets on 2D Plane and Z being distance from the camera.<br>\n",
    "All keypoints of face, hands and pose are concatenated to a single array. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a661ee4f-860c-42dc-8d43-d23556e2b9ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_keypoints(results):\n",
    "    pose = np.array([[res.x, res.y, res.z, res.visibility] for res in results.pose_landmarks.landmark]).flatten() if results.pose_landmarks else np.zeros(33*4)\n",
    "    face = np.array([[res.x, res.y, res.z] for res in results.face_landmarks.landmark]).flatten() if results.face_landmarks else np.zeros(468*3)\n",
    "    lh = np.array([[res.x, res.y, res.z] for res in results.left_hand_landmarks.landmark]).flatten() if results.left_hand_landmarks else np.zeros(21*3)\n",
    "    rh = np.array([[res.x, res.y, res.z] for res in results.right_hand_landmarks.landmark]).flatten() if results.right_hand_landmarks else np.zeros(21*3)\n",
    "    return np.concatenate([pose, face, lh, rh])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "547d852a-9c10-4b23-8a73-ce936c559f81",
   "metadata": {},
   "source": [
    "## Creating Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a89dcb1-faba-47f2-8c92-f43b86aef19e",
   "metadata": {},
   "source": [
    "#### Initiating Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "20bb33a5-7f68-4886-8a34-13ec89d527f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Path to save keypoints Array\n",
    "DATA_PATH = os.path.join('Keypoints_DATA')\n",
    "\n",
    "# Actions to be detected\n",
    "actions = np.array(['Hello', 'ThankYou'])\n",
    "\n",
    "# actions are used to store and make directories hence cant have space\n",
    "words = np.array(['Hello', 'Thank You'])\n",
    "\n",
    "# no of videos for a single action\n",
    "no_sequence = 30\n",
    "\n",
    "# frames for a single action\n",
    "sequence_length = 30 "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51c96ef0-c1f7-4794-a15e-e4c1feea6b0a",
   "metadata": {},
   "source": [
    "#### Making Directories to store keypoint arrays for dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87249ef6-4800-4c97-b17c-9cc705f93c48",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for action in actions:\n",
    "#     for sequence in range(no_sequence):\n",
    "#         try:\n",
    "#             os.makedirs(os.path.join(DATA_PATH, action, str(sequence)))\n",
    "#         except:\n",
    "#             pass"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "235cb299-6df6-42d3-8e89-19e50152ac30",
   "metadata": {},
   "source": [
    "#### Detecting and storing keypoints to directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8331a7e3-b237-4d09-b1cd-14e43a96e7e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# cap = cv2.VideoCapture(0)\n",
    "\n",
    "# with mp_holistic.Holistic(min_detection_confidence=0.7, min_tracking_confidence=0.7) as holistic:\n",
    "#     for action in actions:\n",
    "#         for sequence in range(no_sequence):\n",
    "#             for frame_num in range(sequence_length):\n",
    "#                 ret, frame = cap.read()\n",
    "\n",
    "#                 image, results = mediapipe_detection(frame, holistic)\n",
    "#                 # print(results)\n",
    "#                 draw_landmarks(image, results)\n",
    "\n",
    "#                 if frame_num == 0: \n",
    "#                     cv2.putText(image, 'STARTING COLLECTION', (120,200), \n",
    "#                                cv2.FONT_HERSHEY_SIMPLEX, 1, (0,255, 0), 4, cv2.LINE_AA)\n",
    "#                     cv2.putText(image, 'Collecting frames for {} Video Number {}'.format(action, sequence), (15,12), \n",
    "#                                cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1, cv2.LINE_AA)\n",
    "#                     cv2.imshow('OpenCV Feed', image)\n",
    "#                     cv2.waitKey(2000)\n",
    "#                 else: \n",
    "#                     cv2.putText(image, 'Collecting frames for {} Video Number {}'.format(action, sequence), (15,12), \n",
    "#                                cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1, cv2.LINE_AA)\n",
    "#                     cv2.imshow('OpenCV Feed', image)\n",
    "                \n",
    "#                 keypoints = extract_keypoints(results)\n",
    "#                 npy_path = os.path.join(DATA_PATH, action, str(sequence), str(frame_num))\n",
    "#                 np.save(npy_path, keypoints)\n",
    "\n",
    "#                 if cv2.waitKey(10) & 0xFF == ord('q'):\n",
    "#                     break\n",
    "                \n",
    "#     cap.release()\n",
    "#     cv2.destroyAllWindows()   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e775dc27-a604-46c6-b33d-f0af1bd27e28",
   "metadata": {},
   "outputs": [],
   "source": [
    "    cap.release()\n",
    "    cv2.destroyAllWindows()   "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2c22557-be6a-4650-a8d8-a32e2f5e30e3",
   "metadata": {},
   "source": [
    "## Data Preprocessing and Model Generation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc9098ab-ab75-47d6-bfb3-a006c9cbc87c",
   "metadata": {},
   "source": [
    "#### Imports for train test split and categorise data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "52340956-af3b-495a-8e74-dbc2fb36b9e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8f4194ac-4b92-40ba-a7c8-907f95bf4a49",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Hello': 0, 'ThankYou': 1}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Numbers are assigned to each action\n",
    "label_map = {label: num for num, label in enumerate(actions)}\n",
    "label_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "245f8dd9-1bbc-4811-942e-2fec142b87ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saved numpy keypoint arrays are loaded into arrays\n",
    "sequences, labels = [], []\n",
    "for action in actions:\n",
    "    for sequence in range(no_sequence):\n",
    "        window = []\n",
    "        for frame_num in range(sequence_length):\n",
    "            res = np.load(os.path.join(DATA_PATH, action, str(sequence), \"{}.npy\".format(frame_num)))\n",
    "            window.append(res)\n",
    "        sequences.append(window)\n",
    "        labels.append(label_map[action])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7caa3cfd-9eba-4ef5-b525-282a40e0fd95",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c01e072b-c806-48b6-9528-1c2b103d4374",
   "metadata": {},
   "outputs": [],
   "source": [
    "y = to_categorical(labels).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "bdb71444-9276-4931-a4b3-3d247cad4084",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.05)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a01b787b-f750-4e07-8909-c617d2d59743",
   "metadata": {},
   "source": [
    "#### Model Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "add0164b-cfe9-4fa0-944c-5b8dbe3d4644",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import LSTM, Dense\n",
    "from tensorflow.keras.callbacks import TensorBoard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3eb0f356-3923-4282-9f4b-ad021acbdd44",
   "metadata": {},
   "outputs": [],
   "source": [
    "# logs to check model train progress on tensor board\n",
    "## Open terminal in Logs folder and type: tensorboard --logdir=. \n",
    "\n",
    "log_dir = os.path.join('Logs_Gestures')\n",
    "tb_callbacks = TensorBoard(log_dir=log_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d498d1b1-efd3-4cf8-a081-5687ffc790a5",
   "metadata": {},
   "source": [
    "#### Training Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "65454f08-0408-4cea-bf43-8d0bd285869f",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(LSTM(64, return_sequences=True, activation='relu', input_shape=(30,1662)))\n",
    "model.add(LSTM(128, return_sequences=True, activation='relu'))\n",
    "model.add(LSTM(64, return_sequences=False, activation='relu'))\n",
    "model.add(Dense(64, activation='relu'))\n",
    "model.add(Dense(32, activation='relu'))\n",
    "model.add(Dense(actions.shape[0], activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d46fb3e0-3e94-47c1-bdb5-7417a177f8ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='Adam', loss='categorical_crossentropy', metrics=['categorical_accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5de32d55-06d7-4133-85e2-3e53ebebdc08",
   "metadata": {},
   "source": [
    "#### Fitting Model with train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ab8d63e-6a13-4ca4-b386-c95eb20510b9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "model.fit(X_train, y_train, epochs=2000, callbacks=[tb_callbacks])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f1e5a31-59a0-4d9b-a427-4621ab06b803",
   "metadata": {},
   "source": [
    "#### Testing the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0e3144e5-0450-48df-ad15-3c71b8ba2be7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correct Prediction\n"
     ]
    }
   ],
   "source": [
    "res = model.predict(X_test)\n",
    "if actions[np.argmax(res[0])] == actions[np.argmax(y_test[0])]:\n",
    "    print('Correct Prediction')\n",
    "else:\n",
    "    print('Incorrect Prediction')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb7919e0-106a-4831-90f8-10af00f0043e",
   "metadata": {},
   "source": [
    "#### Saving Model Weights\n",
    "Model weights are saved to a file so that model does not need to be trained everytime."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "349af59f-6da9-4175-b9e3-2b6b688fdf46",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('ASL_Gestures.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b614389-7292-43f3-86cd-ec671ec6b234",
   "metadata": {},
   "outputs": [],
   "source": [
    "del model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ccf7aa76-3540-4bd2-89dd-c6cbc631fb9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# model is loaded using this method\n",
    "model.load_weights('ASL_Gestures.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8fd4ddee-24de-4d1d-9ab8-919cb1a39631",
   "metadata": {},
   "source": [
    "## Multilevel Confusion Matrix and Prediction Accuracy Score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4563f74c-020a-45b2-a177-bb01d4f63863",
   "metadata": {},
   "source": [
    "#### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ecac8079-a7d6-464c-8701-94ec169a9800",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import multilabel_confusion_matrix, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f835f167-6bea-4691-aadf-3c7224ef57ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat = model.predict(X_test)\n",
    "ytrue = np.argmax(y_test, axis=1).tolist()\n",
    "yhat = np.argmax(yhat, axis=1).tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06e992a2-b11d-4bfa-a412-d56de6b593e0",
   "metadata": {},
   "source": [
    "##### Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "47747be5-51e0-4de5-b98f-768432193a13",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b20a44ba-299f-4f58-bc11-d46ae7aecf40",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[2, 0],\n",
       "        [0, 1]],\n",
       "\n",
       "       [[1, 0],\n",
       "        [0, 2]]], dtype=int64)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix = multilabel_confusion_matrix(ytrue, yhat)\n",
    "confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "d027126b-08a9-42aa-ae1c-51012e1f0d5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_heatmap(confusion_matrix, axes, class_label, class_names):\n",
    "    df_cm = pd.DataFrame(confusion_matrix, index=class_names, columns=class_names)\n",
    "    try:\n",
    "        heatmap = sns.heatmap(df_cm, annot=True, fmt=\"d\", cbar=False, ax=axes)\n",
    "    except ValueError:\n",
    "        raise ValueError(\"Confusion matrix values must be integers.\")\n",
    "    heatmap.yaxis.set_ticklabels(heatmap.yaxis.get_ticklabels(), rotation=0, ha='right', fontsize=14)\n",
    "    heatmap.xaxis.set_ticklabels(heatmap.xaxis.get_ticklabels(), rotation=0, ha='right', fontsize=14)\n",
    "    axes.set_ylabel('True label')\n",
    "    axes.set_xlabel('Predicted label')\n",
    "    axes.set_title(\"Confusion Matrix for the action - \" + class_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "2354d049-e576-404b-adca-4687646fce8e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsgAAAFgCAYAAACmDI9oAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAA2oUlEQVR4nO3dd5hkdZ3v8fdnCBIkiIAwpAFGERAJCgpeuYABUFBXXVDRBRNgAgzg9SqoLJiuOQusi6IioBhYgoisBNcAssASFBQQmCEjCjMjDjPf+8c5PZxpOtQ03V0t8349Tz3USb/zPad6ik/96ndOpaqQJEmS1JjW7wIkSZKkqcSALEmSJHUYkCVJkqQOA7IkSZLUYUCWJEmSOgzIkiRJUocBWdKUk2TFJGck+UuS0x5FO/slOXc8a+uHJGcn2X+M2x6T5O4kt493XW37uyS5dSLafjSSfDXJkf2uYyyS3JTk+e3zDyX5Vr9rkpY2BmRJY5bkNUkuTfJAktvaIPe/xqHpVwJPAp5YVf881kaq6ttV9cJxqGcxbSisJD8YNH/rdv7Pe2ynp/BTVXtW1TfGUOeGwLuBLapqnSXdfpg2K8nM8WhrvCQ5IMnF3XlVdXBV/WsfahnyA0OSnyd502TXI2lsDMiSxiTJu4DPAh+hCbMbAl8GXjoOzW8EXFdVD41DWxPlLmDHJE/szNsfuG68dpDGo3mf3hC4p6ruHMO+l30U+5Wkf2gGZElLLMlqwNHA26rq9KqaU1Xzq+qMqjq8XedxST6bZHb7+GySx7XLdklya5J3J7mz7X1+fbvsw8BRwL5tz/QbB/e0JpnR9mQu204fkOSGJPcnuTHJfp35F3e22ynJJe3QjUuS7NRZ9vMk/5rkF2075yZZc4TT8Hfgh8Cr2u2XAfYFvj3oXH0uyS1J/prkt0me287fA/i/neO8olPHsUl+AcwFNun2Pib5SpLvd9r/eJKfJcmg/T4f+CkwvW3/xHb+S5JcneS+tt3NO9vclOS9Sa4E5gwOyUkubJ9e0ba5b2fZI17Ldv7jknwyyc1J7miHPqw41AlNsmmS85Pck2ZYyLeTrN5ZvkGS05Pc1a7zxbb+r9J8WHkgyX3tuicmOaaz7ZuT/CHJvUl+nGR6Z1klOTjJ9e15+dLg8znekuyV5PJ2f/+V5Ok9bjfs6ydp/BiQJY3FjsAKwA9GWOf9wLOBbYCtgR2AD3SWrwOsBqwHvBH4UpInVNUHaXqlT6mqx1fVv41USJKVgc8De1bVKsBOwOVDrLcGcGa77hOBTwNnZvEe4NcArwfWBpYH3jPSvoFvAv/SPt8duAqYPWidS2jOwRrAd4DTkqxQVecMOs6tO9u8DjgQWAX406D23g1s1Yb/59Kcu/2rqrorVdV5wJ7A7Lb9A5I8BTgZOAxYCzgLOCPJ8p1NXw28GFh9cA9+Ve3cPt26bfOUdnrI17Jd9jHgKe05mNmucxRDC/BRYDqwObAB8CFY9AHkP9rzMaNt57tVdS1wMPDLtqbVH9Foslvb7j7Aum0b3x202l7A9sDT2/V2H6bGRy3JtsDXgYNo/ha/Bvw47QfIEbbr5fWTNA4MyJLG4onA3aMMgdgPOLqq7qyqu4AP0wS/AfPb5fOr6izgAWCzMdazEHhakhWr6raqunqIdV4MXF9VJ1XVQ1V1MvA7YO/OOv9eVddV1TzgVJpQN6yq+i9gjSSb0QTlbw6xzreq6p52n58CHsfox3liVV3dbjN/UHtzac7jp4FvAe+oql4vktsXOLOqftq2+0lgRZoPFQM+X1W3tOegV0O+lm0v7IHAO6vq3qq6n+ZDwauGaqSq/tDW9mD7N/Np4H+3i3egCc6Ht99Y/K2qLh6qnSHsB3y9qi6rqgeB99H0OM/orPOxqrqvqm4G/pNRXvtRTG97eBc9gO7Y/AOBr1XVr6tqQTu+/EGaD5Qj6eX1kzQODMiSxuIeYM3BX8EPMp3Fez//1M5b1MaggD0XePySFlJVc2iCw8HAbUnOTPLUHuoZqGm9znT3Tg+91nMS8HZgV4boUU/yniTXtsM67qPpaR1p6AbALSMtrKpfAzfQ9Lie2kONAxY7B1W1sN1X9xyMuO9hDPdargWsBPy2ExTPaec/QpInJflukllJ/krzAWDgXG0A/GmM49IHH/cDNH/DS/zat8M4Bh4bDrO/2VW1evcBdMP8RsC7BwXoDVj830cvxzHU6ydpHBiQJY3FL2l6vF42wjqzaYLAgA155PCDXs2hCVoDFrsjQ1X9pKpeQPP1+e+A43uoZ6CmWWOsacBJwFuBs9re3UXaIRBH0Hxl/4Q2KP2FJtgCLDYsomO4+QPtvo2mJ3p2236vFjsHbQ/vBix+Dkbc9xK6G5gHbNkJi6tV1XAfPD7S7n+rqloVeC0Pn6tbgA2H+VA2Ws2Dj3tlmm9Blvi1b4dxDDxuXtLtW7cAxw4K0Su132qMpJfXT9I4MCBLWmJV9ReacaRfSvKyJCslWS7Jnkk+0a52MvCBJGuludjtKJoewbG4HNg5yYZpLhB838CCttfxpW3oeZDm6/2FQ7RxFvCUNLemW7a9wGwLmnGtY1ZVN9IMA3j/EItXAR6iuePFskmOAlbtLL8DmJEluFNFOw71GJrw+DrgiCTb9Lj5qcCLkzwvyXI045kfBP6r1/23NW/Sy4ptD+fxwGeSrN3Wv16S4cb3rkLz+v0lyXrA4Z1lvwFuAz6WZOUkKyR5Tqem9UcYi3sy8Pok27TjfD8C/LqqburlOCbA8cDBSZ6VxspJXpxklVG2G4/XT1IPDMiSxqQdT/sumgvv7qLpFXs7zZ0doAlxlwJXAv8DXNbOG8u+fgqc0rb1WxYPtdPaOmYD99KE1bcM0cY9NBdivZvm6/UjgL2q6u6x1DSo7Yuraqje8Z/QDCm4juar8b+x+BCGgR9BuSfJZaPtp+09/Rbw8aq6oqqup7kTxkmjXeDV1vl7mmD9BZre3b2Bvavq76Nt2/Eh4Bvt0IB9elj/vcAfgF+1wybOY/gx2B8GtqPpZT8TOL1T+4K23pnAzcCtNENrAM4HrgZuT/KI17O9YPFI4Ps0IXtThhkHPRmq6lLgzcAXgT/TnJ8DethuPF4/ST3IoAufJUmSpKWaPciSJElShwFZkiRJ6jAgS5IkSR0GZEmSJKljpJv8LxXm332DVylKWmqtOP25/S5Bkvrmob/PylDz7UGWJEmSOgzIkiRJUocBWZIkSeowIEuSJEkdBmRJkiSpw4AsSZIkdRiQJUmSpA4DsiRJktRhQJYkSZI6DMiSJElShwFZkiRJ6jAgS5IkSR0GZEmSJKnDgCxJkiR1GJAlSZKkDgOyJEmS1GFAliRJkjoMyJIkSVKHAVmSJEnqMCBLkiRJHQZkSZIkqcOALEmSJHUYkCVJkqQOA7LUB7fdcRevf/t7ecl+B/LS/Q7ipFN/2O+SJGlS7f7CXbj6qgv53TUXc8Thb+t3OdJilp3MnSWpUVb5RlUdMBm1SP207DLLcPg73swWm81kzpy57PPGQ9hp+23ZdOON+l2aJE24adOm8fnPHcseL3o1t956G7/65Vmc8R/ncu211/e7NAmY/B7kdTuPNw8x79DuykmWm9TqpEmy1pprsMVmMwFYeeWV2GSjDbjjrnv6XJUkTY4dtt+WP/7xJm688Wbmz5/Pqaf+iJfsvXu/y5IWmdSAXFW3DzyA+7rzgBWA+5K8Osn5SeYBByU5IMkD3XaS7JKkkqzZmbdTkguSzE0yK8lXkqw6iYcnjcms2+7g2uv/yNO33KzfpUjSpJi+3jrccuvsRdO3zrqN6dPX6WNF0uKm4hjkjwJfBrYAftjLBkm2As4FfgxsDbwc2Ab4+oRUKI2TuXPn8c73H8N7DzmIx6+8cr/LkSRJTM2A/IWq+l5V3VhVt/a4zeHAKVX1qaq6vqp+DbwFeEWStQevnOTAJJcmufSEb548nrVLPZv/0EMc9v5jePELd+UFuzyn3+VI0qSZPet2Nlh/+qLp9ddbl9mzb+9jRdLiJvUivR5dOoZtngHMTLJvZ17a/24K3NlduaqOA44DmH/3DaNdOCiNu6riqI9+lk022oD9X/XyfpcjSZPqkksvZ+bMjZkxYwNmzbqdffZ5Ka/7F+9koaljKgbkOYOmF/Jw2B0w+OK9acAJwGeGaG/WONUljZv/vvJqzjjnZzx50xm8Yv/mfwqHHrQ/O++0Q58rk6SJt2DBAg497AOcdeZ3WGbaNE78xilcc811/S5LWmQqBuTB7gJWSrJqVf21nbfNoHUuA7asqj9MamXSGG239dO46hdn97sMSeqbs885n7PPOb/fZUhDmopjkAf7NU2v8keTzEzyCuCtg9b5OLBDkq8m2bZdb68kX5v0aiVJkvQPbcoH5Kq6F9gPeAHwP8CBwJGD1rkS2BmYAVwAXEFzN4w7JrNWSZIk/eNL1dDXqCX5AjDsBWxVdchEFTWZvEhP0tJsxenP7XcJktQ3D/191uDr3ICRxyCP5W4SkiRJ0j+0YQNyVX2jO51kpaqaO/ElSZIkSf0z6hjkJDsmuQb4XTu9dZIvT3hlkiRJUh/0cpHeZ4HdgXsAquoKmgviJEmSpMecnu5iUVW3DJq1YAJqkSRJkvqulx8KuSXJTkAlWQ44FLh2YsuSJEmS+qOXHuSDgbcB6wGzaX7Fzh9MlyRJ0mPSqD3IVXU3zQ91SJIkSY95vdzFYpMkZyS5K8mdSX6UZJPJKE6SJEmabL0MsfgOcCqwLjAdOA04eSKLkiRJkvqll4C8UlWdVFUPtY9vAStMdGGSJElSPww7BjnJGu3Ts5P8H+C7QAH7AmdNQm2SJEnSpEtVDb0guZEmEGeIxVVVj4lxyPPvvmHoEyBJS4EVpz+33yVIUt889PdZQ+Xc4XuQq2rjiStHkiRJmpp6+aEQkjwN2ILO2OOq+uZEFSVJkiT1y6gBOckHgV1oAvJZwJ7AxYABWZIkSY85vdzF4pXA84Dbq+r1wNbAahNalSRJktQnvQTkeVW1EHgoyarAncAGE1uWJEmS1B+9jEG+NMnqwPHAb4EHgF9OZFGSJElSvwx7m7chV05mAKtW1ZUTVtEk8zZvkpZm3uZN0tJsiW/zlmS7kZZV1WXjUZgkSZI0lYw0xOJTIywrYLdxrqUv7D2RtDT71drb97sESZpyRvqhkF0nsxBJkiRpKujlLhaSJEnSUsOALEmSJHUYkCVJkqSOUQNyGq9NclQ7vWGSHSa+NEmSJGny9dKD/GVgR+DV7fT9wJcmrCJJkiSpj3r5Jb1nVdV2Sf4boKr+nGT5Ca5LkiRJ6oteepDnJ1mG5t7HJFkLWDihVUmSJEl90ktA/jzwA2DtJMcCFwMfmdCqJEmSpD4ZdYhFVX07yW+B5wEBXlZV1054ZZIkSVIfjBqQk2wIzAXO6M6rqpsnsjBJkiSpH3q5SO9MmvHHAVYANgZ+D2w5gXVJkiRJfdHLEIututNJtgPeOmEVSZIkSX20xL+kV1WXAc+agFokSZKkvutlDPK7OpPTgO2A2RNWkSRJktRHvYxBXqXz/CGaMcnfn5hyJEmSpP4aMSC3PxCySlW9Z5LqkSRJkvpq2DHISZatqgXAcyaxHkmSJKmvRupB/g3NeOPLk/wYOA2YM7Cwqk6f4NokSZKkSdfLGOQVgHuA3Xj4fsgFGJAlSZL0mDNSQF67vYPFVTwcjAfUhFYlSZIk9clIAXkZ4PEsHowHGJAlSZL0mDRSQL6tqo6etEo6kuwC/CewVlXd3Y8aJEmStHQa6Zf0huo5XiJJTkxSQzy2ebRtS//odn/hLlx91YX87pqLOeLwt/W7HEmaNDM++Xa2vvxEtjzvc/0uRRrSSAH5eeO0j/OAdQc9rhqntqV/SNOmTePznzuWvfZ+LVttvSv77vsyNt/8yf0uS5Imxd2nnc/1r+3Ll9RST4YNyFV17zjt48Gqur37AA5JcmWSOUlmJTkhyerDNZBktSQnJbkzyd+S3JDksEHLj2uX35/kgiTPHKf6pXG3w/bb8sc/3sSNN97M/PnzOfXUH/GSvXfvd1mSNCke+PU1PHTfA/0uQxrWSD3IE2khcBiwJfAaYAfgCyOsfwywFbAXsBnwBmAWQJLQ/Pz1eu3ybYELgfOTrDsx5UuPzvT11uGWW2cvmr511m1Mn75OHyuSJEkDerkP8qO1R5Lux8SLqmrPzvRNSY4AfpRk/6paOEQbGwGXVdVv2uk/dZbtCmxDc0HfvHbekUn2Bl4HfGJwY0kOBA4EyDKrMW3aymM5LkmSJD0GTUZAvpA2jLbmJdkNeB+wObAazS3llgfWAWY/ogX4CvC9JM8AfgqcUVUXtMueAawE3NV0Ji+yArDpUAVV1XHAcQDLLr+et6zTpJs963Y2WH/6oun111uX2bNv72NFkiRpwGQMsZhbVX8YeNCE8jOBa4F/pgm4b2jXXX6oBqrqbJpe5E8CawJnJvn3dvE04A6aXuTu46nAkeN+NNI4uOTSy5k5c2NmzNiA5ZZbjn32eSln/Me5/S5LkiQxOT3Igz2TJgi/s6oWACTZa7SN2vshnwSclORs4OQkBwOXAU8CFlbVDRNXtjR+FixYwKGHfYCzzvwOy0ybxonfOIVrrrmu32VJ0qTY+IvvYpUdn8aya6zK0y85gdmf+i53f/e8fpclLdKPgHw9Ta/vYUlOB55Nc8HesJIcTROEr6ap+eXADVX1YJLzgF/QjGE+AvgdzVCNPYDzquqiiToQ6dE4+5zzOfuc8/tdhiRNuhvf/ul+lyCNaNLvYlFVVwKHAu8CrgHeBLxnlM0eBI4FrqAJw6sAe7ftFfAi4HzgeOD3wKk0d7sYajyzJEmSNKw0+XLp5UV6kpZmv1p7+36XIEl988xbfzjkL0f36z7IkiRJ0pRkQJYkSZI6DMiSJElShwFZkiRJ6jAgS5IkSR0GZEmSJKnDgCxJkiR1GJAlSZKkDgOyJEmS1GFAliRJkjoMyJIkSVKHAVmSJEnqMCBLkiRJHQZkSZIkqcOALEmSJHUYkCVJkqQOA7IkSZLUYUCWJEmSOgzIkiRJUocBWZIkSeowIEuSJEkdqap+1yAttZIcWFXH9bsOSeoH3wM1VdmDLPXXgf0uQJL6yPdATUkGZEmSJKnDgCxJkiR1GJCl/nLsnaSlme+BmpK8SE+SJEnqsAdZkiRJ6jAgS5IkSR0GZEmS1JMkuySpJGv2uxZpIhmQJaB9wx/pcWK/a5Sk8ZLkxGHe67bpd23SVLBsvwuQpoh1O8/3Ao4fNG9ed+Uky1XV/MkoTJImyHnA6wbNu7sfhUhTjT3IElBVtw88gPu684AVgPuSvDrJ+UnmAQclOSDJA912hvr6MclOSS5IMjfJrCRfSbLqJB6eJA3lwe57X/t+d0iSK5PMad+vTkiy+nANJFktyUlJ7kzytyQ3JDls0PLj2uX3t++Fz5yEY5MeFQOy1LuPAl8GtgB+2MsGSbYCzgV+DGwNvBzYBvj6hFQoSY/OQuAwYEvgNcAOwBdGWP8YYCuab942A94AzAJIEuBMYL12+bbAhcD5SdYdsjVpinCIhdS7L1TV9wYmmvf+UR0OnFJVn+ps9xbgv5OsXVV3jn+ZktSTPQZ9C3ZRVe3Zmb4pyRHAj5LsX1ULh2hjI+CyqvpNO/2nzrJdaToE1qqqgWFqRybZm2ZoxyfG5SikCWBAlnp36Ri2eQYwM8m+nXkDyXpTwIAsqV8uBA7sTM9LshvwPmBzYDVgGWB5YB1g9hBtfAX4XpJnAD8FzqiqC9plzwBWAu4a1KGwAs37nzRlGZCl3s0ZNL2Qh8PugOUGTU8DTgA+M0R7s8apLkkai7lV9YeBiSQb0QyJOB44CrgH2A44mSYkP0JVnd1utyfwPODMJKdV1etp3v/uAJ47xKZ/Hc8DkcabAVkau7uAlZKsWlUDb/bbDFrnMmDL7v+EJGmKeiZNEH5nVS0ASLLXaBtV1d3AScBJSc4GTk5yMM3735OAhVV1w8SVLY0/L9KTxu7XNL3KH00yM8krgLcOWufjwA5Jvppk23a9vZJ8bdKrlaSRXU+TCw5LsnGSV9NcsDesJEcneVmSJyfZnOZC5Buq6kGa28j9gmYM855tmzsm+XCSoXqVpSnDgCyNUVXdC+wHvAD4H5qxfEcOWudKYGdgBnABcAXN3TDumMxaJWk07fvVocC7gGuANwHvGWWzB4Fjad7bfgGsAuzdtlfAi4DzaYZt/B44leZuF0ONZ5amjDR/v5IkSZLAHmRJkiRpMQZkSZIkqcOALEmSJHUYkCVJkqQOA7IkSZLUYUCWJEmSOgzIkjRJkixIcnmSq5KclmSlR9HWiUle2T4/IckWI6y7S5KdxrCPm5Ks2ev8Qes8sIT7+lCS0e65K0mTwoAsSZNnXlVtU1VPA/4OHNxdmGTZsTRaVW+qqmtGWGUXYIkDsiQtrQzIktQfFwEz297di5L8GLgmyTJJ/l+SS5JcmeQggDS+mOT3Sc4D1h5oKMnPkzyzfb5HksuSXJHkZ0lm0ATxd7a9189NslaS77f7uCTJc9ptn5jk3CRXJzkByGgHkeSHSX7bbnPgoGWfaef/LMla7bxNk5zTbnNRkqeOy9mUpHE0pt4KSdLYtT3FewLntLO2A55WVTe2IfMvVbV9kscBv0hyLrAtzU/0bgE8ieangL8+qN21aH7Sd+e2rTWq6t4kXwUeqKpPtut9B/hMVV2cZEPgJ8DmwAeBi6vq6CQvBt7Yw+G8od3HisAlSb5fVfcAKwOXVtU7kxzVtv124Djg4Kq6PsmzgC8Du43hNErShDEgS9LkWTHJ5e3zi4B/oxn68JuqurGd/0Lg6QPji4HVgCcDOwMnV9UCYHaS84do/9nAhQNtVdW9w9TxfGCLZFEH8apJHt/u4+Xttmcm+XMPx3RIkn9qn2/Q1noPsBA4pZ3/LeD0dh87Aad19v24HvYhSZPKgCxJk2deVW3TndEGxTndWcA7quong9Z70TjWMQ14dlX9bYhaepZkF5qwvWNVzU3yc2CFYVavdr/3DT4HkjTVOAZZkqaWnwBvSbIcQJKnJFkZuBDYtx2jvC6w6xDb/grYOcnG7bZrtPPvB1bprHcu8I6BiSTbtE8vBF7TztsTeMIota4G/LkNx0+l6cEeMA0Y6AV/Dc3Qjb8CNyb553YfSbL1KPuQpElnQJakqeUEmvHFlyW5Cvgazbd9PwCub5d9E/jl4A2r6i7gQJrhDFfw8BCHM4B/GrhIDzgEeGZ7EeA1PHw3jQ/TBOyraYZa3DxKrecAyya5FvgYTUAfMAfYoT2G3YCj2/n7AW9s67saeGkP50SSJlWqqt81SJIkSVOGPciSJElShwFZkiRJ6jAgS5IkSR0GZEmSJKnDgCxJkiR1GJAlSZKkDgOyJEmS1GFAliRJkjoMyJIkSVKHAVmSJEnqMCBLkiRJHQZkSZIkqcOALGnCJFkxyRlJ/pLktEfRzn5Jzh3P2vohydlJ9h/jtsckuTvJ7eNdV9v+LklunYi2H40kX01yZL/rGCzJTUmePwHtzkhSSZYd77Yl9c6ALIkkr0lyaZIHktzWBrn/NQ5NvxJ4EvDEqvrnsTZSVd+uqheOQz2LaUNhJfnBoPlbt/N/3mM7H0ryrdHWq6o9q+obY6hzQ+DdwBZVtc6Sbj9Mm5Vk5ni0NV6SHJDk4u68qjq4qv51kuvYr/238ECSeUkWdqYfmMxaOjWtmOT6wR+wkhyV5BdJ/P+5NI78ByUt5ZK8C/gs8BGaMLsh8GXgpePQ/EbAdVX10Di0NVHuAnZM8sTOvP2B68ZrB2k8mvfbDYF7qurOMezbnsgl1H4ge3xVPR7YE5g9MN3O60dN84A3Ap9M8iSAJJvTfHB6Y1Ut7Edd0mOVAVlaiiVZDTgaeFtVnV5Vc6pqflWdUVWHt+s8Lslnk8xuH59N8rh22S5Jbk3y7iR3tr3Pr2+XfRg4Cti37Xl74+Ce1sFfJ7c9iDckuT/JjUn268y/uLPdTkkuaYduXJJkp86ynyf517ZX7f4k5yZZc4TT8Hfgh8Cr2u2XAfYFvj3oXH0uyS1J/prkt0me287fA/i/neO8olPHsUl+AcwFNmnnvald/pUk3++0//EkP0uSQft9PvBTYHrb/ont/JckuTrJfW27m3e2uSnJe5NcCcwZHJKTXNg+vaJtc9/Oske8lu38xyX5ZJKbk9yRZujDikOd0CSbJjk/yT1phoV8O8nqneUbJDk9yV3tOl9s6/8qzYeVB5Lc1657YpJjOtu+Ockfktyb5MdJpneWVZKD257W+5J8afD5HGfbJLmy/Ts8JckKbR1PSPIf7fH9uX2+fqfOnv9Gk7yifT2fVlUXAqcCX2yP6wTgo8B1ST6Q5E/ta/fNNP+2hxw6kwkaHiI9lhiQpaXbjsAKwA9GWOf9wLOBbYCtgR2AD3SWrwOsBqxH08P1pSRPqKoP0vRKn9L2vP3bSIUkWRn4PLBnVa0C7ARcPsR6awBntus+Efg0cGYW7wF+DfB6YG1geeA9I+0b+CbwL+3z3YGrgNmD1rmE5hysAXwHOC3JClV1zqDj3LqzzeuAA4FVgD8Nau/dwFZt+H8uzbnbv6qqu1JVncfivZgHJHkKcDJwGLAWcBZwRpLlO5u+GngxsPrgHvyq2rl9unXb5int9JCvZbvsY8BT2nMws13nKIYWmuA2Hdgc2AD4ECz6APIf7fmY0bbz3aq6FjgY+GVb0+qPaDTZrW13H2Ddto3vDlptL2B74OntersPU+N42AfYA9i43d8B7fxpwL/TfIOyITAP+OKgbUf9G20/oHwceH5VXdXOfi/N8X0feBzw/9r9HgDsCmwCPH6I/UlaAgZkaen2RODuUYZA7AccXVV3VtVdwIdpgt+A+e3y+VV1FvAAsNkY61kIPC3JilV1W1VdPcQ6Lwaur6qTquqhqjoZ+B2wd2edf6+q69qvpU+lCXXDqqr/AtZIshlNUP7mEOt8q6ruaff5KZpwMtpxnlhVV7fbzB/U3lya8/hp4FvAO6qq14vk9gXOrKqftu1+EliR5kPFgM9X1S3tOejVkK9l21t5IPDOqrq3qu6n+VDwqqEaqao/tLU92P7NfBr43+3iHWiC8+HtNxZ/q6qLh2pnCPsBX6+qy6rqQeB9ND3OMzrrfKyq7quqm4H/ZJTX/lH6fFXNrqp7gTMG9tX+nXy/qua25+pYHj7+AaP9jR4GHA7sUlV/GJhZVQ8AbwP+iWZoxQKa8/LpqrqhXf4+4FWDvzmQ1DsDsrR0uwdYc5T/kU5n8d7PP7XzFrUxKGDPpenBWiJVNYcm+B0M3JbkzCRP7aGegZrW60x37/TQaz0nAW+n6YV7RI96kvckubb9Ov0+mp7WkYZuANwy0sKq+jVwA02P66k91DhgsXPQjj+9hcXPwYj7HsZwr+VawErAb9uhC/cB57TzHyHJk5J8N8msJH+l+QAwcK42AP40xnHpg4/7AZq/4SV+7dO56C7NRZBjMeS+kqyU5GvtkIe/AhcCq7e9573WeTjwpWE+NF096L9D/RtdluaaAkljYECWlm6/BB4EXjbCOrNpvioesCGPHH7Qqzk0QWvAYndkqKqfVNULaL4+/x1wfA/1DNQ0a4w1DTgJeCtwVtu7u0g7BOIImq/Un9B+/f8XmmALsNiwiI7h5g+0+zaanujZbfu9WuwctD28G7D4ORhx30vobpphAltW1ertY7URLlj7SLv/rapqVeC1PHyubgE2HOZD2Wg1Dz7ulWm+BVni17570V3b2zye3k3z7cKz2uMfGNKyJOOhXwh8IMkrelh3qH+jDwF3MOjfXBvSh/xgI+lhBmRpKVZVf6EZR/qlJC9re76WS7Jnkk+0q51M8z/qtdoLiY6i6REci8uBnZNs2F5E9L6BBW2v40vb0PMgzdf7Q12ZfxbwlDS3pls2zQVmW9CMax2zqrqR5mvw9w+xeBWawHEXsGySo4BVO8vvAGZkCe5U0Y4jPoYmPL4OOCLJNj1ufirw4iTPS7IcTSB7EPivXvff1rxJLyu2PdTHA59JsnZb/3pJhhvfuwrN6/eXJOvR9IYO+A1wG/CxJCsnWSHJczo1rT9oLHXXycDrk2yT5kLRjwC/rqqbejmOSbQKzQeK+9ox8x8cQxtX04xv/lKSl4yy7snAO5NsnOTxPDwm/iGau7GskOTF7d/KB2g+lEkagQFZWsq142nfRfM/zrtoevjeTnNnB2hC3KXAlcD/AJe188ayr58Cp7Rt/ZbFQ+20to7ZwL00YfUtQ7RxD82FWO+m+Xr9CGCvqrp7LDUNavviqhqqd/wnNEMKrqP5+vpvLD6EYeBHUO5Jctlo+2l7T78FfLyqrqiq62nuhHFSG/xGq/P3NMH6CzS9u3sDe1fV30fbtuNDwDfaIRP79LD+e4E/AL9qhw2cx/BjsD8MbEfTy34mcHqn9gVtvTOBm4FbaYbWAJxPEwxvT/KI17O9YPFImgvUbgM2ZZhx0H32WZox4XcDv6L521liVXUFzd/68Un2HGHVr9N8A3IhcCPN3+c72jb+QvPNyAk0Pe1zaM65pBFk0AXTkiRJ0lLNHmRJkiSpw4AsSZIkdRiQJUmSpA4DsiRJktSx1P/KzqXrv8yrFCUttZ595yX9LkGS+uahv88a8v7k9iBLkiRJHQZkSZIkqcOALEmSJHUYkCVJkqQOA7IkSZLUYUCWJEmSOgzIkiRJUocBWZIkSeowIEuSJEkdBmRJkiSpw4AsSZIkdRiQJUmSpA4DsiRJktRhQJYkSZI6DMiSJElShwFZkiRJ6jAgS5IkSR0GZEmSJKnDgCxJkiR1GJAlSZKkDgOyJEmS1GFAliRJkjoMyJIkSVLHpAbkJDXK48TJrEfqlxmffDtbX34iW573uX6XIkl9sfsLd+Hqqy7kd9dczBGHv63f5UiLmewe5HU7jzcPMe/Q7spJlpvU6qRJcvdp53P9a4/udxmS1BfTpk3j8587lr32fi1bbb0r++77Mjbf/Mn9LktaZFIDclXdPvAA7uvOA1YA7kvy6iTnJ5kHHJTkgCQPdNtJskvb47xmZ95OSS5IMjfJrCRfSbLqJB6e1LMHfn0ND933wOgrStJj0A7bb8sf/3gTN954M/Pnz+fUU3/ES/bevd9lSYtMxTHIHwW+DGwB/LCXDZJsBZwL/BjYGng5sA3w9QmpUJIkjdn09dbhlltnL5q+ddZtTJ++Th8rkhY3FQPyF6rqe1V1Y1Xd2uM2hwOnVNWnqur6qvo18BbgFUnWHrxykgOTXJrk0tPn3DSOpUuSJOkf3bL9LmAIl45hm2cAM5Ps25mX9r+bAnd2V66q44DjAC5d/2U1liIlSdLYzJ51OxusP33R9Prrrcvs2bf3sSJpcVMxIM8ZNL2Qh8PugMEX700DTgA+M0R7s8apLkmSNA4uufRyZs7cmBkzNmDWrNvZZ5+X8rp/8U4WmjqmYkAe7C5gpSSrVtVf23nbDFrnMmDLqvrDpFYmjdHGX3wXq+z4NJZdY1WefskJzP7Ud7n7u+f1uyxJmhQLFizg0MM+wFlnfodlpk3jxG+cwjXXXNfvsqRFUtWfEQZJXgmcVlVpp2cANwLbV9WlnfXWAG4GvkHTQ7w18ElgBrBWVd2d5OnAr4BvAl8D7geeCuxdVQeNVIdDLCQtzZ595yX9LkGS+uahv88aPEoBmJoX6S2mqu4F9gNeAPwPcCBw5KB1rgR2pgnNFwBX0NwN447JrFWSJEn/+IbtQU7yBWDY3tWqOmSiippM9iBLWprZgyxpaTZcD/JIY5DHcjcJSZIk6R/asAG5qr7RnU6yUlXNnfiSJEmSpP4ZdQxykh2TXAP8rp3eOsmXJ7wySZIkqQ96uUjvs8DuwD0AVXUFzQVxkiRJ0mNOT3exqKpbBs1aMAG1SJIkSX3Xyw+F3JJkJ6CSLAccClw7sWVJkiRJ/dFLD/LBwNuA9YDZNL9i5+9BSpIk6TFp1B7kqrqb5oc6JEmSpMe8Xu5isUmSM5LcleTOJD9KsslkFCdJkiRNtl6GWHwHOBVYF5gOnAacPJFFSZIkSf3SS0BeqapOqqqH2se3gBUmujBJkiSpH4Ydg5xkjfbp2Un+D/BdoIB9gbMmoTZJkiRp0o10kd5vaQJx2umDOssKeN9EFSVJkiT1y7ABuao2nsxCJEmSpKmglx8KIcnTgC3ojD2uqm9OVFGSJElSv4wakJN8ENiFJiCfBewJXAwYkCVJkvSY08tdLF4JPA+4vapeD2wNrDahVUmSJEl90ktAnldVC4GHkqwK3AlsMLFlSZIkSf3RyxjkS5OsDhxPc2eLB4BfTmRRkiRJUr+MGpCr6q3t068mOQdYtaqunNiyJEmSpP4Y6YdCthtpWVVdNjElSZIkSf2Tqhp6QfKfI2xXVbXbxJQ0uZZdfr2hT4AkLQXmzb6o3yVIUt8st+YmGWr+SD8UsuvElSNJkiRNTb3cxUKSJElaahiQJUmSpA4DsiRJktQxakBO47VJjmqnN0yyw8SXJkmSJE2+XnqQvwzsCLy6nb4f+NKEVSRJkiT1US+/pPesqtouyX8DVNWfkyw/wXVJkiRJfdFLD/L8JMsABZBkLWDhhFYlSZIk9UkvAfnzwA+AtZMcC1wMfGRCq5IkSZL6ZNQhFlX17SS/BZ4HBHhZVV074ZVJkiRJfTBqQE6yITAXOKM7r6punsjCJEmSpH7o5SK9M2nGHwdYAdgY+D2w5QTWJUmSJPVFL0MstupOJ9kOeOuEVSRJkiT10RL/kl5VXQY8awJqkSRJkvqulzHI7+pMTgO2A2ZPWEWSJElSH/UyBnmVzvOHaMYkf39iypEkSZL6a8SA3P5AyCpV9Z5JqkeSJEnqq2HHICdZtqoWAM+ZxHokSZKkvhqpB/k3NOONL0/yY+A0YM7Awqo6fYJrkyRJkiZdL2OQVwDuAXbj4fshF2BAliRJ0mPOSAF57fYOFlfxcDAeUBNalSRJktQnIwXkZYDHs3gwHmBAliRJ0mPSSAH5tqo6etIqkSRJkqaAkX5Jb6ie40mRZJcklWTNftUgTbTdX7gLV191Ib+75mKOOPxt/S5HkibNbXfcxevf/l5est+BvHS/gzjp1B/2uyRpMSP1ID/v0Tae5ERg/yEWbVtVlz/a9qV/VNOmTePznzuWPV70am699TZ+9cuzOOM/zuXaa6/vd2mSNOGWXWYZDn/Hm9lis5nMmTOXfd54CDttvy2bbrxRv0uTgBF6kKvq3nHax3nAuoMeV41T29I/pB2235Y//vEmbrzxZubPn8+pp/6Il+y9e7/LkqRJsdaaa7DFZjMBWHnlldhkow244657+lyV9LCRhliMlwer6vbuAzgkyZVJ5iSZleSEJKsP10CS1ZKclOTOJH9LckOSwwYtP65dfn+SC5I8cxKOTRqT6eutwy23zl40feus25g+fZ0+ViRJ/THrtju49vo/8vQtN+t3KdIikxGQh7IQOAzYEngNsAPwhRHWPwbYCtgL2Ax4AzALIEmAM4H12uXbAhcC5ydZd2LKlyRJj9bcufN45/uP4b2HHMTjV1653+VIi/TyQyGP1h5JHuhMX1RVe3amb0pyBPCjJPtX1cIh2tgIuKyqftNO/6mzbFdgG2CtqprXzjsyyd7A64BPDG4syYHAgQBZZjWmTfMfpSbX7Fm3s8H60xdNr7/eusyefXsfK5KkyTX/oYc47P3H8OIX7soLdnlOv8uRFjMZAflC2jDampdkN+B9wObAajT3XF4eWAeY/YgW4CvA95I8A/gpcEZVXdAuewawEnBX05m8yArApkMVVFXHAccBLLv8et7TWZPukksvZ+bMjZkxYwNmzbqdffZ5Ka/7F+9kIWnpUFUc9dHPsslGG7D/q17e73KkR5iMgDy3qv4wMJFkI5ohEccDR9H8jPV2wMk0IfkRqursdrs9ae6ucWaS06rq9TTDRO4AnjvEpn8dzwORxsuCBQs49LAPcNaZ32GZadM48RuncM011/W7LEmaFP995dWccc7PePKmM3jF/k3nwKEH7c/OO+3Q58qkRqomrgO1vc3bmlW1V2feK4BTgeWrakE7713Ap4CNq+qmJLsA/0kzbOLuIdrdlyZQrwjsDPwEmFlVNyxpjfYgS1qazZt9Ub9LkKS+WW7NTYb83Y/J6EEe7HqaXt/DkpwOPJvmgr1hJTkauAy4mqbmlwM3VNWDSc4DfkEzhvkI4Hc0QzX2AM6rKt/9JUmS1LNJv4tFVV0JHAq8C7gGeBPwnlE2exA4FriCJgyvAuzdtlfAi4DzaYZt/J6mh3ozhh7PLEmSJA1rQodY/CNwiIWkpZlDLCQtzYYbYtGv+yBLkiRJU5IBWZIkSeowIEuSJEkdBmRJkiSpw4AsSZIkdRiQJUmSpA4DsiRJktRhQJYkSZI6DMiSJElShwFZkiRJ6jAgS5IkSR0GZEmSJKnDgCxJkiR1GJAlSZKkDgOyJEmS1GFAliRJkjoMyJIkSVKHAVmSJEnqMCBLkiRJHQZkSZIkqcOALEmSJHWkqvpdg7TUSnJgVR3X7zokqR98D9RUZQ+y1F8H9rsASeoj3wM1JRmQJUmSpA4DsiRJktRhQJb6y7F3kpZmvgdqSvIiPUmSJKnDHmRJkiSpw4AsSZIkdRiQJUlST5LskqSSrNnvWqSJZECWgPYNf6THif2uUZLGS5ITh3mv26bftUlTwbL9LkCaItbtPN8LOH7QvHndlZMsV1XzJ6MwSZog5wGvGzTv7n4UIk019iBLQFXdPvAA7uvOA1YA7kvy6iTnJ5kHHJTkgCQPdNsZ6uvHJDsluSDJ3CSzknwlyaqTeHiSNJQHu+997fvdIUmuTDKnfb86IcnqwzWQZLUkJyW5M8nfktyQ5LBBy49rl9/fvhc+cxKOTXpUDMhS7z4KfBnYAvhhLxsk2Qo4F/gxsDXwcmAb4OsTUqEkPToLgcOALYHXADsAXxhh/WOArWi+edsMeAMwCyBJgDOB9drl2wIXAucnWXfI1qQpwiEWUu++UFXfG5ho3vtHdThwSlV9qrPdW4D/TrJ2Vd05/mVKUk/2GPQt2EVVtWdn+qYkRwA/SrJ/VS0coo2NgMuq6jft9J86y3al6RBYq6oGhqkdmWRvmqEdnxiXo5AmgAFZ6t2lY9jmGcDMJPt25g0k600BA7KkfrkQOLAzPS/JbsD7gM2B1YBlgOWBdYDZQ7TxFeB7SZ4B/BQ4o6ouaJc9A1gJuGtQh8IKNO9/0pRlQJZ6N2fQ9EIeDrsDlhs0PQ04AfjMEO3NGqe6JGks5lbVHwYmkmxEMyTieOAo4B5gO+BkmpD8CFV1drvdnsDzgDOTnFZVr6d5/7sDeO4Qm/51PA9EGm8GZGns7gJWSrJqVQ282W8zaJ3LgC27/xOSpCnqmTRB+J1VtQAgyV6jbVRVdwMnASclORs4OcnBNO9/TwIWVtUNE1e2NP68SE8au1/T9Cp/NMnMJK8A3jponY8DOyT5apJt2/X2SvK1Sa9WkkZ2PU0uOCzJxkleTXPB3rCSHJ3kZUmenGRzmguRb6iqB2luI/cLmjHMe7Zt7pjkw0mG6lWWpgwDsjRGVXUvsB/wAuB/aMbyHTlonSuBnYEZwAXAFTR3w7hjMmuVpNG071eHAu8CrgHeBLxnlM0eBI6leW/7BbAKsHfbXgEvAs6nGbbxe+BUmrtdDDWeWZoy0vz9SpIkSQJ7kCVJkqTFGJAlSZKkDgOyJEmS1GFAliRJkjoMyJIkSVKHAVmSJEnqMCBL0iRJsiDJ5UmuSnJakpUeRVsnJnll+/yEJFuMsO4uSXYawz5uSrJmr/MHrfPAEu7rQ0lGu+euJE0KA7IkTZ55VbVNVT0N+DtwcHdhkmXH0mhVvamqrhlhlV2AJQ7IkrS0MiBLUn9cBMxse3cvSvJj4JokyyT5f0kuSXJlkoMA0vhikt8nOQ9Ye6ChJD9P8sz2+R5JLktyRZKfJZlBE8Tf2fZePzfJWkm+3+7jkiTPabd9YpJzk1yd5AQgox1Ekh8m+W27zYGDln2mnf+zJGu18zZNck67zUVJnjouZ1OSxtGYeiskSWPX9hTvCZzTztoOeFpV3diGzL9U1fZJHgf8Ism5wLY0P9G7BfAkmp8C/vqgdtei+Unfndu21qiqe5N8FXigqj7Zrvcd4DNVdXGSDYGfAJsDHwQurqqjk7wYeGMPh/OGdh8rApck+X5V3QOsDFxaVe9MclTb9tuB44CDq+r6JM8CvgzsNobTKEkTxoAsSZNnxSSXt88vAv6NZujDb6rqxnb+C4GnD4wvBlYDngzsDJxcVQuA2UnOH6L9ZwMXDrRVVfcOU8fzgS2SRR3EqyZ5fLuPl7fbnpnkzz0c0yFJ/ql9vkFb6z3AQuCUdv63gNPbfewEnNbZ9+N62IckTSoDsiRNnnlVtU13RhsU53RnAe+oqp8MWu9F41jHNODZVfW3IWrpWZJdaML2jlU1N8nPgRWGWb3a/d43+BxI0lTjGGRJmlp+ArwlyXIASZ6SZGXgQmDfdozyusCuQ2z7K2DnJBu3267Rzr8fWKWz3rnAOwYmkmzTPr0QeE07b0/gCaPUuhrw5zYcP5WmB3vANGCgF/w1NEM3/grcmOSf230kydaj7EOSJp0BWZKmlhNoxhdfluQq4Gs03/b9ALi+XfZN4JeDN6yqu4ADaYYzXMHDQxzOAP5p4CI94BDgme1FgNfw8N00PkwTsK+mGWpx8yi1ngMsm+Ra4GM0AX3AHGCH9hh2A45u5+8HvLGt72rgpT2cE0maVKmqftcgSZIkTRn2IEuSJEkdBmRJkiSpw4AsSZIkdRiQJUmSpA4DsiRJktRhQJYkSZI6DMiSJElSx/8H3zBc9nDRdfwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(2, 1, figsize=(10, 5))\n",
    "for axes, cfs_matrix, label in zip(ax.flatten(), confusion_matrix, actions):\n",
    "        generate_heatmap(cfs_matrix, axes, label, [\"True\", \"False\"])\n",
    "fig.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4cd6bea-bdd4-4c47-bac3-ca491b9258bc",
   "metadata": {},
   "source": [
    "##### Accuracy Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "ac5ff533-4d27-4fb9-9925-f410ee3b6948",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(ytrue, yhat)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02b91850-5a40-4f3e-900f-bd7ff10492ee",
   "metadata": {},
   "source": [
    "## Real Time Prediction Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "e4b54579-777b-4ab5-b09e-d9a099591141",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyttsx3 as px"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "52e174ee-abb2-4841-9982-1dc8ad715e20",
   "metadata": {},
   "outputs": [],
   "source": [
    "# text-to-speech initiation\n",
    "engine = px.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "c12b0eb2-3a14-46ef-bc44-929ac4feb025",
   "metadata": {},
   "outputs": [],
   "source": [
    "sequence = []\n",
    "sentence = []\n",
    "predictions = []\n",
    "threshold = 0.5\n",
    "\n",
    "cap = cv2.VideoCapture(0)\n",
    "\n",
    "with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:\n",
    "    while cap.isOpened():\n",
    "        ret, frame = cap.read()\n",
    "        \n",
    "        image, results = mediapipe_detection(frame, holistic)\n",
    "\n",
    "        keypoints = extract_keypoints(results)\n",
    "        sequence.append(keypoints)\n",
    "        sequence = sequence[-30:]\n",
    "        \n",
    "        if len(sequence) == 30:\n",
    "            res = model.predict(np.expand_dims(sequence, axis=0))[0]\n",
    "            predictions.append(np.argmax(res))\n",
    "            \n",
    "            if np.unique(predictions[-10:])[0]==np.argmax(res): \n",
    "                if res[np.argmax(res)] > threshold: \n",
    "                    if len(sentence) > 0: \n",
    "                        if actions[np.argmax(res)] != sentence[-1]:\n",
    "                            sentence.append(actions[np.argmax(res)])\n",
    "                            engine.say(words[np.argmax(res)])\n",
    "                            engine.runAndWait()\n",
    "                    else:\n",
    "                        sentence.append(actions[np.argmax(res)])\n",
    "                        engine.say(words[np.argmax(res)])\n",
    "                        engine.runAndWait()\n",
    "\n",
    "            if len(sentence) > 5: \n",
    "                sentence = sentence[-5:]\n",
    "            \n",
    "        cv2.rectangle(image, (0,0), (640, 40), (245, 117, 16), -1)\n",
    "        cv2.putText(image, ' '.join(sentence), (3,30), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA)\n",
    "        \n",
    "        cv2.imshow('OpenCV Feed', image)\n",
    "\n",
    "        if cv2.waitKey(10) & 0xFF == ord('q'):\n",
    "            break\n",
    "\n",
    "cap.release()\n",
    "cv2.destroyAllWindows()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}