chatbot_with_gpt2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "chatbot_with_gpt2.ipynb",
      "private_outputs": true,
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNtMKmc3z8fL6KxNerlmNnA",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ksk0629/chatbot_with_gpt2/blob/refactoring/chatbot_with_gpt2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Chatbot with GPT-2\n",
        "- References\n",
        "  - https://qiita.com/Yokohide/items/e74254f334e1335cd502\n",
        "  - https://huggingface.co/rinna"
      ],
      "metadata": {
        "id": "zU1EdfzkwSup"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Preparation"
      ],
      "metadata": {
        "id": "pIYEQNKHye8Z"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from google.colab import drive\n",
        "import pickle"
      ],
      "metadata": {
        "id": "V0KAPjRBxW9h"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Mount my google drive\n",
        "drive_path = \"/content/gdrive\"\n",
        "drive.mount(drive_path)\n",
        "\n",
        "# Prepare environment\n",
        "!pip install PyYAML==5.4  # reference: https://github.com/ultralytics/yolov5/issues/414]\n",
        "\n",
        "!pip install sentencepiece\n",
        "!pip install datasets\n",
        "!pip install git+https://github.com/huggingface/transformers\n",
        "\n",
        "# from pyngrok import ngrok\n",
        "import yaml\n",
        "\n",
        "# Load general config\n",
        "config_path = os.path.join(drive_path, \"MyDrive\", \"config\", \"general_config.yaml\")\n",
        "with open(config_path, 'r') as yml:\n",
        "  config = yaml.safe_load(yml)\n",
        "\n",
        "config_github = config[\"github\"]\n",
        "\n",
        "# Set git configs\n",
        "!git config --global user.email {config_github[\"email\"]}\n",
        "!git config --global user.name {config_github[\"username\"]}\n",
        "\n",
        "# Clone the repository\n",
        "repository_name = \"chatbot_with_gpt2\"\n",
        "git_repository = f\"https://github.com/ksk0629/\" + repository_name + \".git\"\n",
        "repository_path = \"/content/\" + repository_name\n",
        "!git clone --recursive {git_repository}\n",
        "\n",
        "# Change directory to the cloned directory\n",
        "%cd {repository_name}"
      ],
      "metadata": {
        "id": "HUCmSo3VwfBZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Checkout\n",
        "branch_name = \"main\"\n",
        "!git checkout {branch_name}"
      ],
      "metadata": {
        "id": "UOSFUYbjXF8i"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Pull\n",
        "!git pull\n",
        "# !git submodule update --init --recursive"
      ],
      "metadata": {
        "id": "57wCeUGhXGuY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Preprocessing data"
      ],
      "metadata": {
        "id": "MrgsGf9IEXnu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!python src/preprocessor.py"
      ],
      "metadata": {
        "id": "cBTrjs6AEX4O"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Training data preparation"
      ],
      "metadata": {
        "id": "emdoZoWDmXg2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!python src/training_data_maker.py"
      ],
      "metadata": {
        "id": "zNJAqo5kmcUC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Building model"
      ],
      "metadata": {
        "id": "vgePrN8tq_JP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import yaml\n",
        "with open(\"model_config.yaml\", \"rb\") as yaml_f:\n",
        "  config = yaml.safe_load(yaml_f)\n",
        "config_general = config[\"general\"]\n",
        "config_dataset = config[\"dataset\"]\n",
        "config_train = config[\"train\"]\n",
        "\n",
        "!python ./transformers/examples/pytorch/language-modeling/run_clm.py \\\n",
        "    --model_name_or_path={config_general[\"basemodel\"]} \\\n",
        "    --train_file={config_dataset[\"output_path\"]} \\\n",
        "    --validation_file={config_dataset[\"output_path\"]} \\\n",
        "    --do_train \\\n",
        "    --do_eval \\\n",
        "    --num_train_epochs={config_train[\"epochs\"]} \\\n",
        "    --save_steps={config_train[\"save_steps\"]} \\\n",
        "    --save_total_limit={config_train[\"save_total_limit\"]} \\\n",
        "    --per_device_train_batch_size={config_train[\"per_device_train_batch_size\"]} \\\n",
        "    --per_device_eval_batch_size={config_train[\"per_device_eval_batch_size\"]} \\\n",
        "    --output_dir={config_train[\"output_dir\"]} \\\n",
        "    --use_fast_tokenizer={config_train[\"use_fast_tokenizer\"]}"
      ],
      "metadata": {
        "id": "jMwdakVcrCai"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!cp -r model ./../gdrive/MyDrive/"
      ],
      "metadata": {
        "id": "0zHkFKbLGgRZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Talking with the model"
      ],
      "metadata": {
        "id": "eqebw_V7h3OR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!python src/conversational_model.py"
      ],
      "metadata": {
        "id": "_RYKIf9Ih4AJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## MLflow"
      ],
      "metadata": {
        "id": "NP89dlXuYpa3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# # Run MLflow\n",
        "# get_ipython().system_raw(\"mlflow ui --port 5000 &\") # run tracking UI in the background\n",
        "\n",
        "# # Terminate open tunnels if exist\n",
        "# ngrok.kill()\n",
        "\n",
        "# # Setting the authtoken of ngrok\n",
        "# ngrok.set_auth_token(config_ngrok[\"token\"])\n",
        "\n",
        "# # Open an HTTPs tunnel on port 5000 for http://localhost:5000\n",
        "# ngrok_tunnel = ngrok.connect(addr=\"5000\", proto=\"http\", bind_tls=True)\n",
        "# print(\"MLflow Tracking UI:\", ngrok_tunnel.public_url)"
      ],
      "metadata": {
        "id": "FZ5qQSFWwC4A"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Adding files to the git repository"
      ],
      "metadata": {
        "id": "m6uMRCNk827k"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# add_objects = os.path.join(repository_path, \"mlruns\", \"*\")\n",
        "# !git add {add_objects}"
      ],
      "metadata": {
        "id": "ELNkcvhk8-7B"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# commit_msg = \"Added new mlruns data\"\n",
        "# !git commit -m {commit_msg}"
      ],
      "metadata": {
        "id": "3kOh17IE9AOM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# html = f\"https://{config_github['token']}@github.com/{config_github['username']}/{repository_name}.git\"\n",
        "# !git remote set-url origin {html}\n",
        "# !git push origin {branch_name}"
      ],
      "metadata": {
        "id": "S_yfeUVs9AQ_"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}