play-transformer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_cls validation set size 2658\n"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "local_path = './'\n",
    "\n",
    "\"\"\"## Prepare fastai\"\"\"\n",
    "from fastai import *\n",
    "from fastai.text import *\n",
    "from fastai.metrics import *\n",
    "\n",
    "\"\"\"## Prepare Dataset\"\"\"\n",
    "local_project_path = local_path + 'dna-10class/'\n",
    "if not os.path.exists(local_project_path):\n",
    "  os.makedirs(local_project_path)\n",
    "print('local_project_path:', local_project_path)\n",
    "\n",
    "\"\"\"## Create Language Model\"\"\"\n",
    "class dna_tokenizer(BaseTokenizer):\n",
    "  def tokenizer(slef, t):\n",
    "    return list(t)\n",
    "tokenizer = Tokenizer(tok_func=dna_tokenizer, pre_rules=[], post_rules=[], special_cases=[])\n",
    "processor = [TokenizeProcessor(tokenizer=tokenizer, include_bos= False, include_eos=False), NumericalizeProcessor(max_vocab=30000)]\n",
    "# batch size\n",
    "bs = 128\n",
    "data_lm = TextLMDataBunch.from_csv(local_project_path, 'combined.csv',\n",
    "                                   text_cols ='Text', valid_pct= 0.1, tokenizer=tokenizer,\n",
    "                                   include_bos= False, include_eos=False)\n",
    "\n",
    "print('data_cls validation set size', len(data_lm.valid_ds))\n",
    "learn_lm = language_model_learner(data_lm, TransformerXL, drop_mult=0.3, pretrained=False).to_fp16()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ0AAAEKCAYAAADJvIhZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4VeW5/vHvk4QwhJmEeQjzIJNMigOTVVHboscO2lq1arWnDqeDnc/R03raagf91U6KVlGrVmtrrVoFpCIWQQVB5jGEmSSEKRMZn98fe1EiDSEhe2ftndyf69oX2Wvaz34Tcme9693vMndHRESkMSSFXYCIiDQfCh0REWk0Ch0REWk0Ch0REWk0Ch0REWk0Ch0REWk0Ch0REWk0Ch0REWk0Ch0REWk0KWEX0BjS09M9MzMz7DJERBLK8uXL97t7RjSP2SxCJzMzk2XLloVdhohIQjGz7dE+prrXRESk0Sh0RESk0cQsdMzsMTPLNbM1J1k/y8xWmdlKM1tmZudVW/dTM1trZuvN7EEzs2D5QjPbGOyz0sy6xqp+ERGJvlie6cwBZtayfgEwxt3HAjcAjwKY2TnAucBoYCQwEZhabb/Pu/vY4JEbi8JFRCQ2YhY67r4IOFDL+kI/fjOfNODY1w60AlKBlkALICdWdYqISOMJ9ZqOmV1hZhuAV4mc7eDuS4A3gb3BY667r6+22+NB19r/HOt2ExGRxBBq6Lj7i+4+DLgcuAfAzAYBw4HeQC9ghpmdH+zyeXcfBZwfPL5wsmOb2c3BtaJleXl5sXwbIiJSR3Exei3oihtgZunAFcDSoPutEHgNmBxstzv4twB4BphUyzFnu/sEd5+QkRHVzzaJiMS9/YWl/ODltRwtrwy7lI8ILXTMbFC1UWnjiFy/yQd2AFPNLMXMWhAZRLA+eJ4ebN8C+DhQ48g4EZHmrKSskhufWMaz7+1ga15h2OV8RMxmJDCzZ4FpQLqZ7QLuJjIoAHd/CLgSuNbMyoES4LPu7mb2AjADWE1kUMHr7v6ymaUBc4PASQbeAB6JVf0iIomossq5/dkVrN51iIeuGc8ZPTuEXdJHxCx03P3qU6y/D7ivhuWVwC01LC8CxketQBGRJsbd+d+/reWN9Tn8cNYZXHRG97BL+jdxcU1HREQa7uFFWTy1dDu3TBnAtZMzwy6nRgodEZEm4KWVu7n3tQ18YkxPvj1zWNjlnJRCR0QkwS3Zms+df/qQs/p35uefHk1SUvx+hFGhIyKSwLLyCrn5qWVkdklj9hcm0DIlOeySaqXQERFJYI8vzqa8sorHvziRDm1ahF3OKSl0REQSVFWVM3ftPqYN6UrvTm3CLqdOFDoiIglqxc6D5BaUcsmo+BsafTIKHRGRBPX6mn20SDamD0ucW4spdEREEpC789qafZw3KJ32reL/Ws4xCh0RkQS0ds8Rdh0sYebIxOlaA4WOiEhCmrt2H0kGF45Q6IiISIy9tmYfZ/XvQue01LBLqReFjohIgtmSW8CW3MKE61oDhY6ISMJ5fc0+AC6Ow1mkT0WhIyKSYF5fu48z+3ake4dWYZdSbwodEZEEsvNAMWt2H+GSBOxaA4WOiEhCmbs20rU284weIVdyehQ6IiIJ5LU1+xjRoz19uyTGXGsnUuiIiCSI3CNHWb79YEKOWjtGoSMikiDmrssBUOiIiEjsvb5mLwMy0hjctW3YpZw2hY6ISAI4WFTG0qwDzDyjO2bxezvqU1HoiIgkgHnr9lFZ5VwyMjFHrR2j0BERiXNHyyv51T+2MLRbO0b2ah92OQ2SEnYBIiJSu98u3MqugyX88eazE7prDXSmIyIS17bnF/HQW1v55JienD2gS9jlNJhCR0Qkjv3w5XW0SDK+f9nwsEuJCoWOiEicWrA+hwUbcvmvjw2mW/vEm9yzJgodEZE4dLS8kh+8vI5BXdvyxXP7h11O1GgggYhIHHr4rSx2HCjmmZvOokVy0zk/aDrvRESkidh5oJjfLtzCZaN7cM6g9LDLiSqFjohInPnBy+tITjL+u4kMHqhOoSMiEkfe3JjLG+tzuH3GYHp0aB12OVGn0BERiROVVc6PX11P//Q0bjyv6QweqE6hIyISJ/66Yjebcwu586KhpKY0zV/PTfNdiYgkmLKKKh54YxMje7XnkgS+X86pKHREROLAs+/tYNfBEr558TCSkhJ7frXaKHREREJWVFrBr/6xmbP6d2bK4KY1RPpECh0RkZA9vngb+wvL+NbMYQk/i/SpxDR0zOwxM8s1szUnWT/LzFaZ2UozW2Zm51Vb91MzW2tm683sQQu+E2Y23sxWm9mW6stFRBLRoeIyHl6UxceGd2V8v05hlxNzsT7TmQPMrGX9AmCMu48FbgAeBTCzc4BzgdHASGAiMDXY53fAl4DBwaO244uIxLXfvbWVwtIK7rx4aNilNIqYho67LwIO1LK+0N09eJoGHPvagVZAKtASaAHkmFkPoL27Lw32exK4PFb1i4jEUs6Ro8xZnM2sMT0Z1j2x7whaV6Ff0zGzK8xsA/AqkbMd3H0J8CawN3jMdff1QC9gV7XddwXLREQSzoMLNlNZ5Xz9wuZxlgNxEDru/qK7DyNyxnIPgJkNAoYDvYmEygwzO78+xzWzm4PrRMvy8vKiXbaISINszy/iufd3cvWkvvTt0ibschpN6KFzTNAVN8DM0oErgKVB91sh8BowGdhNJIiO6R0sq+l4s919grtPyMjIiHH1IiL188s3NpOSbNw+Y1DYpTSqUEPHzAZVG5U2jsj1m3xgBzDVzFLMrAWRQQTr3X0vcMTMzg72uxZ4KaTyRUROy55DJbz04R4+N6kfXZvIHUHrKqY3cTOzZ4FpQLqZ7QLuJjIoAHd/CLgSuNbMyoES4LPu7mb2AjADWE1kUMHr7v5ycNivEBkV15rIGdBrsXwPIiLR9vjibQDccF5muIWEIKah4+5Xn2L9fcB9NSyvBG45yT7LiAyjFhFJOEeOlvPsezu5bFQPendqPtdyjombazoiIs3BH9/bQWFpBV86f0DYpYRCoSMi0kjKKqp47J/ZTB7QhVG9O4RdTigUOiIijeTV1XvYd+QoN09tnmc5oNAREWkU7s7sRdsY3LUt04Y0349xKHRERBrB4i35rN97hC9NGdDkZ5KujUJHRKQRzH47i4x2LZk1tmfYpYRKoSMiEmPr9x5h0aY8rj8nk5YpyWGXEyqFjohIjD3ydhZtUpP5/Fl9wy4ldAodEZEY2nu4hL+t3MNnJvShY5vUsMsJnUJHRCSG5ryTTZU7N57XP+xS4oJCR0QkRorLKnj23R1cMrIHfTo3vylvaqLQERGJkRdX7ObI0Qq+eG5m2KXEDYWOiEgMuDtPvJPNGT3bM75fp7DLiRsKHRGRGFiSlc+mnEKuOyezWX8Y9EQKHRGRGJizOJvOaal8ckzz/jDoiRQ6IiJRtvNAMW+sz+GqiX1o1aJ5fxj0RAodEZEo+8PS7ZgZ15zdL+xS4o5CR0QkikrKKvnj+zu5+Ixu9OzYOuxy4o5CR0Qkil5auZvDJeVcNzkz7FLikkJHRCRK3J0572QzrHs7JvXvHHY5cUmhIyISJe9uO8CGfQV88VwNkz4ZhY6ISJTMWZxNxzYtmDW2V9ilxC2FjohIFOw+VMK8dfu4amJfDZOuhUJHRKSBKqucR9/OAuCas3XPnNqkhF2AiEgiKjhaztub97NgfS4LN+aSX1TGx0f3oHcnzSZdG4WOiEg9/HXFbl5Yvot3t+VTXul0aN2CaUMzuGB4Ny4a0S3s8uKeQkdEpI7+8sEuvv78hwxIT+OGc/szY1hXxvfrREqyrlTUlUJHRKQOVu06xHf+spqzB3TmqRvPooWC5rSo1URETiGvoJRbnlpORtuW/OZz4xQ4DaAzHRGRWpRVVHHr0x9wsLiMF758Dl3atgy7pISm0BERqcU9r6zjvewD/PKqsYzs1SHschKezhFFRE7ij+/t4Kml27l5ygDNMhAlCh0RkRos336Qu15ay/mD0/n2zGFhl9NkqHtNRKSa3YdKeO79nTy5JJvuHVrxq6vPJDlJk3dGi0JHRJq9yirnzQ25PPPeDhZuzMWBKYMzuOsTI+jYJjXs8poUhY6INFtVVc7Di7J4ckk2ew8fJaNdS/5z2kCumtiXPp01nU0sKHREpNn624d7uO/1DZwzsAt3f2IEFwzvps/gxJhCR0SapbKKKu6fv4nhPdrzhxvPIknXbRqFIl1EmqXn3t/BjgPFfOvioQqcRlSn0DGzgWbWMvh6mpndYWYdT7HPY2aWa2ZrTrJ+lpmtMrOVZrbMzM4Llk8Plh17HDWzy4N1c8xsW7V1Y+v3dkVEoLisggf/sYVJmZ2ZNjQj7HKalbqe6fwZqDSzQcBsoA/wzCn2mQPMrGX9AmCMu48FbgAeBXD3N919bLB8BlAMzKu23zePrXf3lXWsX0TkXx5fnE1eQSnfmjkUM53lNKa6hk6Vu1cAVwC/cvdvAj1q28HdFwEHallf6O4ePE0DvIbNPgW85u7FdaxTRKRWh4vLefitrVwwrCsTMjuHXU6zU9fQKTezq4HrgFeCZS0a+uJmdoWZbQBeJXK2c6KrgGdPWPajoFvugWNdfiIidfW7t7ZSUFrBnRcPDbuUZqmuofNFYDLwI3ffZmb9gaca+uLu/qK7DwMuB+6pvs7MegCjgLnVFn8XGAZMBDoD3z7Zsc3s5uBa0bK8vLyGlioiTUDOkaPMeWcbs8b0ZHiP9mGX0yzVKXTcfZ273+Huz5pZJ6Cdu98XrSKCrrgBZpZebfFngBfdvbzadns9ohR4HJhUyzFnu/sEd5+QkaELhSICDy7YTEWl87ULh4RdSrNV19FrC82svZl1Bj4AHjGz+xvywmY2yIIreGY2DmgJ5Ffb5GpO6FoLzn4I9rscqHFknIjIibL3F/Hc+zu5elJf+nVJC7ucZquuHw7t4O5HzOwm4El3v9vMVtW2g5k9C0wD0s1sF3A3wXUgd38IuBK41szKgRLgs8cGFphZJpERcm+dcNinzSwDMGAl8OU61i8izdz98zfRIjmJ22cMCruUZq2uoZMSnGV8Bvh+XXZw96tPsf4+oMYuOnfPBv7t5hXuPqMury0iUt1bm/L424d7+Mq0gXRt3yrscpq1ug4k+CGRC/pb3f19MxsAbI5dWSIi0fHetgPc8tQyhvdoz39OGxh2Oc1enc503P1PwJ+qPc8i0j0mIhK3Vu06xA1z3qdnx9Y8deMk2rVq8Cc9pIHqOpCgt5m9GExrk2tmfzaz3rEuTkTkdG3KKeC6x96jY5sWPH3TWaS31cf64kFdu9ceB/4G9AweLwfLRETiTvb+Ij7/6Lu0SE7i6ZvOokeH1mGXJIG6hk6Guz/u7hXBYw6gD7+ISNzZc6iEzz/6LhWVVTx901kaHh1n6ho6+WZ2jZklB49r+OhnakREQnewqIxrHn2XIyXlPHnDWQzu1i7skuQEdQ2dG4gMl94H7CUyEef1MapJROS0/HbhFrLzi/j99RMZ1btD2OVIDeo6Dc52d/+ku2e4e1d3vxyNXhOROLK/sJSnlm5n1theTOqv2aPjVUPuHPr1qFUhItJAjyzKoqyiits040Bca0jo6M5HIhIX8gtLeXLJdj4xpicDM9qGXY7UoiGhU9NN10REGt0jb2/jaEWl5lVLALXOSGBmBdQcLgZo4LuIhO5AURlPLsnm46N7MqirRqvFu1pDx931HRSRuPbo21mUlFdyh85yEkJDutdEREJ1sKiMJ97J5tJRPfSZnASh0BGRhPX7f26jqKySO2YMDrsUqSOFjogkpEPFZcx5J5vLRvVgaHed5SQKhY6IJKTH/rmNwtIKbr9A13ISiUJHRBLO4eJyHl+czSUjuzOse/uwy5F6UOiISEJZt+cIn374HYrKKrhd13ISTp3uHCoiEraqKufRf2bx87mb6NCmBY9dP5ERPXWWk2gUOiIS9/YcKuEbz3/Ikqx8LhrRjXuvHE3ntNSwy5LToNARkbj2tw/38N8vrqaiyvnplaP59ITemGnqx0Sl0BGRuPXkkmzuemktZ/btyP/77FjdBbQJUOiISFwqLqvgl29sZvKALjx14yRSkjXuqSnQd1FE4tJTS7aTX1TGnRcPUeA0IfpOikjcKS6r4OFFWZw/OJ3x/XQX0KZEoSMicefJJds5UFTGVz82JOxSJMoUOiISV4pKK5i9KIspQzIY369T2OVIlCl0RCSuHD/L0WwDTZFCR0TiRuQsZytTh2Qwrq/OcpoihY6IxI0nlmRzsLhcZzlNmEJHROJCYWkFjyzKYtrQDM7UWU6TpdARkbjwxDvHznI0Yq0pU+iISOgKSyt45O0spg/NYGyfjmGXIzGk0BGR0M1ZvI1DOstpFhQ6IhKqvIJSfrdwKxeO6MYYneU0eQodEQnVA29sorSiiu9eMizsUqQRKHREJDSbcgr443s7uObsfgzIaBt2OdIIFDoiEpof/309aS1TuOMCfS6nuVDoiEgo3t6cx8KNedw+Y5BuPd2MxCx0zOwxM8s1szUnWT/LzFaZ2UozW2Zm5wXLpwfLjj2Omtnlwbr+ZvaumW0xs+fMTD+pIgmossr50avr6dO5Ndedkxl2OdKIYnmmMweYWcv6BcAYdx8L3AA8CuDub7r72GD5DKAYmBfscx/wgLsPAg4CN8aodhGJoReW72TDvgK+PXMYLVOSwy5HGlHMQsfdFwEHallf6O4ePE0DvIbNPgW85u7FZmZEQuiFYN0TwOVRLFlEGkFRaQU/n7eJcX07ctmoHmGXI40s1Gs6ZnaFmW0AXiVytnOiq4Bng6+7AIfcvSJ4vgvoVcuxbw667Zbl5eVFs2wRaYDZi7LIKyjl+5eNIPK3pDQnoYaOu7/o7sOInLHcU32dmfUARgFzT/PYs919grtPyMjIaHixItJgOUeOMntRFpeN6qEbtDVTcTF6LeiKG2Bm6dUWfwZ40d3Lg+f5QEczSwme9wZ2N2KZItJA98/bRGWV8+2Z+iBocxVa6JjZoOA6DWY2DmhJJFiOuZrjXWsE13/eJHKdB+A64KXGqVZEGmprXiF/Wr6Tz5/dl75d2oRdjoQk5dSbnB4zexaYBqSb2S7gbqAFgLs/BFwJXGtm5UAJ8NljAwvMLBPoA7x1wmG/DfzRzP4PWAH8Plb1i0h0PTB/E61aJHPr9EFhlyIhilnouPvVp1h/H5Eh0DWty6aGQQLungVMikZ9ItJ41u45zCur9nLb9EGkt20ZdjkSori4piMiTdv98zbRvlUKX5oyIOxSJGQKHRGJqeXbD7JgQy63TB1Ih9Ytwi5HQqbQEZGYcXd+NncD6W1T+eK5mWGXI3FAoSMiMbN4Sz5Lsw5w6/RBtEmN2SVkSSAKHRGJCXfnZ/M20rNDKz53Vt+wy5E4odARkZiYvy6HD3ce4r8+NliTesq/KHREJOqqqpxfzNtE//Q0rhzXO+xyJI4odEQk6l5etYeNOQV87cIhpCTr14wcp58GEYm63/9zG0O6teXjunWBnEChIyJRtfdwCat2HebyM3uRlKRbF8hHKXREJKreWJcDwEUjuoVcicQjhY6IRNW8dTn0T09jYEbbsEuROKTQEZGoOXK0nKVZ+Vw4opvuCio1UuiISNQs2pRHeaVzobrW5CQUOiISNfPX5dA5LZVxfXUraqmZQkdEoqK8soo3N+QyY1hXkjVqTU5CoSMiUfHetgMcOVqhrjWplUJHRKJi/rocWqYkcf7g9LBLkTim0BGRBnN35q/L4fzB6bqFgdRKoSMiDbZu7xF2HypR15qckkJHRBrsjXW5mMGMYQodqZ1CR0QabP76fZzZpyMZ7VqGXYrEOYWOiDTInkMlrNl9hAtHdA+7FEkACh0RaZA31kcm+NT1HKkLhY6INMj8dTkMSE9jUFdN8CmnptARkdNWfYJPkbpQ6IjIaVu4URN8Sv0odETktM1fl0OXtFTO1ASfUkcKHRE5Ldn7i3h9zV4uHdVDE3xKnSl0ROS0/Pjv60lNTuL2GYPCLkUSiEJHROrtnS37mbcuh69MH0TX9q3CLkcSiGbmE4kjh0sio8H2F5ZyxZm94nLyzMoq54evrKNXx9bceF7/sMuRBBN/P9EizUh5ZRUrdhzin5vzeHvLfj7ceYgqj6z71YItfPfSYXxyTE/MTn7NZO2ewyzZms/FZ3SnT+c2Ma/5+WU72bCvgN98bhytWiTH/PWkaTF3D7uGmJswYYIvW7Ys7DIkwWTvLyKtZUpM5hOrrHJ+uWAzv387i6KySpIMRvfuyPmD0zlvUOR+NP/36npW7z7M+H6duPsTIxjdu+O/9i+rqOL1tft48p1slm0/CECLZOPqSX25LYZdXkeOljP9ZwsZkJHG87dMrjUMJfGZ2XJ3nxDNY+pMR6QGz72/g//561owuGpiH26ZOpBeHVtH5dh5BaX81x9X8M7WfC4b1YNPjOnJ5AFd6NCmxUe2e+nWc3lh+S5+OncDs36zmE+N683152Yyd20Oz7y7g/2FpfTr0ob/vmw4U4Zk8MQ72Tzz7g6eX7aT6yZn8uWpA+mUlhqVmo/5zZtbOFBcxpyPT1LgyGnRmY5INeWVVdzzyjqeXLKd8wen06tja/78wS7c4cpxvfnPaQPJTE877eO/t+0Atz3zAYdLyrnn8pF8ZkKfU+5TcLScX/9jC48t3kZ5pWMG04d25QuT+zF1cAZJ1YYrb88v4pdvbObFlbtJS03hhvP6M3VIOoMy2v1bqNXX9vwiLrx/EZ8c25Off3pMg44liSEWZzoKHZFAfmEpX3n6A97ddoCbpwzgWxcPJSU5id2HSpj91laefX8nFZVVfHJMT6YMyaC4rJKi0gqKyiopDv5t3yqFcf06MaFfJ7q0Pd4t5+7MXpTFT+dupE+n1vz28+MZ0bN9verbtr+IhRtzuWBYN/p2qf3azaacAu6ft4nX1+7717L0tqkMzGjLwK5tGdy1LZeN7kHXdnXvhvvyU8tZtDmPN++cRjeNWGsWFDqnSaHTvLk7ew8fJXt/ER3bpNI/PY3WqR+9AL5m92FueWo5+wtLue/K0Vx+Zq9/O05uwVEefXsbf1i6neKyyo+sa5OaTJvUFI6UlFNWWQVA//Q0JvTrxITMTryxPpf563K4ZGR37vvUaNq3athZR13tOljMppwCtuQWsjW3iC15hWzJLeRwSTltUpO56fwB3DxlAG1b1t7TvmRrPlc/spQ7LxrCbTMGN0rtEj6FzmlqrNCpqKyiqLSywd0Ycmruzrb9RRSXVVJWWUV5RRUVVU5ZZRWl5ZVk5xezOaeQLbkFbM0rorC04iP79+rYmgEZaQzMaEv71i2YvWgrnduk8vAXJjCqd4daX/vI0XLyC8tIa5lMWmoKrVsk/6uL62h5JWt2H2bZ9oMsyz7I8u0HOFhcTkqS8b1Lh/PFczNDvxbi7mTtL+L++Zt4ddVe0tumcscFg7l6Ul9aJB//6F5llbNy50H+sSGXPy/fTXKSseAbUzVirRlJqNAxs8eAjwO57j6yhvWzgHuAKqAC+Kq7/zNY1xd4FOgDOHCpu2eb2RxgKnA4OMz17r7yVLU0Rugsyz7A915czY4Dxdz/mbFcOqpHTF+vOXJ31uw+wiur9/Dqqr3sOlhS6/bd2rdkUNe2DO7ajoFd2zIgPY2DxWVk5RWxNa+QrLwisvIKKSqrZFJmZ357zTjS20Z3pNqxX/CpyUmNMpy5vlbuPMRP/r6ed7cdILNLG75+0VDcnTc35PLWpjwOFpeTnGRM6NeJb148lAmZncMuWRpRooXOFKAQePIkodMWKHJ3N7PRwPPuPixYtxD4kbvPD7arcvfiIHRecfcX6lNLLEPncEk5972+gWfe3UGvjq1Jb5vKh7sO87WPDeGOCwaF/ldtU7B2z2FeWbWXV1ftZceBYlKSjPMGp3PRiO5ktGtJi2QjNTmJlOSkyNcpSfTu1IYOrU99xunuHCgqo3NaarP9Xrk7Czfmce9rG9iYUwBA57RUpg3JYMbwrpw/OKNObSlNT0INmXb3RWaWWcv6wmpP04ic0WBmI4AUd59fw3Zxw915ZdVefvDyOg4UlXLTef352oVDSEk2vvuX1TzwxiY25Rbw80+N+bfrB6dyqLiM3y3cyr4jRzlSUs6RoxXBv+WUlFUyqGtbxvbpxNi+HTmzT0d6d2rdJH9hVh9JlpxknDsondumD+KiM7rRsU10hgKb2Ucu+DdHZsb0YV2ZMiSDhRtz6ZSWypjeHTWJp8REqJ/TMbMrgJ8AXYHLgsVDgENm9hegP/AG8B13P3bl9kdmdhewIFhe2shls/tQCd9/cTULN+YxqlcH5nxxIiN7Hb8O8ItPj2Fot3bc+/oGduQX88i1E+jeoW6jfSqrnNufjXyGo3en1rRv1YL2rVPo2q4t7VqlkJqSxMZ9BTz97nYeW7wNiIxKGtunE9dO7seUIRkxec+N7UBRGbc+/QFLsvK56bz+fGX6IDpH+TMn8lHJScYFw3VfHImtmA4kCM50Xqmpe+2E7aYAd7n7x8zsU8DvgTOBHcBzwN/d/fdm1gPYB6QCs4Gt7v7DkxzzZuBmgL59+47fvn17VN7T8u0HuPnJ5ZSUV/KNi4Zy3eR+pCTXPG/qgvU53PHsCtJapjD72gmM7dOxxu2q++nrG/jtwq3c+x+juGpS35NuV15ZxcZ9BazYeYiVOw7xztb97D18lAuGdeX7lw1nQEZ0bx187OekLmdUJWWVPLkkm437Cjh7YBemDsmo1xDbjfsKuOnJ98k5Usq9/zGK/xjX+3TLFpEGSKhrOlD30Am2zQImAYOA+9x9arD8C8DZ7n7rCdtPA+5094+f6tjRuqbz1xW7+dYLq+jZsRW/v34iA+vwi33jvgJufOJ98gpKeeCztQ8wmLt2H7c8tZyrJ/XhJ/8xul61lVZU8vjibH79jy2UVlRy3eRMbr9gcJ374g+XlLN292FW7z7M7kMl5BeVcaCwjANFZeQXlXGwuIyeHVvxuUn9+MyE3jV2SZVXVvHc+zt5cMFmcgtK6dimBYeKywEY1r0dU4ZkMGVwBhP7d6JlSs1djvPW7uNrz60krWUKD39hvG4OJhKiJhU6Zjak7tQ+AAAKlUlEQVSIyJmKm9k44GWgN5HbLXwAfMzd88zscWCZu//GzHq4+16L/Ln9AHDU3b9zqjoaGjruzgNvbObBBZs5q39nHrpmfL2mF8kvLOVLTy5jxc5DfO+S4dx0fv9/O2PYmlfIrF8vZmBGGs9/efJJfymfSm7BUX4xdxPPL99J5zapfP2iIUzM7ExZRRVllVWUVVRRXllFaXkVW/MKWR0Ezfb84n8do0PrFnRJS6VztUentFRW7DjI0qwDpCYncemo7nxhcj/G9e2EO7y8ag/3z9/E9vxiJmZ24psXD2NiZic25hTw1sY8Fm3O4/1tBymrrCIlyejTuQ2ZXdqQmZ5G/+CxYsch7p+/iTG9O/DwF+reJSkisZFQoWNmzwLTgHQgB7gbaAHg7g+Z2beBa4FyoAT4ZrUh0xcCvwAMWA7c7O5lZvYPICNYvhL4cl0GGjQkdI6WV3Lnnz7klVV7+fT43vzoilGkptT/NkRHyyv5xvMf8urqvVxzdl/+9xNn/Ktbrqi0gst/s5j8ojJeuf08ekZhjq81uw/zw5fX8V72gVq369WxNaN7d2Bkrw6MCh61BermnAL+sHQ7f/lgNwWlFQzv0R53Z8O+Aob3aM+3Lh7KtKEZNXbDFZdVsDQrn2XZB8nOL2Lb/mKy9xdRUn78g5aXj+3JvVeO1mdBROJAQoVOPDnd0MkriJyhfLjrEN+eOYxbpgxo0CixqirnvrkbePitLKYPzeDXnxtHm9RkbntmBa+t2csfbjyLc4IZhqPB3Xlnaz4Hi8tITU4iNSXp+L/BsOLTvThfVFrBSyv38PS72ymvrOLW6YP4xOieH5kHrK415haUsm1/ERWVzrmDujTJkXgiiUihc5pOJ3TcnU89tIR1e47wwGfHMnNk96jV84el27nrpTUM79GeqUMy+O3CrXz3kmHcMnVg1F5DRKShEupzOonOzPjBJ88A+Mhw6Gi45ux+9OrYmluf+YC1e45w6aju3DxlQFRfQ0QkHil0ahHtsKlu+rCu/OnLk3nxg9189cIh6lISkWZBoROiM3p24IyesQs2EZF4U/9hWCIiIqdJoSMiIo1GoSMiIo1GoSMiIo1GoSMiIo1GoSMiIo1GoSMiIo1GoSMiIo2mWcy9ZmaHgc0nWd0BOFzH5XVZVv15OrC/XsXW3cnqjsZ+tW1Tn/aqaXlza6/a1tf35+nE52qv+rUXxK7Nmmp7DXX3dqeor37cvck/gNn1XVfT8rosq/6cyH2AGv09NXS/aLXXqdqnObRXfdtM7RW79oplm6m96v5oLt1rL5/GupqW12VZba8VTaf7OnXZL1rtVdPy5tZeta0/nZ8ntVfty9RetS8Pvb2aRfdaWMxsmUd5WvCmTO1VP2qv+lOb1U8s2qu5nOmEZXbYBSQYtVf9qL3qT21WP1FvL53piIhIo9GZjoiINBqFTh2Z2WNmlmtma05j3/FmttrMtpjZg1btjm1mdruZbTCztWb20+hWHZ5YtJeZ/a+Z7TazlcHj0uhXHo5Y/XwF679hZm5m6dGrOFwx+vm6x8xWBT9b88ysZ/QrD0eM2utnwe+uVWb2opl1rMvxFDp1NweYeZr7/g74EjA4eMwEMLPpwCxgjLufAfy84WXGjTlEub0CD7j72ODx94aVGFfmEIP2MrM+wEXAjgbWF2/mEP32+pm7j3b3scArwF0NLTKOzCH67TUfGOnuo4FNwHfrcjCFTh25+yLgQPVlZjbQzF43s+Vm9raZDTtxPzPrAbR396UeuYD2JHB5sPo/gXvdvTR4jdzYvovGE6P2arJi2F4PAN8CmtTF21i0l7sfqbZpGk2ozWLUXvPcvSLYdCnQuy61KHQaZjZwu7uPB+4EflvDNr2AXdWe7wqWAQwBzjezd83sLTObGNNqw9fQ9gK4LTidf8zMOsWu1LjQoPYys1nAbnf/MNaFxokG/3yZ2Y/MbCfweZrWmU5NovH/8ZgbgNfq8qIp9SxSAmbWFjgH+FO1LvSW9TxMCtAZOBuYCDxvZgO8CQ4pjFJ7/Q64h8hfoPcAvyDyw97kNLS9zKwN8D0iXWtNXpR+vnD37wPfN7PvArcBd0etyDgSrfYKjvV9oAJ4ui7bK3ROXxJwKOj//RczSwaWB0//RuQXZfXTzt7A7uDrXcBfgpB5z8yqiMwNlRfLwkPS4PZy95xq+z1CpN+9qWpoew0E+gMfBr9UegMfmNkkd98X49rDEI3/j9U9DfydJho6RKm9zOx64OPABXX9Y1nda6cp6P/dZmafBrCIMe5eWe1C913uvhc4YmZnB6M+rgVeCg7zV2B6sP8QIJXYTeAYqmi0V9C/fMwVQL1H4iSKhraXu692967ununumUT+wBnXRAMnWj9fg6sdchawobHfR2OJUnvNJHK98JPuXlyfF9ejDg/gWWAvUE7kP/CNRP6SfB34EFgH3HWSfScQ+QW5Ffg1xz+Umwr8IVj3ATAj7PcZ5+31FLAaWEXkr7AeYb/PeG6vE7bJBtLDfp/x3F7An4Plq4jMQdYr7PcZ5+21BdgJrAweD9WlFs1IICIijUbdayIi0mgUOiIi0mgUOiIi0mgUOiIi0mgUOiIi0mgUOtIsmVlhI7/eo2Y2IkrHqrTITMhrzOzlU83ua2Ydzewr0XhtkYbSkGlplsys0N3bRvF4KX588sOYql67mT0BbHL3H9WyfSbwiruPbIz6RGqjMx2RgJllmNmfzez94HFusHySmS0xsxVm9o6ZDQ2WX29mfzOzfwALzGyamS00sxcscp+Rp4NPcRMsnxB8XRhMLPmhmS01s27B8oHB89Vm9n91PBtbwvEJPtua2QIz+yA4xqxgm3uBgcHZ0c+Cbb8ZvMdVZvaDKDajSK0UOiLH/ZLI/XomAlcCjwbLNwDnu/uZRGYe/nG1fcYBn3L3qcHzM4GvAiOAAcC5NbxOGrDU3ccAi4jcq+TY6//S3Ufx0Zl9axTMk3UBkdkZAI4CV7j7OCLTK/0iCL3vAFs9MrXJN83sIiL3RZkEjAXGm9mUU72eSDRowk+R4z4GjKg26277YDbeDsATwdxcDrSots98d69+n5L33H0XgJmtBDKBf57wOmUcn6x0OXBh8PVkjt8L5xlOflO/1sGxewHridxMC8CAHwcBUhWs71bD/hcFjxXB87ZEQmjRSV5PJGoUOiLHJQFnu/vR6gvN7NfAm+5+RXB9ZGG11UUnHKO02teV1Px/rNyPX0w92Ta1KXH3sRa5fcFc4FbgQSL3gMkAxrt7uZllA61q2N+An7j7w/V8XZEGU/eayHHzgNuPPTGzY9O+d+D4dO7Xx/D1lxLp1gO46lQbe2Rm3zuAb5hZCpE6c4PAmQ70CzYtANpV23UucENwFoeZ9TKzrlF6DyK1UuhIc9XGzHZVe3ydyC/wCcHF9XXAl4Ntfwr8xMxWENvega8CXzezVcAg4PCpdnD3FURmRb6ayD1gJpjZaiJT0G8ItskHFgdDrH/m7vOIdN8tCbZ9gY+GkkjMaMi0SJwIustK3N3N7Crganefdar9RBKJrumIxI/xwK+DEWeHaKK34pbmTWc6IiLSaHRNR0REGo1CR0REGo1CR0REGo1CR0REGo1CR0REGo1CR0REGs3/B3M1qMVgPasxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_lm.lr_find()\n",
    "learn_lm.recorder.plot(skip_end = 15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_lm.load('lm-fine-tuned-transformer-10-1-1_a3');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1.3529806, tensor(0.3391)]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn_lm.validate(metrics=[accuracy])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "atggcag g g g g g g g g g g g g g g g g g g g g g g g g g g g g g g g g g g g g g g g g\n",
      "atggcag g g xxeos xxeos xxeos xxeos xxeos xxeos xxeos xxeos xxeos xxeos xxeos xxeos xxeos xxeos xxfake xxfake xxfake xxfake xxfake xxfake xxfake xxfake xxfake xxfake xxfake xxfake xxfake xxfake xxfake xxfake xxfake xxfake xxfake xxfake xxfake xxfake y y\n"
     ]
    }
   ],
   "source": [
    "TEXT = \"atggcag\"\n",
    "N_WORDS = 40\n",
    "N_SENTENCES = 2\n",
    "print(\"\\n\".join(learn_lm.predict(TEXT, N_WORDS, temperature=0.75) for _ in range(N_SENTENCES)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "atggcag a g c t t g t g a c g g g c g a c t c g a c t t g c c g c c g t t g a t g g g c\n",
      "atggcag t a a g a t c t t g t c c c t a t g c c c t t a g t g a g a t c t g a t t t a g\n"
     ]
    }
   ],
   "source": [
    "TEXT = \"atggcag\"\n",
    "N_WORDS = 40\n",
    "N_SENTENCES = 2\n",
    "print(\"\\n\".join(learn_lm.predict(TEXT, N_WORDS, temperature=0.75) for _ in range(N_SENTENCES)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='1069' class='' max='6691', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      15.98% [1069/6691 21:22<1:52:22 1.3101]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_lm.fit_one_cycle(1, 1e-4, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3 (tf-env)",
   "language": "python",
   "name": "tf-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}