MLDA Text Generator Workshop Answer.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Text Generator using RNN Workshop Answer.ipynb","provenance":[],"collapsed_sections":[],"mount_file_id":"1lsUyx2nt-P795GOMe3vvV08kzpenFXl2","authorship_tag":"ABX9TyPLsF04Ecbh0E7WQtW6CFw2"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"a4c8978ff96042438375acf91d5f95ed":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_3720a96a33de44489ea7627657870761","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_0c38e51224414bd89ef554e5e48ec4cf","IPY_MODEL_4ee2890fbf0849c882cb9760434cc6b2"]}},"3720a96a33de44489ea7627657870761":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"0c38e51224414bd89ef554e5e48ec4cf":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_905be1e6305c47f99f8ca530b3cb1fe3","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":718,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":718,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_121e08961fe14b4199d2ffbca1490f56"}},"4ee2890fbf0849c882cb9760434cc6b2":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_00b6630a13924fa5837ab0ceaddb2911","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 718/718 [00:04&lt;00:00, 169B/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_2842cf70bf83431a85971d0ce6428238"}},"905be1e6305c47f99f8ca530b3cb1fe3":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"121e08961fe14b4199d2ffbca1490f56":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"00b6630a13924fa5837ab0ceaddb2911":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"2842cf70bf83431a85971d0ce6428238":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"ac97fde4df884300a92f9e19fdda3984":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_e5ac8eec85eb46958fc663dda865794c","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_dfe298c12b694dbfa189c1cbc7eb64a7","IPY_MODEL_15a8511a37774cef94f8f58b5ce2dc3d"]}},"e5ac8eec85eb46958fc663dda865794c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"dfe298c12b694dbfa189c1cbc7eb64a7":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_1cad36a55df547d4b82f7538e65e8bf6","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":1042301,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1042301,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_a252ade7e4874212a3b60d25dab34bd1"}},"15a8511a37774cef94f8f58b5ce2dc3d":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_2d6a4540b6924af88680d82256abed82","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 1.04M/1.04M [00:02&lt;00:00, 397kB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_ee49f59aa03f493f9f031766df103b66"}},"1cad36a55df547d4b82f7538e65e8bf6":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"a252ade7e4874212a3b60d25dab34bd1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"2d6a4540b6924af88680d82256abed82":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"ee49f59aa03f493f9f031766df103b66":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"6280bdf917d44dc6861df998ea76da28":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_9d9748ed90c0413e801339bedadfb509","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_f2bcf585360d4c24bf501655195871e8","IPY_MODEL_92c1690ab02d414f9e2170a59cdcd78d"]}},"9d9748ed90c0413e801339bedadfb509":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"f2bcf585360d4c24bf501655195871e8":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_be764c6eb46549f9ac146dfbd765ee05","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":456318,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":456318,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_80a19c61e3a2489ea9e5095280e8e60a"}},"92c1690ab02d414f9e2170a59cdcd78d":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_2440de067eec48fdb3dd2586f2d2fb0c","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 456k/456k [00:00&lt;00:00, 611kB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_38e7122a39204173b60c145c7c35cb3b"}},"be764c6eb46549f9ac146dfbd765ee05":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"80a19c61e3a2489ea9e5095280e8e60a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"2440de067eec48fdb3dd2586f2d2fb0c":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"38e7122a39204173b60c145c7c35cb3b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3af5836f5389450d90cf105ca1af1bdb":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_81cb2b8adfbe407fa7cce95bf7f454ea","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_2a1581a8a40e413ea60fb189f34068d4","IPY_MODEL_93a2b412efab4678a8e746def3863e43"]}},"81cb2b8adfbe407fa7cce95bf7f454ea":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"2a1581a8a40e413ea60fb189f34068d4":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_db52bf47ea8148cf974c7ea5363ea115","_dom_classes":[],"description":"Downloading: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":1419628976,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1419628976,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_bc6f762c802146ca8d97afe611fad7b0"}},"93a2b412efab4678a8e746def3863e43":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_c18c18ed4d9e4713b1bb9f3c8e43ed7d","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 1.42G/1.42G [02:07&lt;00:00, 11.1MB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_062484220b3744369ee53ccd4c996ce8"}},"db52bf47ea8148cf974c7ea5363ea115":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"bc6f762c802146ca8d97afe611fad7b0":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"c18c18ed4d9e4713b1bb9f3c8e43ed7d":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"062484220b3744369ee53ccd4c996ce8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"cells":[{"cell_type":"markdown","metadata":{"id":"OjGwcuHygZXo"},"source":["### MLDA@EEE Deep Learning Week Special:\n","# **Text Generator using RNN**"]},{"cell_type":"markdown","metadata":{"id":"NSo4cUWJhxtZ"},"source":["This notebook is part of MLDA@EEE's series of workshops during the Deep Learning week.\n","\n","Designed to run in Google Colab.\n","\n","\n","\n","In this workshop, we assumed that you have attended the workshops in pre-deep learning week and have basic knowledge of **Python** programming, **deep learning** as well as **neural network** Basics.\n","If not, don't worry, as you will be instructed step by step in this pratical session to apply what you learnt during the tutorial session. If you encounter any technical issues or need assistance from us, you can ask us in the ZOOM chat and a helper will come to you as soon as possible.\n","\n","The structure of this pratical session is listed below:\n","1. Text Processing Basics\n","2. RNN Building Basics\n","3. Lyrics Generator\n","4. Shakespeare Generator\n","\n","\n","### **Connect to GPU instance (Recommend)**\n","To connect to GPU instance on Google Colab, follow the instruction below\n","\n","Edit > Notebook settings > Hardware accelerator > GPU"]},{"cell_type":"markdown","metadata":{"id":"-rGPDW--lKI-"},"source":["## **1. Text Processing Basics**"]},{"cell_type":"markdown","metadata":{"id":"GsXF1GX9eSO0"},"source":["Given a character sequence and a defined document unit, tokenization is the task of chopping it up into pieces, called tokens, perhaps at the same time throwing away certain characters, such as punctuation."]},{"cell_type":"markdown","metadata":{"id":"SPddXodNjFhA"},"source":["To give you a more intuitive perspective, we will start with a short file 'eee-overview.txt' and do some practices on text processing basics first."]},{"cell_type":"code","metadata":{"id":"SObQe2xITjBg","executionInfo":{"status":"ok","timestamp":1601646607118,"user_tz":-480,"elapsed":1288,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"a804a188-3367-4af4-8e85-34fe797a57b0","colab":{"base_uri":"https://localhost:8080/","height":237}},"source":["# download 'eee-overview.txt' file\n","!wget https://ycrao573.github.io/rnn-workshop/eee-overview.txt"],"execution_count":1,"outputs":[{"output_type":"stream","text":["--2020-10-02 13:50:05--  https://ycrao573.github.io/rnn-workshop/eee-overview.txt\n","Resolving ycrao573.github.io (ycrao573.github.io)... 185.199.108.153, 185.199.109.153, 185.199.110.153, ...\n","Connecting to ycrao573.github.io (ycrao573.github.io)|185.199.108.153|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 3336 (3.3K) [text/plain]\n","Saving to: ‘eee-overview.txt’\n","\n","\reee-overview.txt      0%[                    ]       0  --.-KB/s               \reee-overview.txt    100%[===================>]   3.26K  --.-KB/s    in 0s      \n","\n","2020-10-02 13:50:05 (37.5 MB/s) - ‘eee-overview.txt’ saved [3336/3336]\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"yxcKV1G-kJGo","executionInfo":{"status":"ok","timestamp":1601646607120,"user_tz":-480,"elapsed":1210,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"f3a1f373-a0cc-4564-9ba2-505c4db5f40a","colab":{"base_uri":"https://localhost:8080/","height":92}},"source":["overview = open('eee-overview.txt', 'r').read()\n","# length of text is the number of characters in it\n","print('Length of text: {} characters'.format(len(overview)))\n","print('First 100 characters: \\n', overview[:100])"],"execution_count":2,"outputs":[{"output_type":"stream","text":["Length of text: 3322 characters\n","First 100 characters: \n"," The School of Electrical and Electronic Engineering (NTU EEE) began as one of the three founding sch\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"sESzG1tjkcvP","executionInfo":{"status":"ok","timestamp":1601646607121,"user_tz":-480,"elapsed":1193,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"92fd6c34-1a64-42cd-afcc-7023149278be","colab":{"base_uri":"https://localhost:8080/","height":35}},"source":["# The unique characters in the file\n","vocab = sorted(set(overview))\n","print ('{} unique characters'.format(len(vocab)))"],"execution_count":3,"outputs":[{"output_type":"stream","text":["68 unique characters\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"X07p-zSGjM6M","executionInfo":{"status":"ok","timestamp":1601646607122,"user_tz":-480,"elapsed":1191,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}}},"source":["overview = overview.replace('\\n', ' ').replace('\\t',' ').lower()"],"execution_count":4,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"uIzWzk1QygzA"},"source":["### ***TASK 1: Text Pre-Processing - Special Characters Cleaning***\n","\n","Please only change the code in **\\# INSERT YOUR CODE HERE** or **None**"]},{"cell_type":"code","metadata":{"id":"30d2HUIHOzRz","executionInfo":{"status":"ok","timestamp":1601646679958,"user_tz":-480,"elapsed":1053,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"ccc25a2f-39a6-41a8-d9b3-faa48ff80eb6","colab":{"base_uri":"https://localhost:8080/","height":55}},"source":["corpus = overview\n","stopChars = [',','(',')','.','-','[',']','\"','/','&','!','\\'','?', '  '] # list of stop charaters (e.g. ',', '(', '-')\n","# use for loop to iterate over stopChars and replace them with space\n","# hint: use string's replace(x, y) method and store in 'corpus'\n","# INSERT YOUR CODE HERE\n","for char in stopChars:\n","  corpus = corpus.replace(char, '')\n","\n","print(corpus[:100])"],"execution_count":6,"outputs":[{"output_type":"stream","text":["the school of electrical and electronic engineering ntu eee began as one of the three founding schoo\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"6MC3HUNHO09K","executionInfo":{"status":"ok","timestamp":1601646685927,"user_tz":-480,"elapsed":1360,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"da298943-ae62-41d1-fa45-fe1bcc1a1a34","colab":{"base_uri":"https://localhost:8080/","height":35}},"source":["corpus_words = [i for i in corpus.split() if i]\n","corpus_words[:5]"],"execution_count":7,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['the', 'school', 'of', 'electrical', 'and']"]},"metadata":{"tags":[]},"execution_count":7}]},{"cell_type":"code","metadata":{"id":"gpevVQR3RAFl","executionInfo":{"status":"ok","timestamp":1601646688323,"user_tz":-480,"elapsed":1110,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"f82eef4e-5f1c-4440-aa5f-4b44fe697a10","colab":{"base_uri":"https://localhost:8080/","height":54}},"source":["map(str.strip, corpus_words)\n","vocab = sorted(set(corpus_words))\n","print('Corpus length (in words):', len(corpus_words))\n","print('Unique words in corpus: {}'.format(len(vocab)))"],"execution_count":8,"outputs":[{"output_type":"stream","text":["Corpus length (in words): 473\n","Unique words in corpus: 260\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"krHsSjAf1MUG"},"source":["### ***TASK 2: Use Index Number to Represent the Word***\n","\n","In this task, you will assign an index to each word in string 's', e.g. {1: 'eee'}\n","\n","You are encouraged to use dict and list comprehension to perform assignment and substitution.\n"]},{"cell_type":"code","metadata":{"id":"lGJmMzjkmAYj","executionInfo":{"status":"ok","timestamp":1601646705140,"user_tz":-480,"elapsed":1237,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"3f3b5c8a-0974-49a5-e447-c3a1067dbd5a","colab":{"base_uri":"https://localhost:8080/","height":74}},"source":["s = 'the school of electrical and electronic engineering ntu eee began as one of the three'\n","# iterate 'vocab', create a dictionary called word2idx whose index start with zero\n","word2idx = {u: i for i, u in enumerate(vocab)}\n","print(word2idx)\n","# iterate word2idx and replace word with number (index)\n","# print them in the below\n","# INSERT YOUR CODE HERE\n","[word2idx[i] for i in s.split()]"],"execution_count":9,"outputs":[{"output_type":"stream","text":["{'$90': 0, '1000': 1, '11th': 2, '13': 3, '150': 4, '194': 5, '1985': 6, '1999': 7, '20': 8, '20000': 9, '2011': 10, '2017': 11, '2020': 12, '2020electricalelectronic': 13, '21': 14, '300': 15, '3000': 16, '4': 17, '50': 18, '6th': 19, '7th': 20, '9': 21, 'a': 22, 'about': 23, 'academic': 24, 'active': 25, 'activities': 26, 'administrative': 27, 'advanced': 28, 'ahead': 29, 'all': 30, 'also': 31, 'alumni': 32, 'ambitions': 33, 'an': 34, 'analytics': 35, 'and': 36, 'annualised': 37, 'aoba': 38, 'apart': 39, 'are': 40, 'areas': 41, 'around': 42, 'artificial': 43, 'as': 44, 'asian': 45, 'autonomous': 46, 'average': 47, 'backed': 48, 'backgrounds': 49, 'batch': 50, 'become': 51, 'began': 52, 'being': 53, 'besides': 54, 'big': 55, 'biomedical': 56, 'both': 57, 'broad': 58, 'by': 59, 'centre': 60, 'centres': 61, 'challengesntu': 62, 'close': 63, 'closely': 64, 'collaborations': 65, 'communications': 66, 'companies': 67, 'competent': 68, 'consistently': 69, 'consists': 70, 'continues': 71, 'corporate': 72, 'counting': 73, 'countries': 74, 'courses': 75, 'data': 76, 'date': 77, 'delta': 78, 'demands': 79, 'development': 80, 'diverse': 81, 'drive': 82, 'each': 83, 'eee': 84, 'eee’s': 85, 'electrical': 86, 'electronic': 87, 'electronics': 88, 'energy': 89, 'engaging': 90, 'engineering': 91, 'engineers': 92, 'ensure': 93, 'equipment': 94, 'established': 95, 'expertise': 96, 'extensive': 97, 'facilities': 98, 'faculty': 99, 'first': 100, 'five': 101, 'for': 102, 'forward': 103, 'founding': 104, 'from': 105, 'frontiers': 106, 'funding': 107, 'global': 108, 'graduate': 109, 'graduated': 110, 'graduates': 111, 'graduatestoday': 112, 'great': 113, 'group': 114, 'has': 115, 'have': 116, 'healthcare': 117, 'hosts': 118, 'hundred': 119, 'impressively': 120, 'in': 121, 'include': 122, 'including': 123, 'industry': 124, 'innovationbeing': 125, 'innovations': 126, 'institute': 127, 'institutes': 128, 'intake': 129, 'intelligencentu': 130, 'intelligent': 131, 'international': 132, 'internet': 133, 'is': 134, 'january': 135, 'joint': 136, 'known': 137, 'laboratories': 138, 'largest': 139, 'launch': 140, 'launched': 141, 'local': 142, 'locallymade': 143, 'marking': 144, 'members': 145, 'membersto': 146, 'million': 147, 'more': 148, 'multinational': 149, 'names': 150, 'nanyang': 151, 'now': 152, 'ntu': 153, 'nurture': 154, 'nxp': 155, 'of': 156, 'offering': 157, 'on': 158, 'one': 159, 'only': 160, 'our': 161, 'output': 162, 'over': 163, 'overseas': 164, 'partners': 165, 'photonics': 166, 'power': 167, 'produces': 168, 'professional': 169, 'programme': 170, 'programmes': 171, 'proudly': 172, 'pushing': 173, 'qs': 174, 'range': 175, 'ranked': 176, 'ranking': 177, 'rankings': 178, 'ready': 179, 'receives': 180, 'renowned': 181, 'research': 182, 'researchdevelopment': 183, 'researchers': 184, 'rollsroyce': 185, 'satellite': 186, 'satellites': 187, 'schaeffler': 188, 'school': 189, 'schools': 190, 'school’s': 191, 'set': 192, 'shanghairankings': 193, 'since': 194, 'singapore': 195, 'singapores': 196, 'singapore’s': 197, 'smrt': 198, 'sophisticated': 199, 'space': 200, 'specialisation': 201, 'st': 202, 'staff': 203, 'startups': 204, 'stateoftheart': 205, 'stationto': 206, 'staying': 207, 'strength': 208, 'strong': 209, 'students': 210, 'subject': 211, 'subjects': 212, 'successfully': 213, 'support': 214, 'supported': 215, 'supporting': 216, 'systems': 217, 'take': 218, 'teaching': 219, 'technical': 220, 'technological': 221, 'thales': 222, 'than': 223, 'that': 224, 'the': 225, 'then': 226, 'there': 227, 'these': 228, 'they': 229, 'things': 230, 'thousand': 231, 'three': 232, 'to': 233, 'tomorrow’s': 234, 'top': 235, 'topranked': 236, 'total': 237, 'transportation': 238, 'undergraduate': 239, 'universities': 240, 'university': 241, 'up': 242, 'v2x': 243, 'veloxiii': 244, 'very': 245, 'we': 246, 'wellequipped': 247, 'which': 248, 'who': 249, 'whom': 250, 'with': 251, 'witnessed': 252, 'works': 253, 'world': 254, 'worldwide': 255, 'world’s': 256, 'xsat': 257, 'year': 258, '–': 259}\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["[225, 189, 156, 86, 36, 87, 91, 153, 84, 52, 44, 159, 156, 225, 232]"]},"metadata":{"tags":[]},"execution_count":9}]},{"cell_type":"markdown","metadata":{"id":"AGG6ljRkmBHW"},"source":["Now, we will introduce tensorflow library so that we can process our text with higher quality and efficiency.\n","\n","Let's start with tokenization:"]},{"cell_type":"code","metadata":{"id":"M6CRNUq2XO9H","executionInfo":{"status":"ok","timestamp":1601646716559,"user_tz":-480,"elapsed":2990,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}}},"source":["import tensorflow as tf\n","from tensorflow import keras\n","from tensorflow.keras.preprocessing.text import Tokenizer"],"execution_count":10,"outputs":[]},{"cell_type":"code","metadata":{"id":"GpEXs0_5eFaU","executionInfo":{"status":"ok","timestamp":1601646716561,"user_tz":-480,"elapsed":1478,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"c09ac17f-0fe7-45a7-a5ef-1ae970887f85","colab":{"base_uri":"https://localhost:8080/","height":74}},"source":["sentences = [\n","    'I love coffee',\n","    'I do not like tea.',\n","    'We all love MLDA!'\n","]\n","tokenizer = Tokenizer(num_words = 32)\n","tokenizer.fit_on_texts(sentences)\n","word_index = tokenizer.word_index\n","print('word_index: ', word_index)\n","test_sen = [\n","    'I like coffee.',\n","    'You really love tea?',\n","    'We love MLDA ah!'\n","]\n","test_seq = tokenizer.texts_to_sequences(test_sen)\n","print(test_seq)"],"execution_count":11,"outputs":[{"output_type":"stream","text":["word_index:  {'i': 1, 'love': 2, 'coffee': 3, 'do': 4, 'not': 5, 'like': 6, 'tea': 7, 'we': 8, 'all': 9, 'mlda': 10}\n","[[1, 6, 3], [2, 7], [8, 2, 10]]\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"lozABdfetTTi"},"source":["We may also consider adding *oov_token*. Keras lets us define an **Out Of Vocab token** - this will replace **any unknown word**s with **a token of our choosing**. This is better than just throwing away unknown words since it tells our model there was information here."]},{"cell_type":"code","metadata":{"id":"-ATZm68WaV7e","executionInfo":{"status":"ok","timestamp":1601646717259,"user_tz":-480,"elapsed":750,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"49fe78e6-584e-4284-c519-ae8a5f246796","colab":{"base_uri":"https://localhost:8080/","height":74}},"source":["tokenizer = Tokenizer(num_words = 32, oov_token=\"<OOV>\")\n","tokenizer.fit_on_texts(sentences)\n","word_index = tokenizer.word_index\n","print('word_index: ', word_index)\n","sequences = tokenizer.texts_to_sequences(sentences)\n","test_seq = tokenizer.texts_to_sequences(test_sen)\n","print(test_seq)"],"execution_count":12,"outputs":[{"output_type":"stream","text":["word_index:  {'<OOV>': 1, 'i': 2, 'love': 3, 'coffee': 4, 'do': 5, 'not': 6, 'like': 7, 'tea': 8, 'we': 9, 'all': 10, 'mlda': 11}\n","[[2, 7, 4], [1, 1, 3, 8], [9, 3, 11, 1]]\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"mQ9JJG97tl0f"},"source":["All the neural networks require to have inputs that **have the same shape and size**. However, when we pre-process and use the texts as **inputs for our model** e.g. LSTM, **not all the sentences have the same length**. In other words, naturally, some of the sentences are longer or shorter. We need to have the inputs with the same size, this is where the **padding is necessary**."]},{"cell_type":"code","metadata":{"id":"Z-vxpLhya3gc","executionInfo":{"status":"ok","timestamp":1601646718604,"user_tz":-480,"elapsed":815,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"ab73dc41-56ce-488a-d75f-1046c9743063","colab":{"base_uri":"https://localhost:8080/","height":128}},"source":["from tensorflow.keras.preprocessing.sequence import pad_sequences\n","\n","tokenizer = Tokenizer(num_words = 32, oov_token=\"<OOV>\")\n","tokenizer.fit_on_texts(sentences)\n","word_index = tokenizer.word_index\n","sequences = tokenizer.texts_to_sequences(sentences)\n","padded = pad_sequences(sequences, maxlen=8)\n","print(word_index)\n","print(sequences)\n","print(padded)"],"execution_count":13,"outputs":[{"output_type":"stream","text":["{'<OOV>': 1, 'i': 2, 'love': 3, 'coffee': 4, 'do': 5, 'not': 6, 'like': 7, 'tea': 8, 'we': 9, 'all': 10, 'mlda': 11}\n","[[2, 3, 4], [2, 5, 6, 7, 8], [9, 10, 3, 11]]\n","[[ 0  0  0  0  0  2  3  4]\n"," [ 0  0  0  2  5  6  7  8]\n"," [ 0  0  0  0  9 10  3 11]]\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"8qM52HbIotlw"},"source":["## **2. RNN Building Basics**"]},{"cell_type":"markdown","metadata":{"id":"iTl0P3QTrXVF"},"source":["**Recurrent neural networks (RNN)** are a class of neural networks that is powerful for modeling sequence data such as time series or natural language. Schematically, a RNN layer uses a for loop to iterate over the timesteps of a sequence, while maintaining an internal state that encodes information about the timesteps it has seen so far.\n","\n","The **Keras RNN API** is designed with a focus on:\n","\n","\n","*   Ease of use: the built-in keras.layers.RNN, keras.layers.LSTM, keras.layers.GRU layers enable you to quickly build recurrent models without having to make difficult configuration choices.\n","\n","*   **Ease of customization**: You can also define your own RNN cell layer (the inner part of the for loop) with custom behavior, and use it with the generic keras.layers.RNN layer (the for loop itself). This allows you to quickly prototype different research ideas in a flexible way with minimal code.\n","\n","For more information about building RNN in keras, please visit TensorFlow official documentation [here](https://www.tensorflow.org/guide/keras/rnn)"]},{"cell_type":"code","metadata":{"id":"40eS9Bh1Zdv2","executionInfo":{"status":"ok","timestamp":1601646720155,"user_tz":-480,"elapsed":690,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}}},"source":["import tensorflow as tf\n","import numpy as np\n","\n","from tensorflow.keras.preprocessing.sequence import pad_sequences\n","from tensorflow.keras.layers import Embedding, SimpleRNN, Dense, LSTM, GRU\n","from tensorflow.keras.preprocessing.text import Tokenizer\n","from tensorflow.keras.models import Sequential\n","from tensorflow.keras.optimizers import Adam"],"execution_count":14,"outputs":[]},{"cell_type":"code","metadata":{"id":"EeQB9ZT-15qr","executionInfo":{"status":"ok","timestamp":1601646720732,"user_tz":-480,"elapsed":697,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"1385c44c-798d-4837-9615-fe4dd5cfe204","colab":{"base_uri":"https://localhost:8080/","height":74}},"source":["# initialize a tokenizer first\n","tokenizer = Tokenizer()\n","\n","overview = open('eee-overview.txt', 'r').read()\n","corpus = overview.lower().split(\"\\n\")\n","tokenizer.fit_on_texts(corpus)\n","total_words = len(tokenizer.word_index) + 1\n","\n","print(tokenizer.word_index)\n","print(total_words)"],"execution_count":15,"outputs":[{"output_type":"stream","text":["{'the': 1, 'and': 2, 'of': 3, 'in': 4, 'research': 5, 'ntu': 6, 'school': 7, 'eee': 8, 'engineering': 9, 'first': 10, 'are': 11, 'to': 12, 'satellite': 13, 'over': 14, 'with': 15, 'a': 16, 'is': 17, 'by': 18, 'also': 19, 'from': 20, 'electrical': 21, 'electronic': 22, 'one': 23, 'schools': 24, 'technological': 25, 'students': 26, 'has': 27, 'ranked': 28, 'world': 29, 'members': 30, 'centres': 31, 'undergraduate': 32, '9': 33, 'as': 34, 'nanyang': 35, 'university': 36, 'graduates': 37, 'that': 38, 'top': 39, '2020': 40, 'for': 41, 'academic': 42, 'faculty': 43, 'broad': 44, 'diverse': 45, 'supported': 46, 'researchers': 47, 'these': 48, 'support': 49, 'satellites': 50, 'partners': 51, '–': 52, 'multinational': 53, 'companies': 54, 'our': 55, 'set': 56, 'up': 57, 'laboratories': 58, 'development': 59, 'being': 60, 'an': 61, 'launched': 62, 'centre': 63, 'space': 64, 'singapore': 65, '13': 66, 'graduate': 67, 'began': 68, 'three': 69, 'founding': 70, 'then': 71, 'known': 72, 'institute': 73, 'intake': 74, '194': 75, 'graduated': 76, 'successfully': 77, '1985': 78, 'marking': 79, 'batch': 80, 'today': 81, 'become': 82, 'world’s': 83, 'largest': 84, 'nurture': 85, 'competent': 86, 'engineers': 87, 'each': 88, 'year': 89, 'produces': 90, 'thousand': 91, 'who': 92, 'ready': 93, 'take': 94, 'on': 95, 'great': 96, 'ambitions': 97, 'challenges': 98, 'now': 99, '6th': 100, 'worldwide': 101, 'qs': 102, 'rankings': 103, 'subject': 104, 'only': 105, 'asian': 106, '20': 107, '11th': 108, \"shanghairanking's\": 109, 'global': 110, 'ranking': 111, 'subjects': 112, 'receives': 113, '90': 114, 'million': 115, 'annualised': 116, 'funding': 117, 'backed': 118, '150': 119, '21': 120, 'countries': 121, 'whom': 122, 'have': 123, 'backgrounds': 124, 'strong': 125, 'professional': 126, 'expertise': 127, 'they': 128, 'about': 129, '300': 130, 'all': 131, 'school’s': 132, 'extensive': 133, 'output': 134, 'big': 135, 'data': 136, 'analytics': 137, 'internet': 138, 'things': 139, 'intelligent': 140, 'transportation': 141, 'v2x': 142, 'photonics': 143, 'autonomous': 144, 'systems': 145, 'power': 146, 'energy': 147, 'biomedical': 148, 'healthcare': 149, 'communications': 150, 'artificial': 151, 'intelligence': 152, 'works': 153, 'closely': 154, 'consistently': 155, 'industry': 156, 'local': 157, 'start': 158, 'ups': 159, 'established': 160, 'names': 161, 'ensure': 162, 'we': 163, 'staying': 164, 'ahead': 165, 'tomorrow’s': 166, 'demands': 167, 'include': 168, 'rolls': 169, 'royce': 170, 'thales': 171, 'delta': 172, 'electronics': 173, 'nxp': 174, 'schaeffler': 175, 'group': 176, 'smrt': 177, 'st': 178, 'impressively': 179, '4': 180, 'counting': 181, 'corporate': 182, 'joint': 183, 'pushing': 184, 'frontiers': 185, 'innovation': 186, \"singapore's\": 187, 'programme': 188, 'proudly': 189, 'total': 190, 'locally': 191, 'made': 192, 'including': 193, 'singapore’s': 194, 'x': 195, 'sat': 196, '2011': 197, 'eee’s': 198, 'continues': 199, 'drive': 200, 'forward': 201, 'innovations': 202, 'since': 203, 'very': 204, '1999': 205, 'january': 206, '2017': 207, 'witnessed': 208, 'launch': 209, '7th': 210, 'aoba': 211, 'velox': 212, 'iii': 213, 'international': 214, 'station': 215, 'advanced': 216, 'hosts': 217, 'facilities': 218, 'which': 219, 'well': 220, 'equipped': 221, 'sophisticated': 222, 'state': 223, 'art': 224, 'equipment': 225, 'besides': 226, 'there': 227, 'more': 228, 'than': 229, '50': 230, 'supporting': 231, 'both': 232, 'teaching': 233, 'activities': 234, 'five': 235, 'hundred': 236, 'technical': 237, 'administrative': 238, 'staff': 239, 'date': 240, 'consists': 241, 'average': 242, 'close': 243, '3000': 244, 'areas': 245, 'specialisation': 246, 'around': 247, '1000': 248, 'programmes': 249, '20000': 250, 'alumni': 251, 'strength': 252, 'apart': 253, 'offering': 254, 'courses': 255, 'active': 256, 'engaging': 257, 'range': 258, 'collaborations': 259, 'renowned': 260, 'overseas': 261, 'universities': 262, 'institutes': 263}\n","264\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"AxLwCeO-jSQ5","executionInfo":{"status":"ok","timestamp":1601646725010,"user_tz":-480,"elapsed":1237,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}}},"source":["input_sequences = []\n","for line in corpus:\n","\ttoken_list = tokenizer.texts_to_sequences([line])[0]\n","\tfor i in range(1, len(token_list)):\n","\t\tn_gram_sequence = token_list[:i+1]\n","\t\tinput_sequences.append(n_gram_sequence)\n","\n","# pad sequences \n","max_sequence_len = max([len(x) for x in input_sequences])\n","input_sequences = np.array(pad_sequences(input_sequences, maxlen=max_sequence_len, padding='pre'))\n","\n","# create predictors and label\n","xs, labels = input_sequences[:,:-1],input_sequences[:,-1]\n","ys = tf.keras.utils.to_categorical(labels, num_classes=total_words)"],"execution_count":16,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"YnDRomKwiGMP"},"source":["Here, you can choose SimpleRNN, LSTM, GRU layers in *model.add(XXX(64))* to see the effects of different RNN layers."]},{"cell_type":"code","metadata":{"id":"tFINq5R-oA49","executionInfo":{"status":"ok","timestamp":1601647429665,"user_tz":-480,"elapsed":1390,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"81fa919e-7f77-49d7-85ab-a18c5c4de89b","colab":{"base_uri":"https://localhost:8080/","height":272}},"source":["# model building\n","model = Sequential()\n","model.add(Embedding(total_words, 64, input_length=max_sequence_len-1))\n","model.add(GRU(64))\n","model.add(Dense(total_words, activation='softmax'))\n","model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n","model.summary()"],"execution_count":30,"outputs":[{"output_type":"stream","text":["Model: \"sequential_5\"\n","_________________________________________________________________\n","Layer (type)                 Output Shape              Param #   \n","=================================================================\n","embedding_5 (Embedding)      (None, 58, 64)            201792    \n","_________________________________________________________________\n","gru_1 (GRU)                  (None, 64)                24960     \n","_________________________________________________________________\n","dense_5 (Dense)              (None, 3153)              204945    \n","=================================================================\n","Total params: 431,697\n","Trainable params: 431,697\n","Non-trainable params: 0\n","_________________________________________________________________\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"PI74kHYJjTG0","executionInfo":{"status":"ok","timestamp":1601647000241,"user_tz":-480,"elapsed":36004,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"e75c8a13-61aa-4144-c78b-46f90c6e384d","colab":{"base_uri":"https://localhost:8080/","height":1000}},"source":["simple_history = model.fit(xs, ys, epochs=100, verbose=1)"],"execution_count":20,"outputs":[{"output_type":"stream","text":["Epoch 1/100\n","16/16 [==============================] - 0s 19ms/step - loss: 5.5725 - accuracy: 0.0165\n","Epoch 2/100\n","16/16 [==============================] - 0s 19ms/step - loss: 5.5435 - accuracy: 0.0723\n","Epoch 3/100\n","16/16 [==============================] - 0s 19ms/step - loss: 5.4039 - accuracy: 0.0640\n","Epoch 4/100\n","16/16 [==============================] - 0s 19ms/step - loss: 5.1872 - accuracy: 0.0599\n","Epoch 5/100\n","16/16 [==============================] - 0s 19ms/step - loss: 5.0970 - accuracy: 0.0579\n","Epoch 6/100\n","16/16 [==============================] - 0s 18ms/step - loss: 5.0525 - accuracy: 0.0579\n","Epoch 7/100\n","16/16 [==============================] - 0s 19ms/step - loss: 5.0151 - accuracy: 0.0579\n","Epoch 8/100\n","16/16 [==============================] - 0s 19ms/step - loss: 4.9822 - accuracy: 0.0682\n","Epoch 9/100\n","16/16 [==============================] - 0s 18ms/step - loss: 4.9078 - accuracy: 0.0682\n","Epoch 10/100\n","16/16 [==============================] - 0s 19ms/step - loss: 4.8462 - accuracy: 0.0806\n","Epoch 11/100\n","16/16 [==============================] - 0s 19ms/step - loss: 4.7704 - accuracy: 0.1116\n","Epoch 12/100\n","16/16 [==============================] - 0s 19ms/step - loss: 4.6898 - accuracy: 0.1260\n","Epoch 13/100\n","16/16 [==============================] - 0s 19ms/step - loss: 4.6066 - accuracy: 0.1653\n","Epoch 14/100\n","16/16 [==============================] - 0s 19ms/step - loss: 4.5073 - accuracy: 0.1839\n","Epoch 15/100\n","16/16 [==============================] - 0s 19ms/step - loss: 4.3897 - accuracy: 0.1818\n","Epoch 16/100\n","16/16 [==============================] - 0s 19ms/step - loss: 4.2733 - accuracy: 0.1983\n","Epoch 17/100\n","16/16 [==============================] - 0s 18ms/step - loss: 4.1706 - accuracy: 0.1921\n","Epoch 18/100\n","16/16 [==============================] - 0s 19ms/step - loss: 4.0429 - accuracy: 0.2149\n","Epoch 19/100\n","16/16 [==============================] - 0s 19ms/step - loss: 3.9149 - accuracy: 0.2335\n","Epoch 20/100\n","16/16 [==============================] - 0s 19ms/step - loss: 3.7870 - accuracy: 0.2541\n","Epoch 21/100\n","16/16 [==============================] - 0s 19ms/step - loss: 3.6587 - accuracy: 0.2583\n","Epoch 22/100\n","16/16 [==============================] - 0s 20ms/step - loss: 3.5347 - accuracy: 0.2727\n","Epoch 23/100\n","16/16 [==============================] - 0s 19ms/step - loss: 3.4078 - accuracy: 0.2872\n","Epoch 24/100\n","16/16 [==============================] - 0s 19ms/step - loss: 3.2824 - accuracy: 0.3079\n","Epoch 25/100\n","16/16 [==============================] - 0s 19ms/step - loss: 3.1631 - accuracy: 0.3182\n","Epoch 26/100\n","16/16 [==============================] - 0s 18ms/step - loss: 3.0436 - accuracy: 0.3512\n","Epoch 27/100\n","16/16 [==============================] - 0s 20ms/step - loss: 2.9353 - accuracy: 0.3533\n","Epoch 28/100\n","16/16 [==============================] - 0s 19ms/step - loss: 2.8192 - accuracy: 0.3698\n","Epoch 29/100\n","16/16 [==============================] - 0s 20ms/step - loss: 2.7106 - accuracy: 0.3946\n","Epoch 30/100\n","16/16 [==============================] - 0s 20ms/step - loss: 2.6082 - accuracy: 0.4360\n","Epoch 31/100\n","16/16 [==============================] - 0s 20ms/step - loss: 2.5100 - accuracy: 0.4938\n","Epoch 32/100\n","16/16 [==============================] - 0s 20ms/step - loss: 2.4016 - accuracy: 0.4752\n","Epoch 33/100\n","16/16 [==============================] - 0s 19ms/step - loss: 2.3033 - accuracy: 0.5207\n","Epoch 34/100\n","16/16 [==============================] - 0s 19ms/step - loss: 2.2076 - accuracy: 0.5310\n","Epoch 35/100\n","16/16 [==============================] - 0s 19ms/step - loss: 2.1183 - accuracy: 0.5868\n","Epoch 36/100\n","16/16 [==============================] - 0s 19ms/step - loss: 2.0257 - accuracy: 0.6033\n","Epoch 37/100\n","16/16 [==============================] - 0s 18ms/step - loss: 1.9447 - accuracy: 0.6467\n","Epoch 38/100\n","16/16 [==============================] - 0s 19ms/step - loss: 1.8592 - accuracy: 0.6632\n","Epoch 39/100\n","16/16 [==============================] - 0s 19ms/step - loss: 1.7790 - accuracy: 0.6777\n","Epoch 40/100\n","16/16 [==============================] - 0s 18ms/step - loss: 1.7058 - accuracy: 0.7004\n","Epoch 41/100\n","16/16 [==============================] - 0s 19ms/step - loss: 1.6361 - accuracy: 0.7149\n","Epoch 42/100\n","16/16 [==============================] - 0s 19ms/step - loss: 1.5637 - accuracy: 0.7252\n","Epoch 43/100\n","16/16 [==============================] - 0s 19ms/step - loss: 1.5017 - accuracy: 0.7459\n","Epoch 44/100\n","16/16 [==============================] - 0s 19ms/step - loss: 1.4345 - accuracy: 0.7397\n","Epoch 45/100\n","16/16 [==============================] - 0s 19ms/step - loss: 1.3744 - accuracy: 0.7479\n","Epoch 46/100\n","16/16 [==============================] - 0s 19ms/step - loss: 1.3179 - accuracy: 0.7645\n","Epoch 47/100\n","16/16 [==============================] - 0s 18ms/step - loss: 1.2655 - accuracy: 0.7893\n","Epoch 48/100\n","16/16 [==============================] - 0s 19ms/step - loss: 1.2185 - accuracy: 0.7975\n","Epoch 49/100\n","16/16 [==============================] - 0s 19ms/step - loss: 1.1690 - accuracy: 0.7975\n","Epoch 50/100\n","16/16 [==============================] - 0s 18ms/step - loss: 1.1233 - accuracy: 0.7955\n","Epoch 51/100\n","16/16 [==============================] - 0s 19ms/step - loss: 1.0802 - accuracy: 0.8140\n","Epoch 52/100\n","16/16 [==============================] - 0s 19ms/step - loss: 1.0384 - accuracy: 0.8223\n","Epoch 53/100\n","16/16 [==============================] - 0s 18ms/step - loss: 0.9989 - accuracy: 0.8347\n","Epoch 54/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.9605 - accuracy: 0.8512\n","Epoch 55/100\n","16/16 [==============================] - 0s 18ms/step - loss: 0.9241 - accuracy: 0.8616\n","Epoch 56/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.8967 - accuracy: 0.8492\n","Epoch 57/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.8612 - accuracy: 0.8657\n","Epoch 58/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.8320 - accuracy: 0.8636\n","Epoch 59/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.8031 - accuracy: 0.8657\n","Epoch 60/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.7735 - accuracy: 0.8802\n","Epoch 61/100\n","16/16 [==============================] - 0s 20ms/step - loss: 0.7487 - accuracy: 0.8864\n","Epoch 62/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.7219 - accuracy: 0.9008\n","Epoch 63/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.6970 - accuracy: 0.9091\n","Epoch 64/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.6745 - accuracy: 0.9112\n","Epoch 65/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.6522 - accuracy: 0.9174\n","Epoch 66/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.6345 - accuracy: 0.9298\n","Epoch 67/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.6121 - accuracy: 0.9318\n","Epoch 68/100\n","16/16 [==============================] - 0s 18ms/step - loss: 0.5891 - accuracy: 0.9318\n","Epoch 69/100\n","16/16 [==============================] - 0s 18ms/step - loss: 0.5703 - accuracy: 0.9360\n","Epoch 70/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.5529 - accuracy: 0.9401\n","Epoch 71/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.5330 - accuracy: 0.9442\n","Epoch 72/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.5156 - accuracy: 0.9442\n","Epoch 73/100\n","16/16 [==============================] - 0s 18ms/step - loss: 0.5013 - accuracy: 0.9566\n","Epoch 74/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.4850 - accuracy: 0.9545\n","Epoch 75/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.4717 - accuracy: 0.9504\n","Epoch 76/100\n","16/16 [==============================] - 0s 18ms/step - loss: 0.4569 - accuracy: 0.9587\n","Epoch 77/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.4428 - accuracy: 0.9607\n","Epoch 78/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.4301 - accuracy: 0.9669\n","Epoch 79/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.4166 - accuracy: 0.9628\n","Epoch 80/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.4040 - accuracy: 0.9669\n","Epoch 81/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.3929 - accuracy: 0.9649\n","Epoch 82/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.3790 - accuracy: 0.9669\n","Epoch 83/100\n","16/16 [==============================] - 0s 18ms/step - loss: 0.3687 - accuracy: 0.9711\n","Epoch 84/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.3580 - accuracy: 0.9711\n","Epoch 85/100\n","16/16 [==============================] - 0s 18ms/step - loss: 0.3463 - accuracy: 0.9731\n","Epoch 86/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.3389 - accuracy: 0.9669\n","Epoch 87/100\n","16/16 [==============================] - 0s 18ms/step - loss: 0.3284 - accuracy: 0.9711\n","Epoch 88/100\n","16/16 [==============================] - 0s 18ms/step - loss: 0.3199 - accuracy: 0.9731\n","Epoch 89/100\n","16/16 [==============================] - 0s 20ms/step - loss: 0.3101 - accuracy: 0.9731\n","Epoch 90/100\n","16/16 [==============================] - 0s 18ms/step - loss: 0.2984 - accuracy: 0.9752\n","Epoch 91/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.2904 - accuracy: 0.9711\n","Epoch 92/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.2828 - accuracy: 0.9773\n","Epoch 93/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.2742 - accuracy: 0.9773\n","Epoch 94/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.2659 - accuracy: 0.9793\n","Epoch 95/100\n","16/16 [==============================] - 0s 20ms/step - loss: 0.2591 - accuracy: 0.9814\n","Epoch 96/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.2518 - accuracy: 0.9793\n","Epoch 97/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.2442 - accuracy: 0.9814\n","Epoch 98/100\n","16/16 [==============================] - 0s 18ms/step - loss: 0.2393 - accuracy: 0.9752\n","Epoch 99/100\n","16/16 [==============================] - 0s 18ms/step - loss: 0.2340 - accuracy: 0.9773\n","Epoch 100/100\n","16/16 [==============================] - 0s 19ms/step - loss: 0.2277 - accuracy: 0.9814\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"tpgGNj33kHCe","executionInfo":{"status":"ok","timestamp":1601647005080,"user_tz":-480,"elapsed":2328,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"e72083f1-1530-4996-dfd8-1ff09d8d857f","colab":{"base_uri":"https://localhost:8080/","height":110}},"source":["seed_text = \"eee is\"\n","next_words = 20\n","  \n","for _ in range(next_words):\n","\ttoken_list = tokenizer.texts_to_sequences([seed_text])[0]\n","\ttoken_list = pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')\n","\tpredicted = model.predict_classes(token_list, verbose=0)\n","\toutput_word = \"\"\n","\tfor word, index in tokenizer.word_index.items():\n","\t\tif index == predicted:\n","\t\t\toutput_word = word\n","\t\t\tbreak\n","\tseed_text += \" \" + output_word\n","print(seed_text)"],"execution_count":21,"outputs":[{"output_type":"stream","text":["WARNING:tensorflow:From <ipython-input-21-5040c2917d24>:7: Sequential.predict_classes (from tensorflow.python.keras.engine.sequential) is deprecated and will be removed after 2021-01-01.\n","Instructions for updating:\n","Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n","eee is also supported by over five hundred research technical and administrative staff members from 21 countries whom have broad diverse backgrounds\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"8EeLky-wMmAh"},"source":["## **3. Lyrics Generator (Word Tokenization)**"]},{"cell_type":"markdown","metadata":{"id":"-EyRi_ZBwmXe"},"source":["This tutorial demonstate how to generator text based on given text using word tokenization and RNN. The text containing the song titles and the lyrics of many famous songs of Beatles (credit: [petrosDemetrakopoulos](https://github.com/petrosDemetrakopoulos/)). So, given a sequence of words from Beatles lyrics, it can predict the next words.\n","\n","\n","\n"]},{"cell_type":"markdown","metadata":{"id":"STvk_xgMp0NN"},"source":["<img src = \"https://miro.medium.com/max/2560/0*SUipu9efyQeKHdlk.\" width = 70%>"]},{"cell_type":"markdown","metadata":{"id":"T50RSvBi49Xq"},"source":["### ***Task 3: Build the Model and Train It!***\n","\n","In this task, you will get the chance to experience the whole process of building a lyrics generator using Tensorflow Keras."]},{"cell_type":"markdown","metadata":{"id":"dWKxNvBk27sO"},"source":["Following the steps in **section two** (very important, make sure you understand them all) and create a lyrics generator for given text file."]},{"cell_type":"code","metadata":{"id":"NUzWitBkMyER","executionInfo":{"status":"ok","timestamp":1601647028629,"user_tz":-480,"elapsed":1721,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}}},"source":["import tensorflow as tf\n","import numpy as np\n","from tensorflow.keras.preprocessing.sequence import pad_sequences\n","from tensorflow.keras.layers import Embedding, LSTM, GRU, SimpleRNN, Dense, Dropout, Bidirectional\n","from tensorflow.keras.preprocessing.text import Tokenizer\n","from tensorflow.keras.models import Sequential\n","from tensorflow.keras.optimizers import Adam\n","from tensorflow.keras import regularizers"],"execution_count":22,"outputs":[]},{"cell_type":"code","metadata":{"id":"7gn8iZdZ6gbo","executionInfo":{"status":"ok","timestamp":1601647031761,"user_tz":-480,"elapsed":1951,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"283d9ca6-f2ad-4cda-f728-e193828da150","colab":{"base_uri":"https://localhost:8080/","height":237}},"source":["!wget https://ycrao573.github.io/rnn-workshop/lyrics.txt"],"execution_count":23,"outputs":[{"output_type":"stream","text":["--2020-10-02 13:57:10--  https://ycrao573.github.io/rnn-workshop/lyrics.txt\n","Resolving ycrao573.github.io (ycrao573.github.io)... 185.199.108.153, 185.199.110.153, 185.199.111.153, ...\n","Connecting to ycrao573.github.io (ycrao573.github.io)|185.199.108.153|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 245890 (240K) [text/plain]\n","Saving to: ‘lyrics.txt’\n","\n","lyrics.txt          100%[===================>] 240.13K  --.-KB/s    in 0.05s   \n","\n","2020-10-02 13:57:10 (4.50 MB/s) - ‘lyrics.txt’ saved [245890/245890]\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"K22C-bbA3MYz","executionInfo":{"status":"ok","timestamp":1601647091487,"user_tz":-480,"elapsed":1960,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"e6b5d491-7d02-4909-9ac1-a9b7b9ff7def","colab":{"base_uri":"https://localhost:8080/","height":199}},"source":["# open the 'lyrics.txt' file\n","text = open('lyrics.txt', 'r').read()\n","# print length of text and first 250 words of the text\n","# INSERT YOUR CODE HERE\n","print ('Length of text: {} characters'.format(len(text)))\n","print (text[:250])"],"execution_count":24,"outputs":[{"output_type":"stream","text":["Length of text: 245664 characters\n","A Day In The Life\n","-----------------\n","I read the news today oh boy\n","About a lucky man who made the grade\n","And though the news was rather sad\n","Well I just had to laugh\n","I saw the photograph.\n","He blew his mind out in a car\n","He didn't notice that the lights hav\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"p9aQkS97TV-I","executionInfo":{"status":"ok","timestamp":1601647267780,"user_tz":-480,"elapsed":1286,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"a2a8d87b-502b-499b-d50e-25b4123dff00","colab":{"base_uri":"https://localhost:8080/","height":74}},"source":["tokenizer = Tokenizer()\n","\n","corpus = text.lower().split(\"\\n\")\n","\n","# Tokenization Process\n","# INSERT YOUR CODE HERE\n","tokenizer.fit_on_texts(corpus)\n","\n","total_words = len(tokenizer.word_index) + 1 # length of the corpus\n","\n","print(tokenizer.word_index)\n","print(total_words)"],"execution_count":27,"outputs":[{"output_type":"stream","text":["{'you': 1, 'i': 2, 'the': 3, 'me': 4, 'to': 5, 'and': 6, 'a': 7, 'love': 8, 'my': 9, 'in': 10, 'be': 11, 'that': 12, \"don't\": 13, 'it': 14, 'do': 15, 'all': 16, \"i'm\": 17, 'on': 18, 'your': 19, 'of': 20, 'is': 21, 'oh': 22, 'know': 23, 'so': 24, 'she': 25, \"it's\": 26, 'for': 27, 'when': 28, 'baby': 29, 'can': 30, 'but': 31, 'now': 32, 'her': 33, 'well': 34, 'no': 35, 'if': 36, 'got': 37, 'what': 38, 'with': 39, 'want': 40, 'just': 41, 'see': 42, 'come': 43, 'say': 44, 'like': 45, 'one': 46, \"you're\": 47, \"i'll\": 48, 'girl': 49, 'yeah': 50, 'gonna': 51, 'little': 52, 'get': 53, 'go': 54, 'down': 55, 'will': 56, 'time': 57, \"she's\": 58, 'let': 59, 'never': 60, 'tell': 61, 'was': 62, 'day': 63, 'said': 64, 'yeh': 65, 'back': 66, 'we': 67, 'they': 68, 'how': 69, 'there': 70, 'way': 71, \"can't\": 72, 'make': 73, 'man': 74, 'good': 75, 'have': 76, 'over': 77, \"i've\": 78, 'here': 79, 'are': 80, 'long': 81, 'not': 82, 'he': 83, 'up': 84, 'hey': 85, 'as': 86, \"that's\": 87, 'night': 88, 'take': 89, 'out': 90, 'please': 91, 'at': 92, 'feel': 93, 'from': 94, 'wanna': 95, 'why': 96, 'need': 97, 'right': 98, 'cry': 99, 'home': 100, 'la': 101, 'hold': 102, 'ever': 103, 'this': 104, 'more': 105, 'only': 106, 'been': 107, 'give': 108, 'world': 109, 'yes': 110, 'better': 111, 'sun': 112, 'blue': 113, 'mine': 114, 'leave': 115, 'life': 116, 'his': 117, \"won't\": 118, 'too': 119, 'really': 120, 'some': 121, 'away': 122, 'eyes': 123, 'look': 124, 'nothing': 125, 'da': 126, 'where': 127, 'mm': 128, 'by': 129, 'had': 130, 'then': 131, 'goodbye': 132, 'something': 133, 'heart': 134, 'again': 135, 'honey': 136, 'shout': 137, \"ain't\": 138, 'hello': 139, 'through': 140, 'true': 141, \"there's\": 142, 'roll': 143, 'ah': 144, 'think': 145, 'would': 146, 'told': 147, 'mind': 148, 'who': 149, 'i’m': 150, \"you've\": 151, 'boy': 152, 'going': 153, 'before': 154, 'call': 155, 'sweet': 156, 'hard': 157, 'things': 158, 'kiss': 159, 'shake': 160, 'dance': 161, 'together': 162, 'always': 163, 'alone': 164, 'much': 165, 'money': 166, 'old': 167, 'find': 168, 'were': 169, 'lonely': 170, 'waiting': 171, 'bye': 172, 'gotta': 173, 'loves': 174, 'tonight': 175, 'could': 176, 'head': 177, 'inside': 178, 'hear': 179, 'free': 180, \"'cos\": 181, \"you'll\": 182, 'keep': 183, 'feeling': 184, 'ooh': 185, 'should': 186, 'about': 187, 'or': 188, 'has': 189, 'them': 190, 'around': 191, 'him': 192, 'am': 193, 'believe': 194, 'bad': 195, 'says': 196, 'care': 197, 'two': 198, 'thing': 199, 'other': 200, 'any': 201, 'oo': 202, 'people': 203, 'nobody': 204, 'may': 205, 'an': 206, 'remember': 207, \"i'd\": 208, 'woman': 209, 'anything': 210, 'change': 211, 'us': 212, 'still': 213, 'happy': 214, 'shoes': 215, 'dear': 216, 'comes': 217, 'done': 218, 'did': 219, 'off': 220, 'glad': 221, 'mean': 222, 'kind': 223, 'real': 224, \"we're\": 225, 'sing': 226, 'bring': 227, 'another': 228, 'run': 229, 'hand': 230, 'fine': 231, 'sad': 232, 'found': 233, 'hide': 234, 'these': 235, 'tight': 236, 'touch': 237, \"didn't\": 238, 'yoko': 239, 'show': 240, 'knew': 241, 'made': 242, 'stay': 243, 'rain': 244, 'our': 245, 'than': 246, 'till': 247, 'darling': 248, 'dead': 249, 'help': 250, 'trying': 251, 'mama': 252, 'goes': 253, 'alright': 254, 'loving': 255, \"d'\": 256, 'today': 257, 'easy': 258, 'bop': 259, 'suede': 260, 'fun': 261, 'though': 262, 'sure': 263, 'every': 264, 'someone': 265, 'hi': 266, 'without': 267, 'words': 268, 'very': 269, 'gone': 270, 'babe': 271, 'saw': 272, 'into': 273, 'dream': 274, \"'cause\": 275, 'play': 276, 'o': 277, 'place': 278, 'rock': 279, 'sleep': 280, 'step': 281, 'understand': 282, 'morning': 283, 'buy': 284, 'penina': 285, 'looking': 286, 'many': 287, 'wait': 288, 'does': 289, 'big': 290, 'everything': 291, 'thinking': 292, 'years': 293, 'came': 294, 'after': 295, \"ev'rything\": 296, 'wrong': 297, 'gimme': 298, 'their': 299, 'ask': 300, 'once': 301, 'because': 302, 'lay': 303, 'wish': 304, 'mother': 305, 'standing': 306, 'turn': 307, 'fool': 308, 'bird': 309, 'same': 310, 'try': 311, 'chance': 312, \"'bout\": 313, 'getting': 314, 'seems': 315, 'door': 316, \"they're\": 317, 'nine': 318, 'brown': 319, 'new': 320, 'die': 321, 'beautiful': 322, 'last': 323, 'music': 324, 'sky': 325, 'since': 326, 'soul': 327, 'wo': 328, 'must': 329, 'crying': 330, 'coming': 331, \"c'mon\": 332, 'lies': 333, 'face': 334, 'while': 335, 'start': 336, 'forever': 337, \"you'd\": 338, 'shine': 339, 'boys': 340, 'lula': 341, 'making': 342, 'tears': 343, 'bill': 344, 'miss': 345, 'twist': 346, 'blues': 347, 'lover': 348, 'forget': 349, 'bit': 350, 'town': 351, 'song': 352, 'hope': 353, 'nowhere': 354, \"ev'ry\": 355, 'mi': 356, 'times': 357, 'cried': 358, 'maybe': 359, 'even': 360, 'each': 361, 'send': 362, 'knows': 363, 'cold': 364, 'hoping': 365, 'listen': 366, 'near': 367, 'such': 368, 'doing': 369, 'car': 370, 'round': 371, 'heard': 372, 'stop': 373, 'yourself': 374, 'rattle': 375, 'name': 376, 'loved': 377, 'lie': 378, 'guy': 379, 'talk': 380, 'child': 381, 'hands': 382, 'bonnie': 383, 'ob': 384, 'writer': 385, 'late': 386, 'somebody': 387, 'black': 388, 'live': 389, 'rich': 390, 'leaves': 391, \"he's\": 392, 'friends': 393, 'pretty': 394, 'sit': 395, 'doctor': 396, 'mad': 397, 'worry': 398, 'lot': 399, \"that'll\": 400, 'across': 401, 'light': 402, 'friend': 403, 'gave': 404, 'sunshine': 405, 'end': 406, 'tomorrow': 407, 'lost': 408, 'walk': 409, 'carry': 410, 'fly': 411, 'mr': 412, 'days': 413, 'dreams': 414, 'arms': 415, 'birthday': 416, 'clarabella': 417, 'sunday': 418, 'gets': 419, 'hurt': 420, 'robert': 421, 'window': 422, 'paperback': 423, 'bungalow': 424, 'smile': 425, 'eye': 426, 'sea': 427, 'enough': 428, 'left': 429, 'makes': 430, 'band': 431, 'took': 432, 'side': 433, 'chains': 434, 'week': 435, 'christmas': 436, 'bloody': 437, 'julia': 438, 'laugh': 439, 'seen': 440, 'went': 441, \"we'll\": 442, 'street': 443, 'far': 444, \"doesn't\": 445, 'thought': 446, 'those': 447, 'break': 448, 'moonlight': 449, 'pain': 450, 'fast': 451, 'dog': 452, 'leaving': 453, 'driving': 454, 'ye': 455, 'gun': 456, 'los': 457, 'paranoias': 458, 'sheila': 459, 'changed': 460, 'until': 461, 'open': 462, 'broken': 463, 'put': 464, 'might': 465, 'bom': 466, 'truth': 467, 'sitting': 468, 'feet': 469, 'afraid': 470, 'use': 471, 'move': 472, 'treat': 473, 'middle': 474, 'falling': 475, \"lovin'\": 476, 'lose': 477, 'lovely': 478, 'peace': 479, 'searching': 480, 'which': 481, 'telling': 482, 'behind': 483, 'trees': 484, 'singing': 485, 'thank': 486, 'prudence': 487, 'first': 488, 'belong': 489, \"she'll\": 490, 'rhythm': 491, 'number': 492, 'wants': 493, 'queenie': 494, 'taxman': 495, 'war': 496, 'bed': 497, 'i’ll': 498, \"nothing's\": 499, 'wind': 500, 'eight': 501, 'learn': 502, 'kill': 503, 'bag': 504, 'doll': 505, 'meet': 506, 'reason': 507, 'jump': 508, 'drive': 509, 'children': 510, 'garden': 511, 'tried': 512, 'lips': 513, 'satisfied': 514, 'looked': 515, 'wonder': 516, 'silly': 517, 'tu': 518, 'strawberry': 519, 'fields': 520, \"ev'rybody\": 521, 'sleeping': 522, 'soldier': 523, 'trouble': 524, 'shaking': 525, 'bang': 526, 'rocky': 527, 'teddy': 528, 'dee': 529, 'saints': 530, 'marching': 531, 'takes': 532, 'joy': 533, 'star': 534, 'tree': 535, 'save': 536, 'sound': 537, 'shot': 538, 'quite': 539, 'sorry': 540, 'hair': 541, \"'round\": 542, 'dark': 543, 'ocean': 544, 'high': 545, 'holding': 546, 'instead': 547, 'doggone': 548, 'prision': 549, 'work': 550, 'hate': 551, 'deep': 552, 'catch': 553, 'ho': 554, 'misery': 555, 'used': 556, \"baby's\": 557, 'apart': 558, 'matter': 559, 'minute': 560, 'saying': 561, 'beethoven': 562, 'tired': 563, \"let's\": 564, 'john': 565, 'needed': 566, 'helter': 567, 'skelter': 568, 'silver': 569, 'mystery': 570, 'tour': 571, 'travelling': 572, 'anyway': 573, 'sexy': 574, 'sadie': 575, 'rather': 576, 'shines': 577, 'act': 578, 'part': 579, 'twice': 580, 'red': 581, \"isn't\": 582, 'girls': 583, 'everywhere': 584, 'looks': 585, 'ground': 586, 'beat': 587, 'close': 588, 'turns': 589, 'ring': 590, 'party': 591, 'devil': 592, 'steal': 593, 'begin': 594, 'king': 595, 'top': 596, 'taking': 597, 'hole': 598, 'dig': 599, 'someday': 600, 'letter': 601, 'whoa': 602, 'anymore': 603, 'myself': 604, 'means': 605, 'brother': 606, 'living': 607, 'anybody': 608, 'xmas': 609, 'answer': 610, 'bulldog': 611, 'jude': 612, 'pie': 613, 'sgt': 614, 'ev´ry': 615, 'nyc': 616, \"octopus's\": 617, 'lucky': 618, 'english': 619, 'upon': 620, 'b': 621, 'moving': 622, 'soon': 623, 'attica': 624, 'ussr': 625, 'warm': 626, \"he'll\": 627, 'besame': 628, 'moment': 629, 'past': 630, 'kitchen': 631, 'drink': 632, 'along': 633, 'uh': 634, 'needs': 635, 'heavy': 636, 'walrus': 637, 'early': 638, 'hurry': 639, 'sigh': 640, 'supposed': 641, 'road': 642, \"ev'ryone\": 643, 'set': 644, 'sha': 645, 'running': 646, 'mister': 647, 'keeps': 648, \"rockin'\": 649, 'working': 650, 'shining': 651, 'beneath': 652, 'everyone': 653, 'loser': 654, 'city': 655, 'clothes': 656, 'sometimes': 657, 'harm': 658, 'year': 659, 'goo': 660, 'softer': 661, 'agree': 662, 'enjoy': 663, 'lucille': 664, 'guilty': 665, \"mummy's\": 666, 'que': 667, 'di': 668, 'penny': 669, 'lane': 670, 'september': 671, 'news': 672, 'four': 673, 'pass': 674, 'sorrow': 675, 'hit': 676, 'confidentially': 677, 'nice': 678, 'ten': 679, 'e': 680, 'bompa': 681, 'wear': 682, 'sister': 683, 'met': 684, 'lives': 685, 'phone': 686, \"they'll\": 687, 'talking': 688, 'being': 689, 'k': 690, 'cha': 691, 'blackbird': 692, 'born': 693, 'promise': 694, 'funny': 695, 'tripper': 696, 'half': 697, 'low': 698, 'seem': 699, 'fall': 700, 'room': 701, 'shuop': 702, 'happiness': 703, 'whisper': 704, 'whenever': 705, 'hill': 706, 'sent': 707, 'lizzie': 708, 'young': 709, 'sand': 710, 'best': 711, 'son': 712, \"here's\": 713, 'wife': 714, \"we'd\": 715, 'cause': 716, 'waited': 717, 'lied': 718, 'hearts': 719, 'rita': 720, 'hammer': 721, 'pasa': 722, 'ny': 723, 'molly': 724, 'bra': 725, 'lord': 726, 'louder': 727, 'irish': 728, \"he'd\": 729, 'fell': 730, 'flat': 731, 'birds': 732, 'calls': 733, 'walking': 734, 'three': 735, 'known': 736, 'angela': 737, 'state': 738, 'stand': 739, 'knock': 740, 'buys': 741, 'queen': 742, 'gives': 743, 'begins': 744, 'mucho': 745, \"'em\": 746, 'boots': 747, 'weight': 748, 'pretend': 749, 'bother': 750, 'dizzy': 751, 'belonged': 752, 'cool': 753, 'admit': 754, 'story': 755, 'shows': 756, 'glass': 757, 'wake': 758, 'line': 759, 'nearly': 760, 'appear': 761, \"day's\": 762, 'bright': 763, 'else': 764, 'clear': 765, 'ma': 766, \"thru'\": 767, 'hung': 768, 'losing': 769, 'spite': 770, 'isolation': 771, 'na': 772, 'lend': 773, 'kitten': 774, 'dirty': 775, 'taken': 776, 'magical': 777, 'martha': 778, \"maxwell's\": 779, 'michelle': 780, 'bay': 781, 'desmond': 782, \"pepper's\": 783, 'georgia': 784, 'comb': 785, 'fill': 786, 'million': 787, 'earth': 788, 'movies': 789, 'win': 790, 'knee': 791, 'tea': 792, 'white': 793, 'yellow': 794, 'everybody': 795, 'prisioners': 796, 'word': 797, 'shoulder': 798, 'poor': 799, 'grow': 800, 'fear': 801, 'clouds': 802, 'school': 803, 'write': 804, 'plays': 805, 'plans': 806, 'hell': 807, 'pots': 808, 'pans': 809, 'it’s': 810, 'crazy': 811, 'suit': 812, 'flowers': 813, 'hay': 814, 'lying': 815, 'next': 816, \"we've\": 817, 'locked': 818, 'ya': 819, 'closer': 820, 'ear': 821, 'wine': 822, 'own': 823, 'everyday': 824, 'wonderful': 825, 'postman': 826, 'tear': 827, 'feels': 828, 'wearing': 829, 'kisses': 830, 'beep': 831, 'died': 832, 'wandering': 833, 'sign': 834, 'luck': 835, 'lady': 836, \"ev'rywhere\": 837, 'tells': 838, 'rings': 839, 'sees': 840, 'knees': 841, 'kansas': 842, 'its': 843, 'tall': 844, 'superior': 845, 'merry': 846, 'ears': 847, 'under': 848, 'air': 849, 'forward': 850, 'stream': 851, 'less': 852, 'worried': 853, 'wisdom': 854, 'dreamers': 855, 'meanwhile': 856, 'meter': 857, 'mailman': 858, 'trés': 859, 'bien': 860, 'ensemble': 861, 'shade': 862, 'broke': 863, 'piggies': 864, 'sails': 865, 'sunset': 866, 'club': 867, 'sixteen': 868, 'ted': 869, 'mummy': 870, \"boy's\": 871, 'crusify': 872, 'read': 873, 'house': 874, 'having': 875, 'count': 876, 'guru': 877, 'de': 878, 'naturally': 879, 'bended': 880, 'six': 881, 'c': 882, 's': 883, 'human': 884, 'often': 885, 'rocking': 886, 'spinning': 887, 'guess': 888, 'somehow': 889, 'second': 890, 'wings': 891, 'thrill': 892, \"they've\": 893, 'medley': 894, 'called': 895, 'self': 896, 'machine': 897, 'outside': 898, 'above': 899, 'woo': 900, 'turkey': 901, 'wide': 902, 'finger': 903, 'playing': 904, 'pictures': 905, 'message': 906, 'ha': 907, 'watching': 908, 't': 909, 'blow': 910, 'held': 911, 'drag': 912, 'anna': 913, 'happened': 914, 'return': 915, 'kissing': 916, 'blame': 917, 'caught': 918, \"sittin'\": 919, 'strong': 920, 'insane': 921, 'pay': 922, 'except': 923, 'allright': 924, 'asked': 925, 'death': 926, \"everybody's\": 927, \"ev'rybody's\": 928, 'yesterday': 929, 'ago': 930, \"couldn't\": 931, 'loretta': 932, 'finally': 933, 'short': 934, 'oothss': 935, 'lead': 936, 'hare': 937, 'onion': 938, 'madonna': 939, 'golden': 940, 'sings': 941, 'suddenly': 942, \"wond'ring\": 943, 'giving': 944, 'yay': 945, 'diamond': 946, 'feelings': 947, 'dressed': 948, 'follow': 949, 'huh': 950, 'ought': 951, 'slow': 952, 'matchbox': 953, 'jealous': 954, 'hoo': 955, 'hallelujah': 956, 'corner': 957, 'ice': 958, 'fight': 959, 'skies': 960, 'fancy': 961, 'bound': 962, 'surprise': 963, 'lucy': 964, \"she'd\": 965, 'calling': 966, 'kids': 967, 'lonesome': 968, 'rest': 969, 'maid': 970, 'memphis': 971, 'tennessee': 972, 'safely': 973, 'christ': 974, 'crowd': 975, \"they'd\": 976, 'book': 977, 'cup': 978, 'thousand': 979, 'loneliness': 980, 'universe': 981, 'waves': 982, 'jai': 983, 'va': 984, 'om': 985, 'ringing': 986, 'cast': 987, 'five': 988, 'sail': 989, 'green': 990, 'meant': 991, 're': 992, 'slowly': 993, 'teacher': 994, 'key': 995, 'sweeter': 996, 'lennon': 997, 'ono': 998, 'judges': 999, 'puts': 1000, 'magazine': 1001, 'ready': 1002, 'rolling': 1003, 'shop': 1004, 'bath': 1005, \"mother's\": 1006, 'drove': 1007, 'table': 1008, 'park': 1009, 'married': 1010, 'pick': 1011, \"who's\": 1012, 'sailing': 1013, 'smiling': 1014, 'kite': 1015, 'arise': 1016, 'asleep': 1017, 'spent': 1018, 'ball': 1019, 'burn': 1020, 'jar': 1021, 'wood': 1022, 'becomes': 1023, 'ride': 1024, 'perfect': 1025, 'spell': 1026, 'yet': 1027, 'harmony': 1028, 'however': 1029, 'roam': 1030, 'fever': 1031, 'crippled': 1032, 'cute': 1033, 'painting': 1034, \"o'clock\": 1035, 'voices': 1036, 'played': 1037, 'miles': 1038, 'watch': 1039, 'pony': 1040, 'boat': 1041, 'p': 1042, 'doubt': 1043, 'seventeen': 1044, 'breaks': 1045, 'cheat': 1046, 'secret': 1047, 'taste': 1048, 'yours': 1049, \"goin'\": 1050, 'record': 1051, \"heart's\": 1052, 'spend': 1053, 'romance': 1054, 'angel': 1055, 'deceive': 1056, 'wanted': 1057, 'father': 1058, 'dirt': 1059, 'finds': 1060, 'monkey': 1061, 'fixing': 1062, 'longer': 1063, 'sight': 1064, 'soft': 1065, 'dope': 1066, 'krishna': 1067, 'cheek': 1068, 'shore': 1069, 'god': 1070, 'measure': 1071, 'full': 1072, 'songs': 1073, 'reply': 1074, 'realise': 1075, 'thinks': 1076, 'choose': 1077, \"dancin'\": 1078, \"what's\": 1079, 'beside': 1080, 'spoil': 1081, 'disappear': 1082, \"movin'\": 1083, 'ow': 1084, 'mary': 1085, \"watchin'\": 1086, 'presents': 1087, 'understands': 1088, 'trust': 1089, 'beg': 1090, 'whatever': 1091, 'u': 1092, 'evening': 1093, 'ones': 1094, 'passing': 1095, 'younger': 1096, 'bottom': 1097, 'dancer': 1098, 'speaking': 1099, 'frightened': 1100, \"haven't\": 1101, 'wa': 1102, 'lazy': 1103, 'eggman': 1104, 'eggmen': 1105, 'forgot': 1106, 'pa': 1107, 'ohoh': 1108, 'indeed': 1109, 'declare': 1110, 'whole': 1111, 'hearted': 1112, 'junelight': 1113, 'bleeding': 1114, 'float': 1115, \"talkin'\": 1116, 'wall': 1117, 'troubles': 1118, 'eat': 1119, 'winding': 1120, 'cloud': 1121, 'yard': 1122, 'heaven': 1123, 'friday': 1124, 'promising': 1125, \"sister's\": 1126, 'diamonds': 1127, 'pam': 1128, 'information': 1129, 'marie': 1130, 'belle': 1131, 'daddy': 1132, \"nature's\": 1133, 'land': 1134, 'couple': 1135, 'named': 1136, 'polythene': 1137, 'raccoon': 1138, 'sunday’s': 1139, 'tax': 1140, \"teddy's\": 1141, 'you’d': 1142, 'notice': 1143, 'army': 1144, 'won': 1145, 'turned': 1146, 'woke': 1147, 'holes': 1148, 'small': 1149, 'lock': 1150, 'allow': 1151, 'moon': 1152, 'okay': 1153, 'slip': 1154, 'scene': 1155, 'direction': 1156, 'h': 1157, 'ship': 1158, 'breathing': 1159, 'returning': 1160, 'share': 1161, 'few': 1162, 'thick': 1163, 'shoot': 1164, 'pulled': 1165, 'revolution': 1166, 'join': 1167, 'zoo': 1168, 'natural': 1169, 'west': 1170, 'chair': 1171, 'junior': 1172, 'behave': 1173, 'barber': 1174, 'threw': 1175, 'laid': 1176, 'both': 1177, 'fire': 1178, 'guaranteed': 1179, 'mccartney': 1180, 'dough': 1181, 'frying': 1182, 'pan': 1183, 'slap': 1184, 'slander': 1185, 'liquor': 1186, 'fruit': 1187, 'carl': 1188, 'perkins': 1189, 'mirror': 1190, 'carol': 1191, 'joint': 1192, 'loud': 1193, \"makin'\": 1194, 'stars': 1195, 'cleanup': 1196, 'water': 1197, 'counting': 1198, 'magic': 1199, 'shoe': 1200, 'church': 1201, 'judge': 1202, 'local': 1203, 'meeting': 1204, 'ticket': 1205, 'stands': 1206, 'sunny': 1207, 'hour': 1208, 'row': 1209, 'treating': 1210, 'george': 1211, 'nights': 1212, 'tasting': 1213, \"there'll\": 1214, 'missing': 1215, \"rollin'\": 1216, 'reel': 1217, 'important': 1218, 'desert': 1219, 'nay': 1220, 'stone': 1221, 'marry': 1222, 'special': 1223, 'works': 1224, 'ee': 1225, 'arrive': 1226, 'between': 1227, 'eleanor': 1228, 'rigby': 1229, 'waits': 1230, 'writing': 1231, 'rise': 1232, 'stops': 1233, 'filling': 1234, 'ran': 1235, 'kept': 1236, \"weren't\": 1237, 'lasted': 1238, 'jojo': 1239, 'joe': 1240, 'rules': 1241, 'hiding': 1242, 'changing': 1243, 'single': 1244, 'voice': 1245, 'most': 1246, 'stick': 1247, 'alan': 1248, 'everytime': 1249, 'speak': 1250, 'weak': 1251, 'flow': 1252, 'clue': 1253, \"tryin'\": 1254, 'ends': 1255, 'bible': 1256, 'jesus': 1257, 'homeward': 1258, 'lullaby': 1259, 'smiles': 1260, 'proud': 1261, 'invites': 1262, 'pride': 1263, 'telephone': 1264, 'although': 1265, 'realize': 1266, 'dresses': 1267, 'jubilee': 1268, 'jamboree': 1269, 'mood': 1270, 'knocking': 1271, 'date': 1272, 'sally': 1273, 'uncle': 1274, 'eh': 1275, \"havin'\": 1276, 'log': 1277, 'cos': 1278, 'wishing': 1279, 'dreaming': 1280, 'blind': 1281, 'brings': 1282, 'fix': 1283, 'sometime': 1284, 'appreciate': 1285, 'ways': 1286, \"majesty's\": 1287, 'knowing': 1288, 'sheep': 1289, 'measured': 1290, 'refrain': 1291, 'hollywood': 1292, 'hot': 1293, 'smart': 1294, 'pigs': 1295, 'joob': 1296, 'mess': 1297, 'anyone': 1298, \"ev'ryone's\": 1299, 'upset': 1300, 'filled': 1301, 'bigger': 1302, 'papa': 1303, 'exactly': 1304, \"dif'rent\": 1305, 'cannot': 1306, 'monday': 1307, 'phony': 1308, 'lift': 1309, 'keeping': 1310, 'contend': 1311, 'vantage': 1312, 'lovers': 1313, 'danger': 1314, 'heartache': 1315, 'shove': 1316, 'sinclair': 1317, 'fat': 1318, 'bliss': 1319, 'model': 1320, 'cover': 1321, 'station': 1322, 'maggie': 1323, 'mae': 1324, 'wrote': 1325, 'maxwell': 1326, 'oan': 1327, 'sont': 1328, 'les': 1329, 'mots': 1330, 'qui': 1331, 'vont': 1332, 'mountain': 1333, 'london': 1334, 'norwegian': 1335, 'drinking': 1336, 'lets': 1337, 'cats': 1338, 'market': 1339, 'built': 1340, \"trav'ling\": 1341, 'fooling': 1342, 'northern': 1343, 'chords': 1344, 'job': 1345, 'style': 1346, 'banker': 1347, 'suburban': 1348, 'clean': 1349, 'northwest': 1350, 'mountie': 1351, 'peter': 1352, 'pair': 1353, 'chills': 1354, 'peacefully': 1355, 'mommy': 1356, 'suprise': 1357, 'sincere': 1358, 'blew': 1359, 'film': 1360, 'downstairs': 1361, 'noticed': 1362, 'hat': 1363, 'tune': 1364, 'flying': 1365, 'endless': 1366, 'bet': 1367, 'oscar': 1368, 'plat': 1369, 'plainly': 1370, 'biggest': 1371, 'perfection': 1372, 'repeat': 1373, 'neat': 1374, 'seven': 1375, 'd': 1376, 'f': 1377, 'j': 1378, 'chop': 1379, 'skip': 1380, 'rope': 1381, 'game': 1382, 'saved': 1383, 'wonders': 1384, 'swing': 1385, 'dies': 1386, 'turning': 1387, 'future': 1388, 'coffee': 1389, 'waste': 1390, 'power': 1391, 'wives': 1392, 'trigger': 1393, 'mates': 1394, 'let’s': 1395, 'movement': 1396, 'rights': 1397, 'paper': 1398, 'flight': 1399, 'hardly': 1400, 'case': 1401, 'ukraine': 1402, 'moscow': 1403, \"georgia's\": 1404, 'south': 1405, \"daddy's\": 1406, 'kid': 1407, 'dime': 1408, 'jukebox': 1409, \"junior's\": 1410, 'cat': 1411, 'intention': 1412, 'letting': 1413, 'curtain': 1414, 'queue': 1415, 'davis': 1416, 'jeans': 1417, 'store': 1418, 'cross': 1419, 'busy': 1420, 'toys': 1421, 'empty': 1422, 'pole': 1423, 'scratch': 1424, 'benefit': 1425, 'hendersons': 1426, 'fair': 1427, 'men': 1428, 'performs': 1429, 'saturday': 1430, 'production': 1431, 'horse': 1432, 'solid': 1433, 'dearest': 1434, 'jay': 1435, 'fools': 1436, 'reach': 1437, 'fate': 1438, 'among': 1439, 'aw': 1440, 'foot': 1441, 'heat': 1442, 'gods': 1443, 'heavens': 1444, 'angels': 1445, 'oracle': 1446, 'spoken': 1447, 'mothers': 1448, 'body': 1449, 'thirty': 1450, 'joo': 1451, 'roller': 1452, 'joker': 1453, 'below': 1454, 'mask': 1455, 'paint': 1456, 'tie': 1457, 'mamma': 1458, 'skin': 1459, 'cooking': 1460, 'piano': 1461, 'teaser': 1462, 'driver': 1463, 'greet': 1464, 'brand': 1465, 'fading': 1466, 'smiled': 1467, 'celebrate': 1468, 'penetrate': 1469, 'radiate': 1470, 'imitate': 1471, 'indicate': 1472, 'syndicate': 1473, 'useless': 1474, \"wouldn't\": 1475, 'trip': 1476, \"m'bop\": 1477, 'fingertips': 1478, 'aaahhh': 1479, 'bundle': 1480, 'concieve': 1481, 'awoke': 1482, 'roses': 1483, 'fragrant': 1484, 'meadows': 1485, 'dawn': 1486, 'card': 1487, 'mail': 1488, 'tchaikowsky': 1489, 'diddle': 1490, \"playin'\": 1491, \"nothin'\": 1492, 'wiggles': 1493, 'partner': 1494, 'quit': 1495, 'chances': 1496, 'hugs': 1497, 'anytime': 1498, 'helps': 1499, 'national': 1500, 'latest': 1501, 'lots': 1502, 'wow': 1503, 'footsteps': 1504, 'front': 1505, 'unfair': 1506, 'screen': 1507, 'understood': 1508, 'breaking': 1509, 'picks': 1510, 'mckenzie': 1511, 'sermon': 1512, 'lifetime': 1513, 'deeper': 1514, 'higher': 1515, \"evrybody's\": 1516, 'disagree': 1517, 'aches': 1518, 'linger': 1519, 'grass': 1520, 'taught': 1521, 'angry': 1522, 'sick': 1523, 'headed': 1524, 'haired': 1525, 'soap': 1526, 'regret': 1527, 'tommy': 1528, 'paul': 1529, 'concept': 1530, 'slumbers': 1531, 'closed': 1532, 'yi': 1533, 'almost': 1534, 'hurting': 1535, 'imagine': 1536, 'imagined': 1537, 'sin': 1538, 'ly': 1539, \"wasn't\": 1540, 'walked': 1541, 'forgive': 1542, 'crossed': 1543, 'frown': 1544, 'mistake': 1545, 'kick': 1546, 'unless': 1547, \"'way\": 1548, 'folks': 1549, 'shook': 1550, 'started': 1551, 'begging': 1552, 'pray': 1553, '1': 1554, '2': 1555, 'stepping': 1556, 'disappointment': 1557, 'wicked': 1558, 'weep': 1559, 'aunt': 1560, 'matches': 1561, \"ol'\": 1562, 'peaches': 1563, 'worth': 1564, 'loted': 1565, 'peasant': 1566, \"foolin'\": 1567, 'hates': 1568, 'realised': 1569, 'happen': 1570, 'vain': 1571, 'learns': 1572, 'hang': 1573, 'tenderly': 1574, 'dry': 1575, 'homing': 1576, 'seemed': 1577, 'spells': 1578, 'begun': 1579, 'easier': 1580, 'streams': 1581, 'hela': 1582, 'heba': 1583, 'heloa': 1584, \"anybody's\": 1585, 'assured': 1586, 'opened': 1587, 'doors': 1588, 'slide': 1589, 'wave': 1590, 'woof': 1591, 'north': 1592, 'ceiling': 1593, 'arrow': 1594, 'piercing': 1595, 'suppose': 1596, 'pepper': 1597, 'freaks': 1598, 'learned': 1599, 'facing': 1600, 'denied': 1601, 'stupid': 1602, 'tuesday': 1603, 'nau': 1604, 'ghty': 1605, 'policemen': 1606, 'showed': 1607, 'hanging': 1608, 'sided': 1609, 'weaving': 1610, 'planned': 1611, 'winds': 1612, 'rivers': 1613, 'shy': 1614, 'laughing': 1615, 'imply': 1616, 'they’ll': 1617, 'tries': 1618, 'nasty': 1619, 'habit': 1620, 'disappearing': 1621, \"diff'rence\": 1622, 'afternoon': 1623, 'sliping': 1624, 'fingers': 1625, 'yawning': 1626, 'wink': 1627, 'putting': 1628, 'joke': 1629, 'brain': 1630, 'weeks': 1631, 'sir': 1632, 'train': 1633, 'wondering': 1634, 'toes': 1635, 'carve': 1636, 'places': 1637, 'affection': 1638, 'expect': 1639, 'floating': 1640, 'cake': 1641, \"who'd\": 1642, 'nobody’d': 1643, 'simple': 1644, \"diff'rent\": 1645, 'ev´rything': 1646, 'felt': 1647, 'began': 1648, 'control': 1649, 'shivering': 1650, 'swallowing': 1651, 'selling': 1652, 'bastards': 1653, 'junk': 1654, 'motorcars': 1655, 'handlebars': 1656, 'bicycles': 1657, 'parachutes': 1658, 'bags': 1659, 'faint': 1660, 'manage': 1661, 'pappie': 1662, 'thanks': 1663, 'cloudy': 1664, 'prairies': 1665, 'prairie': 1666, 'wallflowers': 1667, 'wanting': 1668, 'reaching': 1669, 'asking': 1670, 'dinner': 1671, 'satisfy': 1672, 'picture': 1673, 'kaleidoscope': 1674, 'pies': 1675, 'newspaper': 1676, 'invitation': 1677, 'reservation': 1678, 'dying': 1679, 'sue': 1680, 'class': 1681, 'fifty': 1682, 'mustard': 1683, 'sleeps': 1684, 'shaves': 1685, 'saving': 1686, 'note': 1687, 'waving': 1688, 'homedrops': 1689, 'trickled': 1690, 'legs': 1691, 'police': 1692, 'elephants': 1693, 'memory': 1694, 'grag': 1695, 'village': 1696, 'untrue': 1697, 'paradise': 1698, 'flown': 1699, 'worked': 1700, 'heels': 1701, 'ease': 1702, 'keys': 1703, 'command': 1704, 'singer': 1705, 'twenty': 1706, 'jones': 1707, 'stays': 1708, 'shave': 1709, 'wears': 1710, 'comfort': 1711, 'begged': 1712, 'railman': 1713, 'location': 1714, \"list'ning\": 1715, 'steady': 1716, 'motor': 1717, 'pouring': 1718, 'strange': 1719, 'fireman': 1720, 'shirts': 1721, 'piggy': 1722, 'clutching': 1723, \"weather's\": 1724, 'starts': 1725, 'borrow': 1726, 'straight': 1727, 'reminiscing': 1728, 'sore': 1729, 'plan': 1730, 'somewhere': 1731, 'himself': 1732, 'checked': 1733, \"gideon's\": 1734, 'rival': 1735, 'grin': 1736, 'doc': 1737, 'charlie': 1738, 'chan': 1739, 'simon': 1740, 'smith': 1741, 'sergeant': 1742, 'gunn': 1743, 'ooo': 1744, 'drummon': 1745, 'goodness': 1746, 'wisper': 1747, 'raindrops': 1748, 'spring': 1749, 'layed': 1750, 'greatest': 1751, 'introduce': 1752, 'bathroom': 1753, 'sundays': 1754, 'tuesdays': 1755, 'ourselves': 1756, 'drives': 1757, 'whispers': 1758, 'loneliest': 1759, 'creatures': 1760, 'ugly': 1761, 'duck': 1762, 'surrender': 1763, 'amore': 1764, 'dare': 1765, 'ireland': 1766, 'gal': 1767, 'fellers': 1768, 'boston': 1769, 'philadelphia': 1770, 'texas': 1771, 'frisco': 1772, 'saint': 1773, 'louis': 1774, 'orleans': 1775, 'rainbows': 1776, 'discover': 1777, 'prove': 1778, 'paris': 1779, 'tiger': 1780, 'farther': 1781, 'travels': 1782, 'leads': 1783, 'pool': 1784, 'goddman': 1785, 'genocide': 1786, 'aye': 1787, 'goodbyes': 1788, 'unwise': 1789, 'grade': 1790, 'photograph': 1791, 'lights': 1792, 'stood': 1793, 'stared': 1794, 'lords': 1795, 'dragged': 1796, 'drank': 1797, 'coat': 1798, 'grabbed': 1799, 'bus': 1800, 'seconds': 1801, 'upstairs': 1802, 'smoke': 1803, 'spoke': 1804, 'blackburn': 1805, 'lancashire': 1806, 'albert': 1807, 'hall': 1808, 'rainclouds': 1809, 'papercup': 1810, 'slither': 1811, 'univer': 1812, 'se': 1813, 'pools': 1814, 'drifting': 1815, 'possessing': 1816, 'caressing': 1817, 'images': 1818, 'thoughts': 1819, 'meander': 1820, 'restless': 1821, 'letterbox': 1822, 'tumble': 1823, 'blindly': 1824, 'sounds': 1825, 'laughter': 1826, 'shades': 1827, 'views': 1828, 'inciting': 1829, 'inviting': 1830, 'limitless': 1831, 'undying': 1832, 'suns': 1833, 'beggin': 1834, 'rehearsal': 1835, 'g': 1836, 'pink': 1837, 'orange': 1838, 'sung': 1839, 'shown': 1840, 'prized': 1841, 'possessions': 1842, 'awoken': 1843, 'prison': 1844, 'millions': 1845, 'political': 1846, 'there´s': 1847, 'we´re': 1848, 'hopes': 1849, 'watches': 1850, 'sisters': 1851, 'brothers': 1852, 'reaches': 1853, 'different': 1854, 'races': 1855, 'jailhouse': 1856, 'equality': 1857, 'naking': 1858, 'thin': 1859, 'unhappy': 1860, 'sympathize': 1861, 'faded': 1862, 'towers': 1863, 'forthy': 1864, 'widowed': 1865, 'media': 1866, 'blames': 1867, 'rockefeller': 1868, 'that’s': 1869, 'we’re': 1870, 'justice': 1871, 'suffocation': 1872, 'wathc': 1873, 'now’s': 1874, 'hatred': 1875, 'judgement': 1876, 'mattes': 1877, 'traveled': 1878, 'tuned': 1879, 'flew': 1880, 'miami': 1881, 'beach': 1882, 'dreadful': 1883, 'gee': 1884, 'unpack': 1885, 'disconnect': 1886, 'snow': 1887, 'peaked': 1888, 'mountains': 1889, 'farm': 1890, 'balalaikas': 1891, 'comrad': 1892, 'moved': 1893, 'neighborhood': 1894, 'sits': 1895, 'tacks': 1896, 'teachers': 1897, 'gum': 1898, \"girl's\": 1899, 'worries': 1900, 'poop': 1901, 'hula': 1902, 'hoop': 1903, 'cut': 1904, 'canary': 1905, 'fed': 1906, 'neighbors': 1907, 'cocker': 1908, 'spaniel': 1909, 'laundramat': 1910, \"mama's\": 1911, 'softly': 1912, \"sighin'\": 1913, 'breeze': 1914, \"cryin'\": 1915, 'badge': 1916, 'wander': 1917, 'swans': 1918, 'mabel': 1919, 'wheel': 1920, 'cradle': 1921, 'gene': 1922, 'vincent': 1923, 'sheriff': 1924, 'tex': 1925, 'team': 1926, \"walkin'\": 1927, 'monsters': 1928, 'prayer': 1929, 'age': 1930, 'patient': 1931, 'meantime': 1932, 'happens': 1933, 'sean': 1934, 'streaming': 1935, 'ploys': 1936, 'forty': 1937, 'creating': 1938, 'multiple': 1939, 'fence': 1940, 'settle': 1941, 'blows': 1942, 'trampoline': 1943, 'pablo': 1944, 'fanques': 1945, 'horses': 1946, 'hoops': 1947, 'garters': 1948, 'lastly': 1949, 'hogshead': 1950, 'challenge': 1951, 'celebrated': 1952, 'feat': 1953, 'bishopsgate': 1954, 'flys': 1955, 'messrs': 1956, 'assure': 1957, 'public': 1958, 'none': 1959, 'course': 1960, 'henry': 1961, 'dances': 1962, 'waltz': 1963, 'tricks': 1964, 'demonstrate': 1965, 'summersets': 1966, 'undertake': 1967, 'preparation': 1968, 'splendid': 1969, 'topping': 1970, 'divine': 1971, 'whispering': 1972, 'adore': 1973, 'sunken': 1974, 'fog': 1975, 'l': 1976, 'themselves': 1977, 'policeman': 1978, 'breath': 1979, 'raised': 1980, 'flut': 1981, 'battered': 1982, 'razor': 1983, 'shattered': 1984, 'kicked': 1985, 'thrown': 1986, 'tools': 1987, 'aprision': 1988, 'rule': 1989, 'wardnes': 1990, 'seasons': 1991, 'vision': 1992, 'jumping': 1993, 'highway': 1994, 'cutie': 1995, \"ma'am\": 1996, 'jammed': 1997, 'tap': 1998, 'overcome': 1999, 'pillow': 2000, 'invitations': 2001, 'celebrations': 2002, \"coachin'\": 2003, 'dungeries': 2004, 'already': 2005, \"abc's\": 2006, 'heh': 2007, 'bread': 2008, 'enemies': 2009, 'absolutely': 2010, 'rats': 2011, 'aboard': 2012, 'travel': 2013, 'wherever': 2014, 'centre': 2015, 'circle': 2016, \"temperature's\": 2017, 'rising': 2018, 'aching': 2019, 'goose': 2020, 'pimple': 2021, 'bone': 2022, 'freeze': 2023, 'hours': 2024, 'praying': 2025, 'grooving': 2026, 'eyeball': 2027, 'holly': 2028, 'toejam': 2029, 'football': 2030, 'coca': 2031, 'cola': 2032, 'gumboot': 2033, 'sideboard': 2034, 'spinal': 2035, 'cracker': 2036, 'armchair': 2037, 'disease': 2038, 'coaster': 2039, 'warning': 2040, 'muddy': 2041, 'mojo': 2042, 'filter': 2043, 'sonny': 2044, 'race': 2045, 'collar': 2046, 'itself': 2047, 'hymn': 2048, 'color': 2049, 'marigold': 2050, 'breakfast': 2051, 'parlour': 2052, 'picking': 2053, 'playroom': 2054, 'childrens': 2055, 'holiday': 2056, 'duchess': 2057, 'kircaldy': 2058, 'arriving': 2059, 'duke': 2060, 'problems': 2061, 'bee': 2062, 'twelve': 2063, 'seance': 2064, 'specially': 2065, 'lark': 2066, 'ski': 2067, 'ies': 2068, 'chi': 2069, 'ld': 2070, 'daisy': 2071, 'chain': 2072, 'wilt': 2073, 'flower': 2074, \"spirit's\": 2075, 'v': 2076, 'track': 2077, 'goddess': 2078, 'hog': 2079, 'moondog': 2080, 'stoney': 2081, 'treasure': 2082, 'eternally': 2083, 'complain': 2084, 'pleasing': 2085, 'beyond': 2086, 'compare': 2087, 'zoom': 2088, \"cross'd\": 2089, 'danced': 2090, \"mo'\": 2091, \"searchin'\": 2092, \"darlin'\": 2093, 'imprisoned': 2094, 'touched': 2095, 'lingers': 2096, \"'though\": 2097, \"evr'rybody\": 2098, 'phon': 2099, 'plain': 2100, 'bells': 2101, 'winging': 2102, 'girlfriend': 2103, 'patiently': 2104, 'passed': 2105, 'check': 2106, 'deliver': 2107, 'sooner': 2108, 'awrite': 2109, \"jumpin'\": 2110, 'jockey': 2111, \"temp'rature's\": 2112, \"risin'\": 2113, \"blowin'\": 2114, 'fuse': 2115, \"beatin'\": 2116, \"asingin'\": 2117, 'pneumonia': 2118, 'arthritis': 2119, 'review': 2120, 'trifle': 2121, 'further': 2122, \"mornin'\": 2123, \"givin'\": 2124, \"warnin'\": 2125, 'fiddle': 2126, 'glow': 2127, 'worm': 2128, \"spinnin'\": 2129, 'oughta': 2130, 'badly': 2131, 'madly': 2132, 'wring': 2133, 'split': 2134, 'tantalize': 2135, 'thrilling': 2136, 'em': 2137, 'doris': 2138, 'matt': 2139, 'busby': 2140, \"rock'n'roll\": 2141, 'stroll': 2142, 'fore': 2143, 'bride': 2144, 'helping': 2145, 'succeed': 2146, 'health': 2147, 'lace': 2148, \"powder's\": 2149, 'tomboy': 2150, 'guys': 2151, 'lasts': 2152, 'clock': 2153, 'aticking': 2154, 'mantel': 2155, 'shelf': 2156, 'amoving': 2157, 'doubted': 2158, 'crash': 2159, 'famous': 2160, 'gril': 2161, 'prospects': 2162, 'peanuts': 2163, 'rice': 2164, 'wedding': 2165, 'darning': 2166, 'stocks': 2167, \"re's\": 2168, 'buried': 2169, 'wiping': 2170, 'walks': 2171, 'grave': 2172, 'presses': 2173, 'breast': 2174, 'craks': 2175, 'colourful': 2176, 'kindness': 2177, 'wakes': 2178, 'loner': 2179, \"n't\": 2180, 'tuscon': 2181, 'arizona': 2182, 'california': 2183, 'martin': 2184, 'heel': 2185, 'neck': 2186, 'sweater': 2187, \"teacher's\": 2188, 'cruel': 2189, 'hearing': 2190, 'uptight': 2191, 'sighted': 2192, 'narrow': 2193, 'minded': 2194, 'hypocritics': 2195, 'reading': 2196, 'neurotic': 2197, 'psychotic': 2198, 'pig': 2199, 'politicians': 2200, 'bellied': 2201, 'tricky': 2202, 'dicky': 2203, 'hubbard': 2204, 'pocketful': 2205, 'mope': 2206, 'seeing': 2207, 'lipped': 2208, 'condescending': 2209, 'mommies': 2210, 'chauvinists': 2211, 'scenes': 2212, 'schizophrenic': 2213, 'ego': 2214, 'centric': 2215, 'paranoic': 2216, 'prima': 2217, 'donnas': 2218, \"son't\": 2219, 'promises': 2220, 'acts': 2221, 'pleasu': 2222, 'earn': 2223, 'leasure': 2224, 'food': 2225, \"window's\": 2226, \"money's\": 2227, \"living's\": 2228, 'heartbeat': 2229, 'flesh': 2230, 'bagism': 2231, 'shagism': 2232, 'dragism': 2233, 'madism': 2234, 'ra': 2235, 'gism': 2236, 'tagism': 2237, 'thisism': 2238, 'thatism': 2239, 'ministers': 2240, 'sinisters': 2241, 'banisters': 2242, 'canisters': 2243, 'bishops': 2244, 'fishops': 2245, 'rabbis': 2246, 'popeyes': 2247, 'byes': 2248, 'regulations': 2249, 'integrations': 2250, 'meditations': 2251, 'united': 2252, 'nations': 2253, 'congratulations': 2254, 'timmy': 2255, 'leary': 2256, 'rosemary': 2257, 'smothers': 2258, 'bobby': 2259, 'dylan': 2260, 'cooper': 2261, 'derek': 2262, 'taylor': 2263, 'norman': 2264, 'mailer': 2265, 'ginsberg': 2266, 'goosepimples': 2267, 'mercy': 2268, 'sewed': 2269, 'electric': 2270, 'wire': 2271, 'temperature': 2272, 'ows': 2273, 'bent': 2274, 'backed': 2275, 'fulips': 2276, 'castiron': 2277, 'dovetail': 2278, 'ching': 2279, 'tarot': 2280, 'hitler': 2281, 'kennedy': 2282, 'buddha': 2283, 'mantra': 2284, 'gita': 2285, 'yoga': 2286, 'kings': 2287, 'elvis': 2288, 'zimmerman': 2289, 'beatles': 2290, 'reality': 2291, 'dreamweaver': 2292, 'reborn': 2293, 'awake': 2294, 'blab': 2295, 'burns': 2296, 'shady': 2297, \"how's\": 2298, 'heading': 2299, 'ruin': 2300, 'decide': 2301, 'skirts': 2302, 'flirt': 2303, 'gear': 2304, 'lingered': 2305, 'excite': 2306, 'oogh': 2307, \"ev'ryday\": 2308, 'bees': 2309, 'apologize': 2310, 'resign': 2311, 'diamonmd': 2312, 'talkin': 2313, 'rep': 2314, 'peep': 2315, 'thet': 2316, 'clown': 2317, 'deserve': 2318, 'whim': 2319, 'against': 2320, 'modern': 2321, 'jazz': 2322, 'darn': 2323, 'beauty': 2324, 'melody': 2325, 'symphony': 2326, \"'cross\": 2327, 'tracks': 2328, 'wail': 2329, 'sax': 2330, \"hurrican'\": 2331, 'jokey': 2332, \"drinkin'\": 2333, 'brew': 2334, 'tango': 2335, 'mambo': 2336, 'congo': 2337, 'summernight': 2338, 'beam': 2339, '3': 2340, '4': 2341, '5': 2342, '6': 2343, '7': 2344, '8': 2345, '9': 2346, 'ringo': 2347, 'teen': 2348, 'yake': 2349, 'books': 2350, 'boyfriend': 2351, 'bbbbb': 2352, 'lotta': 2353, 'cares': 2354, \"cummin'\": 2355, 'ducked': 2356, 'alley': 2357, 'puppy': 2358, 'runs': 2359, 'moan': 2360, 'apologise': 2361, 'pleas': 2362, 'ours': 2363, 'wing': 2364, 'soooo': 2365, 'favorite': 2366, 'holds': 2367, 'chick': 2368, 'misses': 2369, 'aquainted': 2370, 'velvet': 2371, 'lizard': 2372, 'pane': 2373, 'multicolored': 2374, 'mirrors': 2375, 'hob': 2376, 'nail': 2377, 'impression': 2378, 'ate': 2379, 'donated': 2380, 'bits': 2381, 'fights': 2382, 'cars': 2383, 'buses': 2384, 'planes': 2385, \"other's\": 2386, 'rough': 2387, 'twinkling': 2388, \"happen'd\": 2389, 'anubody': 2390, 'independence': 2391, 'vanish': 2392, 'haze': 2393, 'insecure': 2394, 'majesty': 2395, 'changes': 2396, 'belly': 2397, 'clod': 2398, 'winter': 2399, 'faces': 2400, 'melting': 2401, 'deny': 2402, 'believing': 2403, 'bullfrog': 2404, 'jack': 2405, 'knife': 2406, 'sweaty': 2407, 'innocence': 2408, 'wigwam': 2409, 'solitude': 2410, \"wha'd'ya\": 2411, \"d'y'know\": 2412, 'shoulders': 2413, 'colder': 2414, 'perform': 2415, 'yourskin': 2416, 'cookie': 2417, 'england': 2418, 'bigtime': 2419, 'position': 2420, 'tragic': 2421, 'became': 2422, 'legend': 2423, 'frantic': 2424, 'atlantic': 2425, 'kinds': 2426, 'kindly': 2427, 'tee': 2428, 'honeymoon': 2429, 'horizon': 2430, 'reelings': 2431, 'straights': 2432, 'jum': 2433, 'muzak': 2434, 'tough': 2435, 'ng': 2436, 'cornflake': 2437, 'van': 2438, 'corporation': 2439, 'teashirt': 2440, 'custard': 2441, 'dripping': 2442, 'dogs': 2443, 'crabalocker': 2444, 'fishwife': 2445, 'pornographic': 2446, 'priestess': 2447, 'kni': 2448, 'ckers': 2449, 'tan': 2450, 'eng': 2451, 'lish': 2452, 'expert': 2453, 'texpert': 2454, 'choking': 2455, 'smokers': 2456, 'laughs': 2457, 'hahaha': 2458, 'sty': 2459, 'snied': 2460, 'semolina': 2461, 'pilchard': 2462, 'climbing': 2463, 'eiffel': 2464, 'tower': 2465, 'elementry': 2466, 'penguin': 2467, 'kicking': 2468, 'edgar': 2469, 'allen': 2470, 'poe': 2471, \"nobody's\": 2472, 'cock': 2473, 'occupied': 2474, 'junkies': 2475, 'religion': 2476, 'cocaine': 2477, 'awww': 2478, 'grumbles': 2479, 'fusses': 2480, 'treats': 2481, 'streets': 2482, \"woman's\": 2483, 'search': 2484, 'crave': 2485, 'sinking': 2486, \"life's\": 2487, 'nervous': 2488, \"one's\": 2489, 'flowing': 2490, 'freely': 2491, 'games': 2492, 'unkind': 2493, 'confusing': 2494, 'mattered': 2495, 'endear': 2496, 'sailor': 2497, 'failure': 2498, 'lawyer': 2499, 'beggar': 2500, 'thief': 2501, 'churchman': 2502, 'hid': 2503, 'later': 2504, \"love's\": 2505, 'chip': 2506, 'stare': 2507, 'ay': 2508, 'throws': 2509, 'moar': 2510, \"an'\": 2511, 'hug': 2512, 'discovered': 2513, 'soothing': 2514, \"aren't\": 2515, 'learnt': 2516, \"stranger's\": 2517, 'wires': 2518, \"communication's\": 2519, 'valley': 2520, 'indecision': 2521, 'remind': 2522, 'stuff': 2523, 'waht': 2524, 'bandaid': 2525, 'carrey': 2526, 'score': 2527, 'scored': 2528, 'jive': 2529, 'speed': 2530, 'staring': 2531, 'slept': 2532, 'blink': 2533, 'cigarette': 2534, 'curse': 2535, 'walter': 2536, 'raleigh': 2537, 'git': 2538, 'shivers': 2539, 'wet': 2540, 'socks': 2541, 'aware': 2542, 'missed': 2543, 'wew': 2544, 'dont': 2545, 'especially': 2546, 'remain': 2547, 'moments': 2548, 'recall': 2549, 'compares': 2550, \"mem'ries\": 2551, 'meaning': 2552, 'others': 2553, 'caused': 2554, 'victim': 2555, 'piece': 2556, 'dries': 2557, 'butterflies': 2558, 'i’d': 2559, 'aloud': 2560, \"i'\": 2561, 'm': 2562, 'beating': 2563, 'inscure': 2564, 'ain’t': 2565, 'stir': 2566, 'won’t': 2567, 'stair': 2568, 'shooting': 2569, 'gooks': 2570, 'vietnam': 2571, 'cia': 2572, 'he’d': 2573, 'they’d': 2574, 'breatthing': 2575, 'gottta': 2576, 'freee': 2577, 'jailed': 2578, 'representing': 2579, 'clutches': 2580, 'oof': 2581, 'lid': 2582, 'meaningless': 2583, 'seashell': 2584, 'shimmering': 2585, 'glimmering': 2586, 'silent': 2587, 'draw': 2588, 'darts': 2589, 'terror': 2590, 'childhood': 2591, 'bell': 2592, 'rent': 2593, 'arrives': 2594, 'suitcase': 2595, 'creep': 2596, 'nun': 2597, \"monday's\": 2598, 'bootlace': 2599, 'ending': 2600, \"wedn'sday\": 2601, 'papers': 2602, 'thursday': 2603, 'stockings': 2604, 'mending': 2605, 'mammie': 2606, 'scold': 2607, 'sugar': 2608, 'darkness': 2609, 'parted': 2610, 'lumps': 2611, 'throat': 2612, 'isle': 2613, 'sweetly': 2614, 'omit': 2615, 'rocker': 2616, \"it'll\": 2617, 'wasting': 2618, 'misplace': 2619, 'looooooooved': 2620, 'tun': 2621, 'alife': 2622, 'bought': 2623, \"who'll\": 2624, 'screw': 2625, 'sins': 2626, 'tow': 2627, 'parking': 2628, 'glimpse': 2629, 'cap': 2630, 'older': 2631, 'military': 2632, 'inquire': 2633, 'discreetly': 2634, 'paid': 2635, 'sofa': 2636, 'river': 2637, 'tangerine': 2638, 'marmelade': 2639, 'cellophane': 2640, 'towering': 2641, 'bridge': 2642, 'fountain': 2643, 'marshmallow': 2644, 'drift': 2645, 'incredibly': 2646, 'taxis': 2647, 'climb': 2648, 'plasticine': 2649, 'porters': 2650, 'ties': 2651, 'turnstile': 2652, 'lisle': 2653, 'mower': 2654, 'robbing': 2655, 'boun': 2656, 'der': 2657, \"robbin'\": 2658, 'liverpool': 2659, 'returned': 2660, 'pound': 2661, 'uo': 2662, 'satisfaction': 2663, 'aaaaah': 2664, 'conversation': 2665, 'inspiration': 2666, 'joan': 2667, 'quizzical': 2668, 'studied': 2669, 'paraphysical': 2670, 'science': 2671, 'testube': 2672, 'edison': 2673, 'majoring': 2674, 'medicine': 2675, 'jo': 2676, 'annoyed': 2677, 'avoid': 2678, 'unpleasant': 2679, 'sce': 2680, 'ene': 2681, 'max': 2682, 'creeps': 2683, 'testimonial': 2684, 'rose': 2685, 'valerie': 2686, 'screaming': 2687, 'gallery': 2688, 'noise': 2689, 'bob': 2690, 'nose': 2691, 'getter': 2692, 'shouts': 2693, 'obscene': 2694, 'distance': 2695, 'placed': 2696, 'add': 2697, 'strolling': 2698, 'hairy': 2699, 'country': 2700, 'waters': 2701, 'flies': 2702, 'field': 2703, 'swaying': 2704, 'daisies': 2705, 'explain': 2706, 'york': 2707, 'jerry': 2708, 'guitar': 2709, 'marijuana': 2710, 'peel': 2711, 'sangs': 2712, 'pope': 2713, 'smokes': 2714, 'shoved': 2715, 'singin': 2716, 'max’s': 2717, 'nitty': 2718, 'gritty': 2719, 'spread': 2720, 'plastic': 2721, 'funky': 2722, 'boogie': 2723, 'tutti': 2724, 'frutti': 2725, 'sally’s': 2726, 'preachman': 2727, 'god’s': 2728, 'herring': 2729, 'staten': 2730, 'island': 2731, 'ferry': 2732, 'telly': 2733, 'fillmore': 2734, 'apollo': 2735, 'freedom': 2736, 'image': 2737, 'cycling': 2738, 'bug': 2739, 'hustle': 2740, 'ud': 2741, 'decided': 2742, 'statue': 2743, 'liberty': 2744, 'ass': 2745, 'anywhere': 2746, 'sat': 2747, 'rug': 2748, 'biding': 2749, 'talked': 2750, 'crawled': 2751, 'lit': 2752, 'vest': 2753, 'ageing': 2754, 'handing': 2755, 'writ': 2756, 'underfed': 2757, 'freak': 2758, 'sikh': 2759, 'leading': 2760, 'astray': 2761, 'mandalay': 2762, 'apple': 2763, 'cart': 2764, 'misled': 2765, \"shakin'\": 2766, \"there'd\": 2767, 'gang': 2768, 'joes': 2769, 'grab': 2770, 'squeeze': 2771, 'tease': 2772, 'spends': 2773, 'point': 2774, 'view': 2775, 'lends': 2776, 'barrow': 2777, 'trolley': 2778, \"jeweller's\": 2779, 'carat': 2780, 'dib': 2781, 'storm': 2782, 'hideaway': 2783, 'resting': 2784, 'cave': 2785, 'swim': 2786, 'coral': 2787, 'safe': 2788, 'replacing': 2789, 'thoughtless': 2790, 'escaping': 2791, 'worrying': 2792, 'imperfect': 2793, 'reject': 2794, 'faster': 2795, 'weather': 2796, 'lip': 2797, 'gald': 2798, '909': 2799, 'ev´rything´s': 2800, 'fare': 2801, 'correct': 2802, 'madam': 2803, 'wri': 2804, 'te': 2805, 'based': 2806, 'novel': 2807, 'lear': 2808, 'clinging': 2809, 'daily': 2810, 'pages': 2811, 'overnight': 2812, 'aldebaran': 2813, 'liquids': 2814, 'drums': 2815, 'arm': 2816, 'board': 2817, 'showing': 2818, 'photographs': 2819, 'pleasure': 2820, 'mack': 2821, 'hourglass': 2822, 'pocket': 2823, 'portrait': 2824, 'likes': 2825, 'engine': 2826, 'fish': 2827, 'summer': 2828, 'shelter': 2829, 'roundabout': 2830, 'nurse': 2831, 'poppies': 2832, 'tray': 2833, \"tho'\": 2834, 'customer': 2835, 'trim': 2836, 'rushes': 2837, 'crawling': 2838, 'worse': 2839, 'starched': 2840, 'stirring': 2841, 'styes': 2842, 'backing': 2843, 'lacking': 2844, \"need's\": 2845, 'damn': 2846, 'whacking': 2847, 'forks': 2848, 'knives': 2849, 'bacon': 2850, 'goodlooking': 2851, 'dose': 2852, 'jackboots': 2853, 'kilt': 2854, 'killer': 2855, 'diller': 2856, 'hilt': 2857, 'attractively': 2858, 'heads': 2859, 'shi': 2860, 'nes': 2861, 'sip': 2862, 'lemonade': 2863, \"ev'rything's\": 2864, 'rains': 2865, 'schemes': 2866, 'forgotten': 2867, 'sailed': 2868, 'dawning': 2869, 'hero': 2870, 'handed': 2871, 'movie': 2872, 'stardom': 2873, 'fifth': 2874, 'november': 2875, 'junkman': 2876, 'sold': 2877, 'cheater': 2878, 'mistreater': 2879, 'evolution': 2880, 'destruction': 2881, 'solution': 2882, 'contribution': 2883, 'minds': 2884, 'constitution': 2885, 'institution': 2886, 'carrying': 2887, 'chairman': 2888, 'mao': 2889, 'anyhow': 2890, 'rip': 2891, 'hills': 2892, 'dakota': 2893, 'booked': 2894, 'saloon': 2895, 'equipped': 2896, 'stealing': 2897, 'magill': 2898, \"call'd\": 2899, 'lil': 2900, 'nancy': 2901, 'dan': 2902, 'hoedown': 2903, 'burst': 2904, 'grinning': 2905, \"'danny\": 2906, 'daniel': 2907, 'drew': 2908, 'collapsed': 2909, 'stinking': 2910, 'gin': 2911, 'proceeded': 2912, 'match': 2913, 'able': 2914, 'gideon': 2915, \"rocky's\": 2916, 'revival': 2917, 'rac': 2918, 'coon': 2919, 'toe': 2920, 'determined': 2921, 'aah': 2922, 'shoo': 2923, 'doo': 2924, \"p'doo\": 2925, \"tumblin'\": 2926, 'oooh': 2927, 'mmm': 2928, 'owned': 2929, 'lighten': 2930, 'raise': 2931, \"pepper'lonely\": 2932, 'certainly': 2933, 'audience': 2934, \"singer's\": 2935, 'billy': 2936, 'shears': 2937, 'protected': 2938, 'spoon': 2939, 'sucks': 2940, 'thumb': 2941, 'banks': 2942, 'lagoon': 2943, 'fifteen': 2944, 'clubs': 2945, 'department': 2946, 'rob': 2947, 'wednesday': 2948, 'silently': 2949, 'closing': 2950, 'bedroom': 2951, 'hoped': 2952, 'handkerchief': 2953, 'quietly': 2954, 'backdoor': 2955, 'steppping': 2956, 'sacrificed': 2957, 'snores': 2958, 'dressing': 2959, 'gown': 2960, 'stairs': 2961, 'cries': 2962, 'husband': 2963, 'thoughtlessly': 2964, 'struggled': 2965, 'appointment': 2966, 'trade': 2967, 'laways': 2968, 'tail': 2969, 'cheeks': 2970, 'rosie': 2971, 'nosey': 2972, \"clappin'\": 2973, \"poppin'\": 2974, 'sway': 2975, 'throw': 2976, 'battlefield': 2977, 'weapons': 2978, 'using': 2979, 'truest': 2980, 'squeezing': 2981, 'desire': 2982, \"sippin'\": 2983, 'moves': 2984, 'attracts': 2985, 'woos': 2986, 'misunderstanding': 2987, 'thee': 2988, 'quando': 2989, 'para': 2990, 'feliece': 2991, 'corazon': 2992, 'mundo': 2993, 'pararazzi': 2994, 'chicka': 2995, 'ferdy': 2996, 'parasol': 2997, 'cuesto': 2998, 'obrigado': 2999, 'tanta': 3000, 'carousel': 3001, 'crys': 3002, 'thirteen': 3003, 'marty': 3004, 'martyrs': 3005, 'derry': 3006, 'nailed': 3007, 'coffin': 3008, 'lidds': 3009, 'claim': 3010, 'majority': 3011, 'you’re': 3012, 'minority': 3013, 'emerald': 3014, 'asle': 3015, 'stormont': 3016, 'bans': 3017, 'marchers': 3018, 'they’ve': 3019, 'internment': 3020, 'mother’s': 3021, 'anglo': 3022, 'scotties': 3023, 'colonize': 3024, 'union': 3025, 'jacks': 3026, 'ransom': 3027, 'concentration': 3028, 'camps': 3029, 'falls': 3030, 'roads': 3031, 'repatriate': 3032, 'britain': 3033, 'rome': 3034, 'knocks': 3035, 'lands': 3036, 'shame': 3037, 'cools': 3038, 'claimed': 3039, 'framed': 3040, 'autopraphs': 3041, 'excited': 3042, 'grown': 3043, 'lipstick': 3044, 'sporting': 3045, 'heeled': 3046, 'trend': 3047, 'become': 3048, 'hurts': 3049, 'surround': 3050, 'nineteen': 3051, 'percent': 3052, 'thankful': 3053, 'seat': 3054, 'wilson': 3055, 'heath': 3056, 'advice': 3057, 'beware': 3058, 'pennies': 3059, 'tales': 3060, 'dad': 3061, 'ta': 3062, 'herself': 3063, 'partners': 3064, 'apply': 3065, 'suicide': 3066, 'turtle': 3067, 'doving': 3068, 'cupid': 3069, 'dart': 3070, 'boldly': 3071, 'bothered': 3072, 'ballad': 3073, 'dock': 3074, 'southampton': 3075, 'holland': 3076, 'france': 3077, 'mac': 3078, 'plane': 3079, 'mooning': 3080, 'seine': 3081, 'gibraltar': 3082, 'spain': 3083, 'amsterdam': 3084, 'hilton': 3085, 'beds': 3086, 'newspapers': 3087, \"what're\": 3088, 'rainy': 3089, 'charity': 3090, 'lighting': 3091, 'vienna': 3092, 'eating': 3093, 'choclate': 3094, 'gurus': 3095, 'acorns': 3096, 'tied': 3097, 'sack': 3098, 'press': 3099, 'success': 3100, 'continuing': 3101, 'hunting': 3102, 'elephant': 3103, 'accidents': 3104, 'mom': 3105, 'american': 3106, 'bullet': 3107, 'saxon': 3108, 'jungle': 3109, 'mighty': 3110, 'elephnats': 3111, 'captain': 3112, 'marvel': 3113, 'zapped': 3114, 'fierce': 3115, 'butted': 3116, 'equal': 3117, 'foolish': 3118, 'perfectly': 3119, 'hears': 3120, 'appears': 3121, 'listens': 3122, 'inner': 3123, 'wild': 3124, 'windy': 3125, 'washed': 3126, 'manytimes': 3127, 'manyways': 3128, 'shold': 3129, 'torture': 3130, 'hunger': 3131, 'beutty': 3132, 'raped': 3133, 'british': 3134, 'brigands': 3135, 'there’d': 3136, 'aharmock': 3137, 'morn': 3138, 'divided': 3139, 'glory': 3140, 'poets': 3141, 'auld': 3142, 'eireland': 3143, 'dew': 3144, 'galeway': 3145, 'leprechauns': 3146, 'beone': 3147, 'blarney': 3148, 'ira': 3149, 'commit': 3150, 'you’s': 3151, 'woah': 3152}\n","3153\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"R6IuWOmyTen5","executionInfo":{"status":"ok","timestamp":1601647355942,"user_tz":-480,"elapsed":1904,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"00f1afbf-c757-49c7-9af9-52133d491970","colab":{"base_uri":"https://localhost:8080/","height":145}},"source":["input_sequences = []\n","for line in corpus:\n","\ttoken_list = tokenizer.texts_to_sequences([line])[0]\n","\tfor i in range(1, len(token_list)):\n","\t\t# Generate n-gram sequences\n","\t\t# INSERT YOUR CODE HERE\n","\t\tn_gram_sequence = token_list[:i+1]\n","\t\tinput_sequences.append(n_gram_sequence)\n","\n","# max of input sequences\n","max_sequence_len = max([len(x) for x in input_sequences])\n","# pad sequences\n","input_sequences = np.array(pad_sequences(input_sequences, maxlen=max_sequence_len, padding='pre'))\n","\n","print(input_sequences)\n","\n","# create predictors and label\n","predictors, labels = input_sequences[:,:-1],input_sequences[:,-1]\n","label = tf.keras.utils.to_categorical(labels, num_classes=total_words)"],"execution_count":28,"outputs":[{"output_type":"stream","text":["[[  0   0   0 ...   0   7  63]\n"," [  0   0   0 ...   7  63  10]\n"," [  0   0   0 ...  63  10   3]\n"," ...\n"," [  0   0   0 ...   3 530  54]\n"," [  0   0   0 ... 530  54 531]\n"," [  0   0   0 ...  54 531  10]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Z2JGdKlSTi67","executionInfo":{"status":"ok","timestamp":1601647395748,"user_tz":-480,"elapsed":1372,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"eceb2ee2-4d5b-4836-a803-52ce20cba569","colab":{"base_uri":"https://localhost:8080/","height":272}},"source":["# Build your own RNN model with LSTM or GRU layers\n","# INSERT YOUR CODE HERE\n","model = Sequential()\n","model.add(Embedding(total_words, 64, input_length=max_sequence_len-1))\n","model.add(Bidirectional(LSTM(64)))\n","model.add(Dense(total_words, activation='softmax'))\n","model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n","\n","model.summary()"],"execution_count":29,"outputs":[{"output_type":"stream","text":["Model: \"sequential_4\"\n","_________________________________________________________________\n","Layer (type)                 Output Shape              Param #   \n","=================================================================\n","embedding_4 (Embedding)      (None, 58, 64)            201792    \n","_________________________________________________________________\n","bidirectional (Bidirectional (None, 128)               66048     \n","_________________________________________________________________\n","dense_4 (Dense)              (None, 3153)              406737    \n","=================================================================\n","Total params: 674,577\n","Trainable params: 674,577\n","Non-trainable params: 0\n","_________________________________________________________________\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"8dz4f0eXqgUD"},"source":["For saving your time, just train 20 epochs (each epoch is around 20 seconds) and test the result."]},{"cell_type":"code","metadata":{"id":"Bk7Kf4eh0qzc","executionInfo":{"status":"ok","timestamp":1601647949246,"user_tz":-480,"elapsed":496829,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"c3347270-e035-4e7d-c1f9-2ccfbccbcda3","colab":{"base_uri":"https://localhost:8080/","height":745}},"source":["# train your model and store it in history\n","history = model.fit(predictors, label, epochs=20, verbose=1)"],"execution_count":31,"outputs":[{"output_type":"stream","text":["Epoch 1/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 6.0493 - accuracy: 0.0510\n","Epoch 2/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 5.3878 - accuracy: 0.0849\n","Epoch 3/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 4.9067 - accuracy: 0.1372\n","Epoch 4/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 4.4968 - accuracy: 0.1820\n","Epoch 5/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 4.1621 - accuracy: 0.2222\n","Epoch 6/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 3.8790 - accuracy: 0.2594\n","Epoch 7/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 3.6316 - accuracy: 0.2935\n","Epoch 8/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 3.4168 - accuracy: 0.3220\n","Epoch 9/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 3.2280 - accuracy: 0.3511\n","Epoch 10/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 3.0616 - accuracy: 0.3777\n","Epoch 11/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 2.9127 - accuracy: 0.4017\n","Epoch 12/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 2.7820 - accuracy: 0.4232\n","Epoch 13/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 2.6597 - accuracy: 0.4430\n","Epoch 14/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 2.5518 - accuracy: 0.4619\n","Epoch 15/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 2.4535 - accuracy: 0.4811\n","Epoch 16/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 2.3641 - accuracy: 0.4970\n","Epoch 17/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 2.2823 - accuracy: 0.5107\n","Epoch 18/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 2.2075 - accuracy: 0.5254\n","Epoch 19/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 2.1391 - accuracy: 0.5376\n","Epoch 20/20\n","1290/1290 [==============================] - 25s 19ms/step - loss: 2.0763 - accuracy: 0.5504\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"DIUy3tb26Fk6"},"source":["### ***Task 4: Our Accuracy Graph over Epochs***"]},{"cell_type":"code","metadata":{"id":"0usM9leDWYj0","executionInfo":{"status":"ok","timestamp":1601648061218,"user_tz":-480,"elapsed":1157,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"7b72501e-7f67-42f0-abde-d52e33b69c18","colab":{"base_uri":"https://localhost:8080/","height":279}},"source":["import matplotlib.pyplot as plt\n","\n","# Use pyplot to plot a graph with x as epoch number, y as accuracy\n","def plot_graphs(history, string):\n","  #INSERT YOUR CODE HERE\n","  plt.plot(history.history[string])\n","  plt.xlabel(\"Epochs\")\n","  plt.ylabel(string)\n","  plt.show()\n","\n","plot_graphs(history, 'accuracy')"],"execution_count":32,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhU5dnH8e9NWMK+7xDZEZBFDIj7UlTcl2oFiyuKWq22vrbV19Za27612tq6oC2ilbqBtqioUARcW2UJ+w4BWRJCWAOEJZDkfv+YoU1jAoPk5Exmfp/rypWzPJO5czIzv5zleY65OyIikryqhV2AiIiES0EgIpLkFAQiIklOQSAikuQUBCIiSa562AUcrWbNmnmHDh3CLkNEpEqZM2fOVndvXta6KhcEHTp0ICMjI+wyRESqFDNbV946HRoSEUlyCgIRkSSnIBARSXIKAhGRJKcgEBFJcgoCEZEkpyAQEUlyCgIRkTjm7izduIs/TF3J8k27AnmOKtehTEQk0RUVO3PX72DK4k1MWbqJDdv3YQbN6tfi+FYNKvz5FAQiInGgoLCIL1Zv48Mlm5i6NJet+QeokWKc1qUZ3zu7C4N7tKR5/VqBPLeCQEQkJPkFhXy8fDNTlmzikxVbyC8opG7NFM4+vgUX9GrFOd2bUz+1RuB1KAhERCrR1vwCpi3NZcqSTfwrcxsHioppWrcmF/duzQUntOTUzs1IrZFSqTUpCEREArZu2x6mLs3lwyW5ZKzbTrFDu8a1uf6U4zi/Z0vSOzQhpZqFVp+CQESkghUVO/M37GDq0s1MX5bLqs35ABzfqj53n9uVC3q1pGfrBpiF9+FfkoJARKQC7Cko5PNVW5m2LJePl29m254DVK9mDOzYhGED0xjcoyVpTeuEXWaZFAQiIt9Qzs59TFsW+a//i9XbOFBYTIPU6pxzfAu+1aMlZ3VrTsPawZ/sPVYKAhGRGLk7SzbuYurSXKYvz2VxdqSD13FN63D9oOP4Vo8WDOjQhBopVauvroJAROQw3J2MdTt4d34205ZuZtOu/ZhB/7TG/GTI8Qzu0YIuLerFzfH+b0JBICJShtVb8nlnXjZvz8sma8c+atdI4cxuzfifHt045/gWNKsXTOeuMCgIRESituYX8P6Cjbw9L5sFWTupZnBal2bcd143LujVirq1EvMjMzF/KxGRGO0/WMSHS3N5Z142n67cQlGx07N1Ax66qAeX9WtDywapYZcYOAWBiCSd4mJnxpptvD0vm8mLN5FfUEjrhqncdkYnrjyxLd1b1Q+7xEqlIBCRpLFi024mzMti4vyN5OzcT71a1bnwhFZceWJbTu7UNNTevWFSEIhIQlu/bS/vL9rIewtyWJazi5RqxlndmvO/F/VgcI+W1K5ZueP6xCMFgYgknKwde5m0KIf3F+awMGsnAP3aN+KRS3tySd82CXXFT0UINAjMbAjwFJACjHH3x0qtvwl4AsiOLnrW3ccEWZOIJKacnfuYtGgT7y/cyLz1eQD0adeQBy88not6t6Z9k/gc3iEeBBYEZpYCjALOA7KA2WY20d2Xlmo63t3vDqoOEUlcm3fvZ3L0w3/22h0A9GzdgB9d0J1L+rTmuKZ1Q66waghyj2AgkOnuawDMbBxwOVA6CEREYrYtv4DJiyMf/jO/2o47dG9Zn/85rxsX92lNp+b1wi6xygkyCNoCG0rMZwEnl9Hu22Z2JrAS+KG7byjdwMxGAiMB0tLSAihVROLZ7v0HmbQoh/cW5PDlmm0UFTudm9flnnO7ckmf1nRtmVyXe1a0sE8Wvwe84e4FZnY7MBY4t3Qjdx8NjAZIT0/3yi1RRMLg7szbkMe4Wet5b0EO+w4W0aFpHe48qzOX9G1N95b1q/T4PvEkyCDIBtqXmG/Hf04KA+Du20rMjgEeD7AeEakC8vYeYMLcbMbP3sCK3N3UqZnC5f3acO2A9vRr30gf/gEIMghmA13NrCORABgKXFeygZm1dvec6OxlwLIA6xGROOXuzFiznXGz1zN58SYOFBbTt11DfnNVby7t24Z6CTrGT7wIbOu6e6GZ3Q1MIXL56EvuvsTMHgUy3H0icI+ZXQYUAtuBm4KqR0Tiz5bdBfxtThbjZ69n7ba9NEitzrAB7bl2QBo92zQIu7ykYe5V65B7enq6Z2RkhF2GiHxDRcXOZ6u2MH7WBqYty6Ww2BnYoQlDB7bnot6tSa2hnr5BMLM57p5e1jrtb4lIpdiYt483MzbwVkYW2Xn7aFK3Jjef1oFrB6TRpYUu+QyTgkBEArUsZxfPf7Ka9xdupNjhjK7NIuP89GxBrer67z8eKAhEJBAZa7fz3Cer+Wj5ZurWTGHE6R254ZQOGuohDikIRKTCuDufrtzCcx+vZtba7TSuU4P7zuvGDaccR6M6NcMuT8qhIBCRY1ZU7ExenMPzn6xmycZdtG6YysOX9GTowPbUqamPmXinv5CIfGMFhUW8PTebP3+2hq+27qFTs7o8fnUfrujXlprVq4VdnsRIQSAiR21PQSFvzFrPmM+/YtOu/ZzQtgHPfbc/F/RqlbR3+arKFAQiErO8vQd4+Yu1vPzFWvL2HmRQpyY8fnUfzujaTEM/VGEKAhE5ok079zPm8zW8Pms9ew8UMbhHS753Tmf6pzUOuzSpAAoCESlX5uZ8Rn+2mrfnZVPscFnfNtxxVme6t9Kwz4lEQSAiXzNv/Q7+9OlqPlyaS63q1bhuYBq3ntFJfQASlIJARIBIH4DPVm3l+U8ymbFmOw1r1+D753ThxlM70FQ3e09oCgKRJFdYVMykxZv40yerWZqzi1YNUvnpxT0YNjCNuhr+OSnoryySpPYfLOKtOVm88Nka1m/fS+fm6gOQrBQEIklm576DvDpjHX/511dszT9Av/aNeOjiHpzXoyXV1AcgKSkIRJJE7q79vPjPr3h95nryCwo5q1tz7jy7Myd3bKI+AElOQSCS4LJ27OX5T1bzVkYWhcXFXNKnDbef1YlebRqGXZrECQWBSIJau3UPz32SyYS52VQz4+r0dtxxZmfSmuoSUPlvCgKRBJO5OZ9RH2fy7vxsaqRUY/ig47j9rE60blg77NIkTikIRBLEik27eeajVXywKIfU6pEbwdx2Zida1E8NuzSJcwoCkSpucfZOnvloFVOW5FK3Zgp3ntWZEad3VCcwiZmCQKSKmr8hj2emr2L68s3UT63OPd/qyi2nddCdwOSoKQhEqpjZa7fz9PRVfL5qK43q1OD+87txw6kdaJBaI+zSpIpSEIhUEV+s3soz0zP5cs02mtatyQMXHs/wQcdRT8NAyDHSK0gkzi3YkMfjU5bzr8xttKhfi59d0pPrBqZRu2ZK2KVJglAQiMSpzM35/P7DFUxevImmdWvy8CU9ue7kNFJrKACkYikIROLMxrx9PDVtFW/N2UDtGin8cHA3RpzRUYeAJDB6ZYnEiR17DvDcJ5mM/XIdONx0akfuOqezLgOVwCkIREK2p6CQl/75FaM/W8OeA4Vc1b8dPxjclXaNNRSEVA4FgUhIDhQWM272ep6ensnW/ALO79mS+y/oTreWuh+wVC4FgUglKy52Ji7YyO+nrmDD9n2c3LEJo284if5pjcMuTZJUoEFgZkOAp4AUYIy7P1ZOu28DfwMGuHtGkDWJhMXd+XjFZh7/xwqWb9pNz9YNGHtLb87s2kz3A5BQBRYEZpYCjALOA7KA2WY20d2XlmpXH7gXmBlULSJhW5iVx6/eX8astds5rmkdnh52Ipf0bq07gklcCHKPYCCQ6e5rAMxsHHA5sLRUu18CvwV+FGAtIqHYvucAT0xZwbjZ62latxa/uuIErh3QnhopuiewxI8gg6AtsKHEfBZwcskGZtYfaO/uH5hZuUFgZiOBkQBpaWkBlCpSsYqKnddnruN3H64kv6CQEad15J7BXTUekMSl0E4Wm1k14EngpiO1dffRwGiA9PR0D7YykWOTsXY7D7+7hKU5uzilU1N+cXkvXQkkcS3IIMgG2peYbxdddkh94ATgk+iJslbARDO7TCeMpSravGs/j01ezoR52bRumMqz153Ixb1b60SwxL0gg2A20NXMOhIJgKHAdYdWuvtOoNmheTP7BLhfISBVzcGiYsZ+sZY/TlvFgcJi7jqnM3ed04U6NXV1tlQNgb1S3b3QzO4GphC5fPQld19iZo8CGe4+MajnFqks/8rcys8nLiFzcz5nd2/Ozy/tRcdmdcMuS+SoBPovi7tPAiaVWvZwOW3PDrIWkYqUnbeP//tgGR8syqF9k9qMuSGdb/VoocNAUiVp31XkKOw/WMSYz9cw6uPVFLtz33ndGHlmJw0NLVWagkAkRh8tz+UX7y1l3ba9DOnViocu7kH7JhoYTqo+BYHIEWzNL+Dn7y7hg0U5dGpel1dGDOSMrs3DLkukwigIRMrhHhkc7pGJS9hTUMT953dj5JmdqVldvYIlsSgIRMqQu2s/D729mGnLcunXvhFPXN2HruoUJglKQSBSgrvz1pwsfvn+Ug4UFvPQRT245fSOpGhwOElgCgKRqOy8fTw4YRGfrdzCwA5N+O3VfdQnQJKCgkCSXnGx8/qs9fxm0jIcePTyXgw/+TgNES1JQ0EgSW3dtj385O8LmbFmO6d3acZvruqtS0Il6SgIJCkVFTtjv1jLE1NWUL2a8dhVvbl2QHv1DJakpCCQpJO5OZ+f/H0hc9bt4Jzuzfm/q3rTumHtsMsSCY2CQJJGYVExL3z+FX+YtpLaNVL4w7V9uaJfW+0FSNJTEEhSyNy8m/veXMDCrJ0M6dWKR6/oRYv6qWGXJRIXYgoCM5sAvAhMdvfiYEsSqTjuzhuzNvDo+0uoU7M6o67rz8V9WoddlkhciXWP4DngZuBpM3sL+Iu7rwiuLJFjt2PPAR6YsJApS3I5o2szfn9NX1o00F6ASGkxBYG7TwOmmVlDYFh0egPwAvCqux8MsEaRo/bF6q3cN34B2/YU8NBFPRhxekf1CxApR8znCMysKTAcuB6YB7wGnA7cCJwdRHEiR+tgUTF/mLqS5z9dTcemdRlz42mc0LZh2GWJxLVYzxG8DXQHXgEudfec6KrxZqZ7DEtcWLdtD/eMm8+CDXkMHdCehy/tqfsGi8Qg1nfJ0+7+cVkr3D29AusROWruztvzsvnZO4tJqWY8993+XNRbJ4RFYhVrEPQ0s3nungdgZo2BYe7+XHCliRzZrv0H+dk7i3l3/kYGdmzCH6/tR5tG6hwmcjRivcPGbYdCAMDddwC3BVOSSGzmrNvBRU99zvsLc7j//G68cdsghYDINxDrHkGKmZm7O4CZpQA1gytLpHxFxc6ojzN5avoq2jRK5a07TqF/WuOwyxKpsmINgn8QOTH85+j87dFlIpUqO28fPxw3n1lrt3NFvzb88ooTqJ9aI+yyRKq0WIPgJ0Q+/O+Mzk8FxgRSkUg5PliYw4MTFlLs8Idr+3Llie3CLkkkIcTaoawYeD76JVKp8gsK+cXEJbw1J4t+7Rvx1NB+HNdUdw4TqSix9iPoCvwG6An8u4++u3cKqC4RAOas284Pxy8ga8de7j6nC/cO7kqNlFivcRCRWMR6aOgvwM+BPwDnEBl3SO9GCczBomKe+SiTZz9aRZtGtXnz9lNI79Ak7LJEElKsQVDb3adHrxxaBzxiZnOAhwOsTZLUV1v38MPx85m/IY9v92/HI5f11AlhkQDFGgQFZlYNWGVmdwPZQL3gypJk5O6Mn72BR99fSo2UahoyWqSSxBoE9wJ1gHuAXxI5PHRjUEVJ8tm+5wAP/H0hHy7N5bQuTfndNX11+0iRSnLEIIh2HrvW3e8H8omcH4iJmQ0BngJSgDHu/lip9XcAdwFF0Z890t2Xxl6+JIJPVmzmR39byM69B/npxT245TQNGS1SmY4YBO5eZGanH+0PjgbIKOA8IAuYbWYTS33Qv+7uf4q2vwx4EhhytM8lVdP+g0U8Nnk5L3+xlu4t6/PXWwbSo3WDsMsSSTqxHhqaZ2YTgbeAPYcWuvuEwzxmIJDp7msAzGwccDnw7yBw910l2tcFPMZ6pIpbsnEn946bT+bmfG45rSM/HtKd1BopYZclkpRiDYJUYBtwbollDhwuCNoCG0rMZwEnl25kZncB9xEZu+jc0uujbUYCIwHS0tJiLFniUXGx88Lna/jdhytoXKcmr4wYyBldm4ddlkhSi7VnccznBY6Wu48CRpnZdcBPKeMktLuPBkYDpKena6+hitqYt4/73pzPjDXbGdKrFb+5qjeN62rsQpGwxdqz+C+UcdjG3W85zMOygfYl5ttFl5VnHBrCImF9tDyXe8fNp7jYefzqPlxzUjvMdEJYJB7Eemjo/RLTqcCVwMYjPGY20NXMOhIJgKHAdSUbmFlXd18Vnb0YWIUkFHfnpX+t5dcfLKVnmwaMuq6/xgkSiTOxHhr6e8l5M3sD+OcRHlMY7Xw2hcjloy+5+xIzexTIcPeJwN1mNhg4COxAfRMSysGiYn4+cQmvz1zPkF6tePLavrqHsEgc+qbvyq5AiyM1cvdJwKRSyx4uMX3vN3x+iXM79x7ke6/P4V+Z2/je2Z25//zu6hsgEqdiPUewm/8+R7CJyD0KRL5m7dY93DJ2Nhu27+V31/Tl6pN03wCReBbroaH6QRciiWHGmm3c8eocDHjt1kEM7KgRQ0XiXUxDSZvZlWbWsMR8IzO7IriypCp6M2MD1784k6Z1a/LOXacpBESqiFjvKfBzd995aMbd84jcn0CE4mLnN5OX8eO/LWRQp6ZM+N5pujJIpAqJ9WRxWYGhyz+EvQcK+cG4+Xy4NJfhg9J45NJeVNcdxESqlFg/zDPM7Ekig8hBZMTQOcGUJFVFzs593Do2g2U5u3jk0p7ceGoHdRITqYJiDYLvAz8DxhO5emgqkTCQJLUwK49bx2aw90ARL940gHO6H/FqYhGJU7FeNbQHeCDgWqSKmLwohx++OZ9m9WrxyoiT6d5KF5WJVGWxXjU01cwalZhvbGZTgitL4pG7M+rjTO58bS692jTknbtOUwiIJIBYDw01i14pBIC77zAzHQtIIgWFRTw4YRET5mZzRb82PPbtPrp/gEiCiDUIis0szd3XA5hZB3QTmaSRX1DI7a9k8K/MbfzPed24+9wuOikskkBiDYKHgH+a2aeAAWcQvVGMJLZt+QXc/PJslmzcxe+v6cu3NVyESMKJ9WTxP8wsnciH/zzgHWBfkIVJ+LLz9nH9izPJ3rGP0defxLd6tAy7JBEJQKyDzt0K3Evk5jLzgUHAl5Rza0mp+jI37+b6F2exp6CQV289mQEdNFyESKKKtQvovcAAYJ27nwOcCOQd/iFSVc1bv4Or//QlhcXO+NtPUQiIJLhYzxHsd/f9ZoaZ1XL35WbWPdDKJBSfrdzCHa/OoXn9Wrxyy8mkNa0TdkkiErBYgyAr2o/gHWCqme0A1gVXloThvQUbue/N+XRpUZ+xtwygRf3UsEsSkUoQ68niK6OTj5jZx0BD4B+BVSWV7pUv1/LwxCUM6NCEMTem0yC1RtgliUglOeoRRN390yAKkXC4O09NX8Ufp61icI+WPHvdieooJpJkNJR0Eisudn7x3hLGfrmOq09qx2NX9dYQ0iJJSEGQpA4UFnP/WwuYuGAjI8/sxIMXHq/ewiJJSkGQhPYeKOSOV+fy2cotPHDh8dxxVuewSxKRECkIkkze3gPc/PJsFmzI4/Fv9+E7A9qHXZKIhExBkEQ27dzP9S/OZN32vTw//CQu6NUq7JJEJA4oCJLEV1v3MHzMTHbuO8jYmwdySuemYZckInFCQZAE1mzJZ9gLMzhY5IwbOYgT2jYMuyQRiSMKggS3eks+w0bPoNgjIdCtpe4oJiL/TUGQwDI3R/YE3J03bhtEV4WAiJRBQZCgMjfvZtgLM3FHISAih6UgSECrciMhYAbjRg6iS4t6YZckInFM4wkkmJW5uxn2wgzMInsCCgEROZJAg8DMhpjZCjPLNLMHylh/n5ktNbOFZjbdzI4Lsp5Et2LTboaNnkE1M+0JiEjMAgsCM0sBRgEXAj2BYWbWs1SzeUC6u/cB/gY8HlQ9iW75pl1c98IMqqdEQqBzc4WAiMQmyD2CgUCmu69x9wPAOODykg3c/WN33xudnUHknshylJbl7OK6F2ZSI6Ua40aeQieFgIgchSCDoC2wocR8VnRZeUYAk8taYWYjzSzDzDK2bNlSgSVWfUs3RvYEalWvxriRg+jYrG7YJYlIFRMXJ4vNbDiQDjxR1np3H+3u6e6e3rx588otLo4t2biT746ZQWqNFMaNHEQHhYCIfANBXj6aDZQc2rJddNl/MbPBwEPAWe5eEGA9CWVx9k6GvziTOjVSeGPkII5rqhAQkW8myD2C2UBXM+toZjWBocDEkg3M7ETgz8Bl7r45wFoSyuLsnXx3zEzq1qzOuJGnKARE5JgEFgTuXgjcDUwBlgFvuvsSM3vUzC6LNnsCqAe8ZWbzzWxiOT9OohZlRUKgXq3qjBs5iLSmdcIuSUSquEB7Frv7JGBSqWUPl5geHOTzJ5qFWXkMHzOT+qk1GDdyEO2bKARE5NhpiIkq4tDhoIa1IyHQrrFCQEQqRlxcNSSHt3brHm58aRYNUhUCIlLxFARxbvPu/dzw0iyK3fnriIEKARGpcDo0FMd27z/IzX+ZzZbdBbx+28kaNkJEAqE9gjhVUFjE7a/MYcWm3Tw/vD8npjUOuyQRSVDaI4hDxcXOfW8u4IvV23jyO305u3uLsEsSkQSmPYI44+48+v5SPliYw4MXHs9V/TUOn4gES0EQZ577ZDUvf7GWW0/vyMgzO4VdjogkAQVBHHkzYwNPTFnBFf3a8L8X9cDMwi5JRJKAgiBOTF+Wy4MTFnFG12Y8fnVfqlVTCIhI5VAQxIE567Zz1+tz6dWmAX8afhI1q+vPIiKVR584IVuVu5tbXs6gVYNUXrppAHVr6UIuEalcCoIQ5ezcxw0vzaJm9Wq8MuJkmtWrFXZJIpKEFAQhydt7gBtenMXu/YW8fPMAjSQqIqFREIRg/8Eibh2bwbptexl9w0n0atMw7JJEJInpgHQlKywq5u7X5zFn/Q5GXdefUzs3C7skEUly2iOoRO7OT99ZzLRluTx6WS8u6t067JJERBQElenJqSsZN3sD3z+3C9ef0iHsckREAAVBpRk/ez3PfJTJ0AHtue+8bmGXIyLybwqCSrAoayc/e3cJZ3Rtxq+uOEFDR4hIXFEQBCxv7wHufG0OzerW5KmhJ1I9RZtcROKLrhoKUHGx88Px88ndtZ+37jiVJnVrhl2SiMjX6N/TAI36OJOPV2zh4Ut70a99o7DLEREpk4IgIJ+v2sKT01Zy5YltGX5yWtjliIiUS0EQgOy8fdzzxjy6tajPr6/UyWERiW8KggpWUFjE916by8Ei5/nh/alTU6dhRCS+6VOqgv36g2Us2JDHn4b3p1PzemGXIyJyRNojqEDvzMvmr1+uY+SZnRhygoaPEJGqQUFQQVZs2s2DExYxsGMTfnxB97DLERGJmYKgAuzef5A7X51DvdTqPDtMncZEpGoJ9BPLzIaY2QozyzSzB8pYf6aZzTWzQjO7OshaguLu/PhvC1m3fS/PDjuRFg1Swy5JROSoBBYEZpYCjAIuBHoCw8ysZ6lm64GbgNeDqiNoL/7zKyYv3sQDQ47n5E5Nwy5HROSoBXnV0EAg093XAJjZOOByYOmhBu6+NrquOMA6AjPrq+38ZvJyhvRqxa1ndAy7HBGRbyTIQ0NtgQ0l5rOiyxLC5t37uev1uaQ1qcMT1/RRpzERqbKqxFlNMxtpZhlmlrFly5awy/n37SZ37z/I88P7Uz+1RtgliYh8Y0EGQTbQvsR8u+iyo+buo9093d3TmzdvXiHFHYsnpqyIHBa6qjfHt2oQdjkiIsckyCCYDXQ1s45mVhMYCkwM8PkqxT8W5/Dnz9YwfFAaV57YLuxyRESOWWBB4O6FwN3AFGAZ8Ka7LzGzR83sMgAzG2BmWcA1wJ/NbElQ9VSENVvyuf+thfRt15CfXVL6AigRkaop0LGG3H0SMKnUsodLTM8mcsgo7u07EBlMrkaK8dzwk6hVPSXskkREKoQGnYvB1vwC7nhlDityd/PyzQNp26h22CWJiFQYBcERLN+0ixEvZ7BtTwHPDuvPWd3CP1ktIlKRFASHMX1ZLve8MY96qdV58/ZT6NNOt5sUkcSjICiDu/PiP7/i15OW0atNA8bcMIBWDTWGkIgkJgVBKQcKi3n43cWMm72Bi3q34vfX9KN2TZ0YFpHEpSAoYceeA9zx6hxmfrWde87twg8Gd6NaNQ0dISKJTUEQlbl5NyPGZpCzcz9PDe3H5f0SZlgkEZHDUhAAn67cwt2vzaVWjRTGjRxE/7TGYZckIlJpkj4Ixn6xll+8t4TurRow5sZ09REQkaSTtEFwsKiYR99byisz1jG4R0ueGtqPurWSdnOISBJLyk++nXsPctfrc/ln5lbuOKszP76gu04Ki0jSSrog+GrrHka8PJsNO/byxNV9uCa9/ZEfJCKSwJIqCL7I3Mqdr80lpZrx+m2DGNChSdgliYiELmmC4O9zsvjJ3xfSqXldXrxxAO2b1Am7JBGRuJA0QXBc0zqce3wLfv+dvrq1pIhICUkTBOkdmpCuQ0EiIl9TJW5eLyIiwVEQiIgkOQWBiEiSUxCIiCQ5BYGISJJTEIiIJDkFgYhIklMQiIgkOXP3sGs4Kma2BVj3DR/eDNhageVUNNV3bFTfsYv3GlXfN3ecuzcva0WVC4JjYWYZ7p4edh3lUX3HRvUdu3ivUfUFQ4eGRESSnIJARCTJJVsQjA67gCNQfcdG9R27eK9R9QUgqc4RiIjI1yXbHoGIiJSiIBARSXIJGQRmNsTMVphZppk9UMb6WmY2Prp+ppl1qMTa2pvZx2a21MyWmNm9ZbQ528x2mtn86NfDlVVf9PnXmtmi6HNnlLHezOzp6PZbaGb9K7G27iW2y3wz22VmPyjVptK3n5m9ZGabzWxxiWVNzGyqma2Kfm9czmNvjLZZZWY3VlJtT5jZ8ujf720za1TOYw/7Wgi4xkfMLLvE3/Gicl/0CkYAAAXTSURBVB572Pd7gPWNL1HbWjObX85jK2UbHhN3T6gvIAVYDXQCagILgJ6l2nwP+FN0eigwvhLraw30j07XB1aWUd/ZwPshbsO1QLPDrL8ImAwYMAiYGeLfehORjjKhbj/gTKA/sLjEsseBB6LTDwC/LeNxTYA10e+No9ONK6G284Hq0enfllVbLK+FgGt8BLg/htfAYd/vQdVXav3vgYfD3IbH8pWIewQDgUx3X+PuB4BxwOWl2lwOjI1O/w34lplZZRTn7jnuPjc6vRtYBrStjOeuQJcDf/WIGUAjM2sdQh3fAla7+zftaV5h3P0zYHupxSVfZ2OBK8p46AXAVHff7u47gKnAkKBrc/cP3b0wOjsDaFeRz3m0ytl+sYjl/X7MDldf9LPjO8AbFf28lSURg6AtsKHEfBZf/6D9d5vom2En0LRSqishekjqRGBmGatPMbMFZjbZzHpVamHgwIdmNsfMRpaxPpZtXBmGUv6bL8ztd0hLd8+JTm8CWpbRJh625S1E9vDKcqTXQtDujh6+eqmcQ2vxsP3OAHLdfVU568PehkeUiEFQJZhZPeDvwA/cfVep1XOJHO7oCzwDvFPJ5Z3u7v2BC4G7zOzMSn7+IzKzmsBlwFtlrA57+32NR44RxN212mb2EFAIvFZOkzBfC88DnYF+QA6Rwy/xaBiH3xuI+/dTIgZBNtC+xHy76LIy25hZdaAhsK1Sqos8Zw0iIfCau08ovd7dd7l7fnR6ElDDzJpVVn3unh39vhl4m8jud0mxbOOgXQjMdffc0ivC3n4l5B46ZBb9vrmMNqFtSzO7CbgE+G40qL4mhtdCYNw9192L3L0YeKGc5w71tRj9/LgKGF9emzC3YawSMQhmA13NrGP0v8ahwMRSbSYCh67OuBr4qLw3QkWLHk98EVjm7k+W06bVoXMWZjaQyN+pUoLKzOqaWf1D00ROKi4u1WwicEP06qFBwM4Sh0AqS7n/hYW5/Uop+Tq7EXi3jDZTgPPNrHH00Mf50WWBMrMhwI+By9x9bzltYnktBFljyfNOV5bz3LG834M0GFju7lllrQx7G8Ys7LPVQXwRuaplJZGrCR6KLnuUyIseIJXIIYVMYBbQqRJrO53IIYKFwPzo10XAHcAd0TZ3A0uIXAExAzi1EuvrFH3eBdEaDm2/kvUZMCq6fRcB6ZX8961L5IO9YYlloW4/IqGUAxwkcpx6BJHzTtOBVcA0oEm0bTowpsRjb4m+FjOBmyuptkwix9YPvQYPXUXXBph0uNdCJW6/V6Kvr4VEPtxbl64xOv+193tl1Bdd/vKh112JtqFsw2P50hATIiJJLhEPDYmIyFFQEIiIJDkFgYhIklMQiIgkOQWBiEiSUxCIRJlZUamRTStsJEsz61By5EqReFI97AJE4sg+d+8XdhEilU17BCJHEB1P/vHomPKzzKxLdHkHM/soOijadDNLiy5vGR3jf0H069Toj0oxsxcsch+KD82sdrT9PRa5P8VCMxsX0q8pSUxBIPIftUsdGrq2xLqd7t4beBb4Y3TZM8BYd+9DZNC2p6PLnwY+9cigd/2J9CgF6AqMcvdeQB7w7ejyB4AToz/njqB+OZHyqGexSJSZ5bt7vTKWrwXOdfc10QEDN7l7UzPbSmTYg4PR5Tnu3szMtgDt3L2gxM/oQOS+A12j8z8Barj7r8zsH0A+kVFS3/HogHkilUV7BCKx8XKmj0ZBieki/nOO7mIiYzf1B2ZHR7QUqTQKApHYXFvi+5fR6S+IjHYJ8F3g8+j0dOBOADNLMbOG5f1QM6sGtHf3j4GfEBkS/Wt7JSJB0n8eIv9Ru9QNyP/h7ocuIW1sZguJ/Fc/LLrs+8BfzOxHwBbg5ujye4HRZjaCyH/+dxIZubIsKcCr0bAw4Gl3z6uw30gkBjpHIHIE0XME6e6+NexaRIKgQ0MiIklOewQiIklOewQiIklOQSAikuQUBCIiSU5BICKS5BQEIiJJ7v8Bj1Bcio2duaEAAAAASUVORK5CYII=\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"markdown","metadata":{"id":"QNhs1_m06Ip0"},"source":["### ***Task 5: Predict Words Using Your Starting Text***"]},{"cell_type":"code","metadata":{"id":"FLac19VEYJ75","executionInfo":{"status":"ok","timestamp":1601648143042,"user_tz":-480,"elapsed":1527,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"cb14acf9-7c5b-477f-80f2-25087380d345","colab":{"base_uri":"https://localhost:8080/","height":55}},"source":["seed_text = \"One day\"\n","next_words = 20\n","  \n","for _ in range(next_words):\n","\t# INSERT YOUR CODE HERE\n","\ttoken_list = tokenizer.texts_to_sequences([seed_text])[0]\n","\ttoken_list = pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')\n","\tpredicted = model.predict_classes(token_list, verbose=0)\n","\toutput_word = \"\"\n","\tfor word, index in tokenizer.word_index.items():\n","\t\tif index == predicted:\n","\t\t\toutput_word = word\n","\t\t\tbreak\n","\tseed_text += \" \" + output_word\n","\n","print(seed_text)"],"execution_count":34,"outputs":[{"output_type":"stream","text":["One day the sun comes the less one thing you want to leave me better leave you now you know you twist\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"YDTcPCytkDSu"},"source":["## **4. Shakespeare Generator (Character Tokenization)**"]},{"cell_type":"markdown","metadata":{"id":"VM8JxBcXwYqI"},"source":["This tutorial demonstrates how to generate text using a character-based RNN. We will work with a dataset of Shakespeare's writing from Andrej Karpathy's The Unreasonable Effectiveness of Recurrent Neural Networks. Given a sequence of characters from this data (\"Shakespear\"), train a model to predict the next character in the sequence (\"e\"). Longer sequences of text can be generated by calling the model repeatedly."]},{"cell_type":"code","metadata":{"id":"qdF6qytakFap","executionInfo":{"status":"ok","timestamp":1601648172120,"user_tz":-480,"elapsed":1290,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"70ea8b71-cbcd-48c3-b3db-4735d8010bcc","colab":{"base_uri":"https://localhost:8080/","height":199}},"source":["import tensorflow as tf\n","import numpy as np\n","import time\n","#Download the dataset\n","path_to_file = tf.keras.utils.get_file('shakespeare.txt',\n","                                       'https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt')\n","#Explore the data\n","text = open(path_to_file, 'r').read()\n","print(text[:100])"],"execution_count":35,"outputs":[{"output_type":"stream","text":["Downloading data from https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt\n","1122304/1115394 [==============================] - 0s 0us/step\n","First Citizen:\n","Before we proceed any further, hear me speak.\n","\n","All:\n","Speak, speak.\n","\n","First Citizen:\n","You\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"npWbbq8z7C1S"},"source":["### ***Task 6: Text Processing on Character Level***"]},{"cell_type":"code","metadata":{"id":"Rq35moPIntyT","executionInfo":{"status":"ok","timestamp":1601648247987,"user_tz":-480,"elapsed":1334,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"d190b4b7-1b62-4284-e474-7572e3f7e547","colab":{"base_uri":"https://localhost:8080/","height":199}},"source":["# Create the vocab of the given text\n","vocab = sorted(set(text))\n","print ('{} unique characters'.format(len(vocab)))\n","\n","# Creating a mapping from unique characters to indices\n","char2idx = {u:i for i, u in enumerate(vocab)}\n","idx2char = np.array(vocab)\n","print(idx2char)\n","\n","# Map the character to corresponding integer (char2idx)\n","text_as_int = np.array([char2idx[c] for c in text])\n","for char,_ in zip(char2idx, range(5)):\n","    print(repr(char), ':', char2idx[char])"],"execution_count":38,"outputs":[{"output_type":"stream","text":["65 unique characters\n","['\\n' ' ' '!' '$' '&' \"'\" ',' '-' '.' '3' ':' ';' '?' 'A' 'B' 'C' 'D' 'E'\n"," 'F' 'G' 'H' 'I' 'J' 'K' 'L' 'M' 'N' 'O' 'P' 'Q' 'R' 'S' 'T' 'U' 'V' 'W'\n"," 'X' 'Y' 'Z' 'a' 'b' 'c' 'd' 'e' 'f' 'g' 'h' 'i' 'j' 'k' 'l' 'm' 'n' 'o'\n"," 'p' 'q' 'r' 's' 't' 'u' 'v' 'w' 'x' 'y' 'z']\n","'\\n' : 0\n","' ' : 1\n","'!' : 2\n","'$' : 3\n","'&' : 4\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"_qDhZElEn2RZ","executionInfo":{"status":"ok","timestamp":1601648259334,"user_tz":-480,"elapsed":1755,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"d96f2d0c-ef9c-49b8-9841-54bb6e4007e4","colab":{"base_uri":"https://localhost:8080/","height":108}},"source":["# The maximum length sentence we want for a single input in characters\n","seq_length = 100\n","examples_per_epoch = len(text)//(seq_length+1)\n","\n","# Create training examples / targets\n","char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)\n","\n","for i in char_dataset.take(5):\n","  print(idx2char[i.numpy()])"],"execution_count":39,"outputs":[{"output_type":"stream","text":["F\n","i\n","r\n","s\n","t\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"gBIS-PGjn4YA","executionInfo":{"status":"ok","timestamp":1601648262685,"user_tz":-480,"elapsed":1465,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"a6300596-e40e-4a58-c631-28db34c30284","colab":{"base_uri":"https://localhost:8080/","height":128}},"source":["sequences = char_dataset.batch(seq_length+1, drop_remainder=True)\n","for item in sequences.take(5):\n","  print(repr(''.join(idx2char[item.numpy()])))"],"execution_count":40,"outputs":[{"output_type":"stream","text":["'First Citizen:\\nBefore we proceed any further, hear me speak.\\n\\nAll:\\nSpeak, speak.\\n\\nFirst Citizen:\\nYou '\n","'are all resolved rather to die than to famish?\\n\\nAll:\\nResolved. resolved.\\n\\nFirst Citizen:\\nFirst, you k'\n","\"now Caius Marcius is chief enemy to the people.\\n\\nAll:\\nWe know't, we know't.\\n\\nFirst Citizen:\\nLet us ki\"\n","\"ll him, and we'll have corn at our own price.\\nIs't a verdict?\\n\\nAll:\\nNo more talking on't; let it be d\"\n","'one: away, away!\\n\\nSecond Citizen:\\nOne word, good citizens.\\n\\nFirst Citizen:\\nWe are accounted poor citi'\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"HGsj19yun_aM","executionInfo":{"status":"ok","timestamp":1601648265776,"user_tz":-480,"elapsed":1221,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}}},"source":["def split_input_target(chunk):\n","  input_text = chunk[:-1]\n","  target_text = chunk[1:]\n","  return input_text, target_text\n","dataset = sequences.map(split_input_target)"],"execution_count":41,"outputs":[]},{"cell_type":"code","metadata":{"id":"asKeTJK1oEKr","executionInfo":{"status":"ok","timestamp":1601648282770,"user_tz":-480,"elapsed":1259,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"e26abaf9-6eb1-42a8-8df2-ddadb29d167d","colab":{"base_uri":"https://localhost:8080/","height":35}},"source":["# Batch size\n","BATCH_SIZE = 64\n","\n","# Buffer size to shuffle the dataset\n","# (TF data is designed to work with possibly infinite sequences,\n","# so it doesn't attempt to shuffle the entire sequence in memory. Instead,\n","# it maintains a buffer in which it shuffles elements).\n","BUFFER_SIZE = 10000\n","\n","dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)\n","\n","dataset"],"execution_count":42,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<BatchDataset shapes: ((64, 100), (64, 100)), types: (tf.int64, tf.int64)>"]},"metadata":{"tags":[]},"execution_count":42}]},{"cell_type":"code","metadata":{"id":"tcbsnOlBoFHp","executionInfo":{"status":"ok","timestamp":1601648288936,"user_tz":-480,"elapsed":991,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}}},"source":["# Length of the vocabulary in chars\n","vocab_size = len(vocab)\n","\n","# The embedding dimension\n","embedding_dim = 256\n","\n","# Number of RNN units\n","rnn_units = 1024"],"execution_count":43,"outputs":[]},{"cell_type":"code","metadata":{"id":"5tcFbOj2oHFG","executionInfo":{"status":"ok","timestamp":1601648293196,"user_tz":-480,"elapsed":1242,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}}},"source":["# Helper function for building new model\n","def build_model(vocab_size, embedding_dim, rnn_units, batch_size):\n","\n","  model = tf.keras.Sequential([\n","    tf.keras.layers.Embedding(vocab_size, embedding_dim,\n","                              batch_input_shape=[batch_size, None]),\n","    tf.keras.layers.LSTM(rnn_units,\n","                        return_sequences=True,\n","                        stateful=True,\n","                        recurrent_initializer='glorot_uniform'),\n","    tf.keras.layers.Dense(vocab_size)\n","  ])\n","\n","  return model\n","\n","# return the chosen loss parameter\n","def loss(labels, logits):\n","  return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)"],"execution_count":44,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"5b77DXXB749n"},"source":["### ***Task 7: Build Shakespeare Generator Model***"]},{"cell_type":"code","metadata":{"id":"rkP4z3dYoKl3","executionInfo":{"status":"ok","timestamp":1601648410891,"user_tz":-480,"elapsed":1302,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"dcf34020-cb5e-44d2-b019-a1484a05c695","colab":{"base_uri":"https://localhost:8080/","height":290}},"source":["# Complete the model infomation below\n","\n","model = build_model(\n","    vocab_size = len(vocab),\n","    embedding_dim=embedding_dim,\n","    rnn_units=rnn_units,\n","    batch_size=BATCH_SIZE)\n","\n","print(model.summary())\n","\n","model.compile(optimizer='adam', loss=loss, metrics=['accuracy'])"],"execution_count":47,"outputs":[{"output_type":"stream","text":["Model: \"sequential_8\"\n","_________________________________________________________________\n","Layer (type)                 Output Shape              Param #   \n","=================================================================\n","embedding_7 (Embedding)      (64, None, 256)           16640     \n","_________________________________________________________________\n","lstm_5 (LSTM)                (64, None, 1024)          5246976   \n","_________________________________________________________________\n","dense_7 (Dense)              (64, None, 65)            66625     \n","=================================================================\n","Total params: 5,330,241\n","Trainable params: 5,330,241\n","Non-trainable params: 0\n","_________________________________________________________________\n","None\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"OOH937ZspOS0","executionInfo":{"status":"ok","timestamp":1601648416321,"user_tz":-480,"elapsed":1111,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}}},"source":["import os\n","# Directory where the checkpoints will be saved\n","checkpoint_dir = './training_checkpoints'\n","# Name of the checkpoint files\n","checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}\")\n","\n","checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(\n","    filepath=checkpoint_prefix,\n","    save_weights_only=True)"],"execution_count":48,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Uin6lj9dsONj"},"source":["Each epoch will take around 30 seconds, choose your own epoch number wisely!"]},{"cell_type":"code","metadata":{"id":"Di8Jyg4d82sD","executionInfo":{"status":"ok","timestamp":1601649026180,"user_tz":-480,"elapsed":608277,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"4960682f-9854-4bfc-b8f7-5d8952b372c1","colab":{"base_uri":"https://localhost:8080/","height":745}},"source":["# Train your model here\n","history = model.fit(dataset, epochs=20, callbacks=[checkpoint_callback], verbose=1)"],"execution_count":49,"outputs":[{"output_type":"stream","text":["Epoch 1/20\n","172/172 [==============================] - 29s 168ms/step - loss: 2.6460 - accuracy: 0.2752\n","Epoch 2/20\n","172/172 [==============================] - 29s 167ms/step - loss: 1.9380 - accuracy: 0.4333\n","Epoch 3/20\n","172/172 [==============================] - 29s 167ms/step - loss: 1.6778 - accuracy: 0.5038\n","Epoch 4/20\n","172/172 [==============================] - 29s 167ms/step - loss: 1.5347 - accuracy: 0.5413\n","Epoch 5/20\n","172/172 [==============================] - 29s 167ms/step - loss: 1.4467 - accuracy: 0.5639\n","Epoch 6/20\n","172/172 [==============================] - 29s 168ms/step - loss: 1.3859 - accuracy: 0.5794\n","Epoch 7/20\n","172/172 [==============================] - 29s 168ms/step - loss: 1.3400 - accuracy: 0.5916\n","Epoch 8/20\n","172/172 [==============================] - 29s 168ms/step - loss: 1.3002 - accuracy: 0.6021\n","Epoch 9/20\n","172/172 [==============================] - 29s 168ms/step - loss: 1.2642 - accuracy: 0.6116\n","Epoch 10/20\n","172/172 [==============================] - 29s 168ms/step - loss: 1.2280 - accuracy: 0.6217\n","Epoch 11/20\n","172/172 [==============================] - 29s 168ms/step - loss: 1.1933 - accuracy: 0.6316\n","Epoch 12/20\n","172/172 [==============================] - 29s 168ms/step - loss: 1.1569 - accuracy: 0.6428\n","Epoch 13/20\n","172/172 [==============================] - 29s 168ms/step - loss: 1.1189 - accuracy: 0.6549\n","Epoch 14/20\n","172/172 [==============================] - 29s 168ms/step - loss: 1.0791 - accuracy: 0.6674\n","Epoch 15/20\n","172/172 [==============================] - 29s 167ms/step - loss: 1.0399 - accuracy: 0.6810\n","Epoch 16/20\n","172/172 [==============================] - 29s 167ms/step - loss: 0.9986 - accuracy: 0.6954\n","Epoch 17/20\n","172/172 [==============================] - 29s 167ms/step - loss: 0.9560 - accuracy: 0.7102\n","Epoch 18/20\n","172/172 [==============================] - 29s 168ms/step - loss: 0.9133 - accuracy: 0.7257\n","Epoch 19/20\n","172/172 [==============================] - 29s 168ms/step - loss: 0.8713 - accuracy: 0.7406\n","Epoch 20/20\n","172/172 [==============================] - 29s 168ms/step - loss: 0.8324 - accuracy: 0.7557\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"0iFdSFjIviPk","executionInfo":{"status":"ok","timestamp":1601649031807,"user_tz":-480,"elapsed":1472,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}}},"source":["# Simplify the output model with batch_size = 1 for ease of prediction\n","model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)\n","model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\n","model.build(tf.TensorShape([1, None]))"],"execution_count":50,"outputs":[]},{"cell_type":"code","metadata":{"id":"bbfotJT-pi5_","executionInfo":{"status":"ok","timestamp":1601649037595,"user_tz":-480,"elapsed":999,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}}},"source":["def generate_text(model, start_string):\n","  # Evaluation step (generating text using the learned model)\n","\n","  # Number of characters to generate\n","  num_generate = 1000\n","\n","  # Converting our start string to numbers (vectorizing)\n","  input_eval = [char2idx[s] for s in start_string]\n","  input_eval = tf.expand_dims(input_eval, 0)\n","\n","  # Empty string to store our results\n","  text_generated = []\n","\n","  # Low temperatures results in more predictable text.\n","  # Higher temperatures results in more surprising text.\n","  # Experiment to find the best setting.\n","  temperature = 1.0\n","\n","  # Here batch size == 1\n","  model.reset_states()\n","  for i in range(num_generate):\n","    predictions = model(input_eval)\n","    # remove the batch dimension\n","    predictions = tf.squeeze(predictions, 0)\n","\n","    # using a categorical distribution to predict the character returned by the model\n","    predictions = predictions / temperature\n","    predicted_id = tf.random.categorical(predictions, num_samples=1)[-1,0].numpy()\n","\n","    # We pass the predicted character as the next input to the model\n","    # along with the previous hidden state\n","    input_eval = tf.expand_dims([predicted_id], 0)\n","\n","    text_generated.append(idx2char[predicted_id])\n","\n","  return (start_string + ''.join(text_generated))"],"execution_count":51,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"kV2EmC-e9d-3"},"source":["### ***Task 8: Check Your Prediction Result***"]},{"cell_type":"code","metadata":{"id":"HygvXOurq2UU","executionInfo":{"status":"ok","timestamp":1601649073246,"user_tz":-480,"elapsed":7613,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"0f25f1fa-71a6-497b-b695-02dc8717fa15","colab":{"base_uri":"https://localhost:8080/","height":726}},"source":["# Based on the helper function generate_text(model, start_string) \n","# for generating text using the learned model above\n","# print the generated text with your preferred starting string\n","\n","# INSERT YOUR CODE HERE\n","print(generate_text(model, start_string=\"ROMEO: \"))"],"execution_count":52,"outputs":[{"output_type":"stream","text":["ROMEO: Send me by joy.\n","I say he laws me presently.\n","\n","Third March:\n","My lord-short in his house, you'll tell me think\n","That he hithest uned the watch alike. Come about me.\n","\n","CLIFFORD:\n","Plassee, my lord:\n","No, no, gentle Cleasing: for we mean to stand\n","not on her daughter's name; and there it is\n","A RIVE:\n","Ond gracio, my grooms are in their way:\n","There speak no general.\n","\n","Clown:\n","If it be mute: I'll be an\n","greaternghy arms:\n","Comes true to me that did you love to live.\n","Go you bring and grants with words.\n","Sirrah, deserve: I have brought you to your\n","soul encompation. If you fight good nd from hence and mistress.\n","\n","SICINIUS:\n","Spoile no uncle?\n","\n","First Servant:\n","This, sitting and are thou further changeance; he\n","Thou incarrems choler thou didst usurp the hour!\n","\n","ANGELO:\n","The guests man.\n","Coiser this is thine eyes commanded.\n","\n","QUEEN:\n","Yea, ay, as heaven as a business purpose.\n","Nor farewell, good cousin, furt up your speech! I desire his\n","That honourable evionc; my wife, like anning me; for whom it will,\n","Till we with cloack on the\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"eKQnHcjUKZmI"},"source":["## **(Optional) Text Generation using GPT-2 Transformer**"]},{"cell_type":"markdown","metadata":{"id":"LwskL9bod5hB"},"source":["OpenAI GPT-2 model was proposed in Language Models are Unsupervised Multitask Learners by Alec Radford, Jeffrey Wu, Rewon Child, David Luan, Dario Amodei and Ilya Sutskever. It’s a causal (unidirectional) transformer pretrained using language modeling on a very large corpus of ~40 GB of text data.\n","\n","GPT-2 is a large transformer-based language model with 1.5 billion parameters, trained on a dataset of 8 million web pages. GPT-2 is trained with a simple objective: predict the next word, given all of the previous words within some text. The diversity of the dataset causes this simple goal to contain naturally occurring demonstrations of many tasks across diverse domains. GPT-2 is a direct scale-up of GPT, with more than 10X the parameters and trained on more than 10X the amount of data.\n","\n","Here, we will use a lighter version, [GPT-2 Medium Model](https://huggingface.co/gpt2-medium) (which is around 1.5 GB) for your convenience and demo purpose."]},{"cell_type":"code","metadata":{"id":"9CRWFJGmKlnP","executionInfo":{"status":"ok","timestamp":1601649098249,"user_tz":-480,"elapsed":9106,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"37d1f8e1-f0a7-420c-9e06-2d8f45e97a7a","colab":{"base_uri":"https://localhost:8080/","height":655}},"source":["!pip install transformers"],"execution_count":53,"outputs":[{"output_type":"stream","text":["Collecting transformers\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/19/22/aff234f4a841f8999e68a7a94bdd4b60b4cebcfeca5d67d61cd08c9179de/transformers-3.3.1-py3-none-any.whl (1.1MB)\n","\u001b[K     |████████████████████████████████| 1.1MB 2.8MB/s \n","\u001b[?25hRequirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n","Collecting sentencepiece!=0.1.92\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n","\u001b[K     |████████████████████████████████| 1.1MB 9.9MB/s \n","\u001b[?25hCollecting tokenizers==0.8.1.rc2\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/80/83/8b9fccb9e48eeb575ee19179e2bdde0ee9a1904f97de5f02d19016b8804f/tokenizers-0.8.1rc2-cp36-cp36m-manylinux1_x86_64.whl (3.0MB)\n","\u001b[K     |████████████████████████████████| 3.0MB 18.2MB/s \n","\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n","Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n","Collecting sacremoses\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n","\u001b[K     |████████████████████████████████| 890kB 16.0MB/s \n","\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n","Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (1.15.0)\n","Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n","Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n","Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n","Building wheels for collected packages: sacremoses\n","  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893257 sha256=e946fbe5d9b91b9bb9c638cd52cb23fd36f9f57256140e9e2249f94b4830659a\n","  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n","Successfully built sacremoses\n","Installing collected packages: sentencepiece, tokenizers, sacremoses, transformers\n","Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.8.1rc2 transformers-3.3.1\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"tIxzduTbU3DF","executionInfo":{"status":"ok","timestamp":1601649131140,"user_tz":-480,"elapsed":5613,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}}},"source":["from transformers import TFGPT2LMHeadModel, GPT2Tokenizer, GPT2Config"],"execution_count":54,"outputs":[]},{"cell_type":"code","metadata":{"id":"yTK-DbTnU0PH","executionInfo":{"status":"ok","timestamp":1601649273319,"user_tz":-480,"elapsed":141115,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"d67e48ed-e327-4ff3-cd0f-2b33c0d76e49","colab":{"base_uri":"https://localhost:8080/","height":310,"referenced_widgets":["a4c8978ff96042438375acf91d5f95ed","3720a96a33de44489ea7627657870761","0c38e51224414bd89ef554e5e48ec4cf","4ee2890fbf0849c882cb9760434cc6b2","905be1e6305c47f99f8ca530b3cb1fe3","121e08961fe14b4199d2ffbca1490f56","00b6630a13924fa5837ab0ceaddb2911","2842cf70bf83431a85971d0ce6428238","ac97fde4df884300a92f9e19fdda3984","e5ac8eec85eb46958fc663dda865794c","dfe298c12b694dbfa189c1cbc7eb64a7","15a8511a37774cef94f8f58b5ce2dc3d","1cad36a55df547d4b82f7538e65e8bf6","a252ade7e4874212a3b60d25dab34bd1","2d6a4540b6924af88680d82256abed82","ee49f59aa03f493f9f031766df103b66","6280bdf917d44dc6861df998ea76da28","9d9748ed90c0413e801339bedadfb509","f2bcf585360d4c24bf501655195871e8","92c1690ab02d414f9e2170a59cdcd78d","be764c6eb46549f9ac146dfbd765ee05","80a19c61e3a2489ea9e5095280e8e60a","2440de067eec48fdb3dd2586f2d2fb0c","38e7122a39204173b60c145c7c35cb3b","3af5836f5389450d90cf105ca1af1bdb","81cb2b8adfbe407fa7cce95bf7f454ea","2a1581a8a40e413ea60fb189f34068d4","93a2b412efab4678a8e746def3863e43","db52bf47ea8148cf974c7ea5363ea115","bc6f762c802146ca8d97afe611fad7b0","c18c18ed4d9e4713b1bb9f3c8e43ed7d","062484220b3744369ee53ccd4c996ce8"]}},"source":["model_name = \"gpt2-medium\"\n","config = GPT2Config.from_pretrained(model_name)\n","tokenizer = GPT2Tokenizer.from_pretrained(model_name)\n","gptmodel = TFGPT2LMHeadModel.from_pretrained(model_name, config=config)"],"execution_count":55,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"a4c8978ff96042438375acf91d5f95ed","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=718.0, style=ProgressStyle(description_…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"ac97fde4df884300a92f9e19fdda3984","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1042301.0, style=ProgressStyle(descript…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"6280bdf917d44dc6861df998ea76da28","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456318.0, style=ProgressStyle(descripti…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"3af5836f5389450d90cf105ca1af1bdb","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1419628976.0, style=ProgressStyle(descr…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"},{"output_type":"stream","text":["All model checkpoint weights were used when initializing TFGPT2LMHeadModel.\n","\n","All the weights of TFGPT2LMHeadModel were initialized from the model checkpoint at gpt2-medium.\n","If your task is similar to the task the model of the checkpoint was trained on, you can already use TFGPT2LMHeadModel for predictions without further training.\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"0Lt17WZYZ8-o","executionInfo":{"status":"ok","timestamp":1601649277867,"user_tz":-480,"elapsed":1148,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}}},"source":["from transformers import set_seed\n","set_seed(23)"],"execution_count":56,"outputs":[]},{"cell_type":"code","metadata":{"id":"6ZA1oVCJVria","executionInfo":{"status":"ok","timestamp":1601649280782,"user_tz":-480,"elapsed":1220,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"3afbe738-2e6f-4e20-f135-302f0edc3f17","colab":{"base_uri":"https://localhost:8080/","height":72}},"source":["input_ids = tokenizer.encode('I love machine learning and data analytics,', return_tensors='tf')\n","input_ids"],"execution_count":57,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<tf.Tensor: shape=(1, 8), dtype=int32, numpy=\n","array([[   40,  1842,  4572,  4673,   290,  1366, 23696,    11]],\n","      dtype=int32)>"]},"metadata":{"tags":[]},"execution_count":57}]},{"cell_type":"code","metadata":{"id":"pdyttlcvWW9p","executionInfo":{"status":"ok","timestamp":1601649284900,"user_tz":-480,"elapsed":2909,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"3ba18c6c-8a4a-40ae-e88a-7f20060ca214","colab":{"base_uri":"https://localhost:8080/","height":90}},"source":["output = gptmodel.generate(input_ids, max_length=15)\n","print('Output:\\n')\n","print(tokenizer.decode(output[0], skip_special_tokens=True))"],"execution_count":58,"outputs":[{"output_type":"stream","text":["Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n"],"name":"stderr"},{"output_type":"stream","text":["Output:\n","\n","I love machine learning and data analytics, but I'm not a big fan\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Ua1NXKQiWpEp","executionInfo":{"status":"ok","timestamp":1601649294006,"user_tz":-480,"elapsed":7332,"user":{"displayName":"Yuchen Rao","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhpKnXgX6-Bki0D16qMjT5ltgGP8kbyYuXGb_GtkQ=s64","userId":"06801052792545274767"}},"outputId":"ef330dc0-e6f9-4989-d117-8a0c5f719c37","colab":{"base_uri":"https://localhost:8080/","height":201}},"source":["sample_outputs = gptmodel.generate(\n","    input_ids,\n","    do_sample=True,\n","    max_length=30,\n","    top_k=50,\n","    top_p=0.95,\n","    num_return_sequences=8\n",")\n","\n","for i, sample_output in enumerate(sample_outputs):\n","  \n","  print(\"{}: {}\".format(i, tokenizer.decode(sample_output)))"],"execution_count":59,"outputs":[{"output_type":"stream","text":["Setting `pad_token_id` to 50256 (first `eos_token_id`) to generate sequence\n"],"name":"stderr"},{"output_type":"stream","text":["0: I love machine learning and data analytics, but for me, the most important is data retention,\" she says. \"Not only will it make me feel\n","1: I love machine learning and data analytics, but the problem I'm facing today is that I feel the need to learn it all over again because it's\n","2: I love machine learning and data analytics, but not all of these jobs are created equal, especially in academia. This is why the AI field as a\n","3: I love machine learning and data analytics, I love the tools I can use to apply some of these techniques to solve problems at scale. I see a\n","4: I love machine learning and data analytics, but I think we've moved beyond the state of the art of predictive modeling and machine learning into some kind of\n","5: I love machine learning and data analytics, so I really have no excuse for anything. The main reason I've taken the plunge into this endeavor is that\n","6: I love machine learning and data analytics, but what keeps me from doing it at a big company is the expense. The costs are too great for me\n","7: I love machine learning and data analytics, so I want to be sure that when building my data science software you're using some kind of machine learning feature\n"],"name":"stdout"}]}]}