16-2-Convert_Date_Enc_Dec.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise 9:\n",
    "\n",
    "Train an Encoder–Decoder model that can convert a date string\n",
    "from one format to another (e.g., from “April 22, 2019” to “2019-\n",
    "04-22”)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "\n",
    "from datetime import date"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's start by creating dataset. We will use random days between 1000-01-01 and 9999-12-31. (Format: YYYY-MM-DD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "MONTHS = [\"January\", \"February\", \"March\", \"April\", \"May\", \"June\", \"July\", \n",
    "            \"August\", \"September\", \"October\", \"November\", \"December\"]\n",
    "\n",
    "def random_dates(n_dates):\n",
    "    min_date = date(1000, 1, 1).toordinal() # toordinal() returns the proleptic Gregorian ordinal of the date. Gregorian ordinal = number of days since 1st January 1 AD\n",
    "    max_date = date(9999, 12, 31).toordinal()\n",
    "    \n",
    "    ordinals = np.random.randint(max_date - min_date, size=n_dates) + min_date\n",
    "    dates = [date.fromordinal(ordinal) for ordinal in ordinals]\n",
    "    \n",
    "    X = [MONTHS[dt.month - 1] + \" \" + dt.strftime(\"%d, %Y\") for dt in dates]\n",
    "    y = [dt.isoformat() for dt in dates] # YYYY-MM-DD\n",
    "    \n",
    "    return X,y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here are few random dates, displayed in both input format and output format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input                    Target                   \n",
      "--------------------------------------------------\n",
      "September 20, 7075       7075-09-20               \n",
      "May 15, 8579             8579-05-15               \n",
      "January 11, 7103         7103-01-11               \n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "\n",
    "n_dates = 3\n",
    "X_example, y_example = random_dates(n_dates)\n",
    "print(\"{:25s}{:25s}\".format(\"Input\", \"Target\"))\n",
    "print(\"-\" * 50)\n",
    "for idx in range(n_dates):\n",
    "    print(\"{:25s}{:25s}\".format(X_example[idx], y_example[idx]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's get list of all the possible characters in the inputs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' ,0123456789ADFJMNOSabceghilmnoprstuvy'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "INPUT_CHARS = \"\".join(sorted(set(\"\".join(MONTHS) + \"0123456789, \")))\n",
    "INPUT_CHARS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "OUTPUT_CHARS = \"0123456789-\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's write a function to convert a strng to a list of character IDs."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here's the list of possible characters in the outputs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def date_str_to_ids(date_str, chars=INPUT_CHARS):\n",
    "    return [chars.index(c) for c in date_str] # ID = index of that character"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[19, 23, 31, 34, 23, 28, 21, 23, 32, 0, 4, 2, 1, 0, 9, 2, 9, 7]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "date_str_to_ids(X_example[0], INPUT_CHARS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[7, 0, 7, 5, 10, 0, 9, 10, 2, 0]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "date_str_to_ids(y_example[0], OUTPUT_CHARS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_date_strs(date_strs, chars=INPUT_CHARS):\n",
    "    X_ids = [date_str_to_ids(dt, chars) for dt in date_strs]\n",
    "    X = tf.ragged.constant(X_ids, ragged_rank=1)\n",
    "    # to_tensor() will create tensor of all same dimension by padding. Need to pad because RNNs can't handle variable length sequences\n",
    "    return (X + 1).to_tensor() # 0 will be used as padding. Therefore need to shift all the values of X by 1\n",
    "\n",
    "def create_dataset(n_dates):\n",
    "    X, y = random_dates(n_dates)\n",
    "    return prepare_date_strs(X, INPUT_CHARS), prepare_date_strs(y, OUTPUT_CHARS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "\n",
    "X_train, Y_train = create_dataset(10000)\n",
    "X_valid, Y_valid = create_dataset(2000)\n",
    "X_test, Y_test = create_dataset(2000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(10,), dtype=int32, numpy=array([ 8,  1,  8,  6, 11,  1, 10, 11,  3,  1], dtype=int32)>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_train[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model-1: Very basic Seq2Seq model\n",
    "\n",
    "Let's first try the simplest possible model: we feed in the input sequence, which first goes through the encoder (an embedding layer followed by a single LSTM layer), which outputs a vector, then it goes through a decoder (a single LSTM layer, followed by a dense output layer), which outputs a sequence of vectors, each representing the estimated probabilities for all possible output character.\n",
    "\n",
    "Since the decoder expects a sequence as input, we repeat the vector (which is output by the encoder) as many times as the longest possible output sequence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m18s\u001b[0m 45ms/step - accuracy: 0.2982 - loss: 1.9901 - val_accuracy: 0.5701 - val_loss: 1.2087\n",
      "Epoch 2/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m18s\u001b[0m 57ms/step - accuracy: 0.5911 - loss: 1.1647 - val_accuracy: 0.6970 - val_loss: 0.8390\n",
      "Epoch 3/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m21s\u001b[0m 68ms/step - accuracy: 0.7165 - loss: 0.7564 - val_accuracy: 0.7741 - val_loss: 0.5747\n",
      "Epoch 4/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m21s\u001b[0m 68ms/step - accuracy: 0.7493 - loss: 0.6931 - val_accuracy: 0.8189 - val_loss: 0.5089\n",
      "Epoch 5/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m11s\u001b[0m 37ms/step - accuracy: 0.8438 - loss: 0.4408 - val_accuracy: 0.9000 - val_loss: 0.2987\n",
      "Epoch 6/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m12s\u001b[0m 39ms/step - accuracy: 0.9165 - loss: 0.2582 - val_accuracy: 0.9569 - val_loss: 0.1679\n",
      "Epoch 7/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m13s\u001b[0m 42ms/step - accuracy: 0.9502 - loss: 0.2016 - val_accuracy: 0.9543 - val_loss: 0.1978\n",
      "Epoch 8/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m12s\u001b[0m 38ms/step - accuracy: 0.9706 - loss: 0.1393 - val_accuracy: 0.9860 - val_loss: 0.0775\n",
      "Epoch 9/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m13s\u001b[0m 41ms/step - accuracy: 0.9903 - loss: 0.0629 - val_accuracy: 0.9924 - val_loss: 0.0460\n",
      "Epoch 10/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m12s\u001b[0m 37ms/step - accuracy: 0.9958 - loss: 0.0372 - val_accuracy: 0.9964 - val_loss: 0.0297\n",
      "Epoch 11/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m13s\u001b[0m 42ms/step - accuracy: 0.9983 - loss: 0.0236 - val_accuracy: 0.9985 - val_loss: 0.0201\n",
      "Epoch 12/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m13s\u001b[0m 42ms/step - accuracy: 0.9992 - loss: 0.0157 - val_accuracy: 0.9990 - val_loss: 0.0141\n",
      "Epoch 13/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m13s\u001b[0m 43ms/step - accuracy: 0.9997 - loss: 0.0108 - val_accuracy: 0.9995 - val_loss: 0.0101\n",
      "Epoch 14/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m12s\u001b[0m 39ms/step - accuracy: 0.9998 - loss: 0.0076 - val_accuracy: 0.9997 - val_loss: 0.0074\n",
      "Epoch 15/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m13s\u001b[0m 41ms/step - accuracy: 1.0000 - loss: 0.0055 - val_accuracy: 0.9999 - val_loss: 0.0056\n",
      "Epoch 16/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m15s\u001b[0m 49ms/step - accuracy: 1.0000 - loss: 0.0041 - val_accuracy: 0.9999 - val_loss: 0.0043\n",
      "Epoch 17/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m13s\u001b[0m 40ms/step - accuracy: 1.0000 - loss: 0.0031 - val_accuracy: 0.9999 - val_loss: 0.0034\n",
      "Epoch 18/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m12s\u001b[0m 39ms/step - accuracy: 1.0000 - loss: 0.0024 - val_accuracy: 0.9999 - val_loss: 0.0027\n",
      "Epoch 19/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m12s\u001b[0m 39ms/step - accuracy: 1.0000 - loss: 0.0019 - val_accuracy: 0.9999 - val_loss: 0.0022\n",
      "Epoch 20/20\n",
      "\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m12s\u001b[0m 39ms/step - accuracy: 1.0000 - loss: 0.0015 - val_accuracy: 0.9999 - val_loss: 0.0017\n"
     ]
    }
   ],
   "source": [
    "keras.backend.clear_session()\n",
    "\n",
    "embedding_size = 32\n",
    "max_output_length = Y_train.shape[1] # 10\n",
    "\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "encoder = keras.models.Sequential([\n",
    "    keras.layers.InputLayer(shape=(None,), ragged=True),\n",
    "    keras.layers.Embedding(input_dim=len(INPUT_CHARS)+1, output_dim=embedding_size),\n",
    "    keras.layers.LSTM(128)\n",
    "])\n",
    "\n",
    "decoder = keras.models.Sequential([\n",
    "    keras.layers.LSTM(128, return_sequences=True),\n",
    "    keras.layers.Dense(len(OUTPUT_CHARS) + 1, activation=\"softmax\")\n",
    "])\n",
    "\n",
    "model = keras.models.Sequential([\n",
    "    encoder,\n",
    "    keras.layers.RepeatVector(max_output_length),\n",
    "    decoder\n",
    "])\n",
    "\n",
    "optimizer = keras.optimizers.Nadam()\n",
    "\n",
    "model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\n",
    "\n",
    "h = model.fit(X_train, Y_train, epochs=20, validation_data=(X_valid, Y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's use model to make some predictions but first we will need a function to convert sequence of character IDs to a readable string."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def id_to_strs(ids, chars=OUTPUT_CHARS):\n",
    "    return [\"\".join([(\"?\" + chars)[index] for index in sequence])\n",
    "            for sequence in ids]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can use the model to convert some dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_new_str = [\"September 17, 2009\", \"July 14, 1789\"]\n",
    "X_new = prepare_date_strs(X_new_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([18])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_new[1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input Date               Date in ISO Format       \n",
      "--------------------------------------------------\n",
      "September 17, 2009       2009-09-17               \n",
      "July 14, 1789            1789-07-14               \n"
     ]
    }
   ],
   "source": [
    "ids = np.argmax(model.predict(X_new, verbose=0), axis=-1)\n",
    "print(\"{:25s}{:25s}\".format(\"Input Date\", \"Date in ISO Format\"))\n",
    "print(\"-\" * 50)\n",
    "\n",
    "date_str = id_to_strs(ids)\n",
    "for idx in range(len(date_str)):\n",
    "    print(\"{:25s}{:25s}\".format(X_new_str[idx], date_str[idx]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Perfect :)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "However, since the model was trained on input strings of length 18 (which is length of the longest date), it does not perform well if we try to make predictions using a shorter sequences:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_new_str = [\"May 02, 2020\", \"July 14, 1789\"]\n",
    "X_new = prepare_date_strs(X_new_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([13])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_new[0].shape # longest is 13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input Date               Date in ISO Format       \n",
      "--------------------------------------------------\n",
      "May 02, 2020             2020-01-02               \n",
      "July 14, 1789            1789-09-14               \n"
     ]
    }
   ],
   "source": [
    "ids = np.argmax(model.predict(X_new, verbose=0), axis=-1)\n",
    "print(\"{:25s}{:25s}\".format(\"Input Date\", \"Date in ISO Format\"))\n",
    "print(\"-\" * 50)\n",
    "\n",
    "date_str = id_to_strs(ids)\n",
    "for idx in range(len(date_str)):\n",
    "    print(\"{:25s}{:25s}\".format(X_new_str[idx], date_str[idx]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Oops! We need to ensure that we always pas sequences of the same length as during training, using padding if necessary. Let's write a little helper function for the same:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_input_length = X_train.shape[1]\n",
    "\n",
    "def prepare_date_strs_padded(date_strs):\n",
    "    X = prepare_date_strs(date_strs)\n",
    "    if X.shape[1] < max_input_length:\n",
    "        X = tf.pad(X, [[0,0], [0, max_input_length - X.shape[1]]])\n",
    "    return X\n",
    "\n",
    "def convert_date_strs(date_strs):\n",
    "    X = prepare_date_strs_padded(date_strs)\n",
    "    ids = np.argmax(model.predict(X, verbose=0), axis=-1)\n",
    "    return id_to_strs(ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input Date               Date in ISO Format       \n",
      "--------------------------------------------------\n",
      "May 02, 2020             2020-05-02               \n",
      "July 14, 1789            1789-07-14               \n"
     ]
    }
   ],
   "source": [
    "X_new_str = [\"May 02, 2020\", \"July 14, 1789\"]\n",
    "\n",
    "preds = convert_date_strs(X_new_str)\n",
    "\n",
    "print(\"{:25s}{:25s}\".format(\"Input Date\", \"Date in ISO Format\"))\n",
    "print(\"-\" * 50)\n",
    "\n",
    "for idx in range(len(preds)):\n",
    "    print(\"{:25s}{:25s}\".format(X_new_str[idx], preds[idx]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cool!\n",
    "\n",
    "However, real-life sequence-to-sequence problems will usually be harder, so far for sake of completeness, let's build more powerful model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model-2: Feeding the shifted targets to the decoder (teacher forcing)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "latest",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}