mimic_prediction_arima/preprocessing_arima_steady_train.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "orig_nbformat": 4,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.7.4 64-bit ('base': conda)"
  },
  "interpreter": {
   "hash": "c24247fa39158f46a54dbb99bb8811b81cd84bf3c9aa6e8294d53a41a5837da9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "# ARIMA(X) Data Preprocessing - Steady Train Size"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# path_to_data = '/hpi/fs00/share/MPSS2021BA1/data/'\n",
    "path_to_data = '../data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pyarrow as pa\n",
    "import time\n",
    "\n",
    "starttime = time.time()\n",
    "print('Start reading the input file.')\n",
    "\n",
    "# Read chartevents_resampled from parquet file to pandas data frame\n",
    "#chartevents_resampled = pd.read_parquet(str(path_to_data)+'resampling/resample_running_output_hr.parquet', engine='pyarrow')\n",
    "#chartevents_resampled = pd.read_parquet(str(path_to_data)+'resampling/resample_output_bp.parquet', engine='pyarrow')\n",
    "#chartevents_resampled = pd.read_parquet(str(path_to_data)+'resampling/resample_output_o2.parquet', engine='pyarrow')\n",
    "#chartevents_resampled = pd.read_parquet(str(path_to_data)+'resampling/resample_output_hr_first1000.parquet', engine='pyarrow')\n",
    "#chartevents_resampled = pd.read_parquet(str(path_to_data)+'resampling/resample_output_bp_first1000.parquet', engine='pyarrow')\n",
    "#chartevents_resampled = pd.read_parquet(str(path_to_data)+'resampling/resample_output_o2_first1000.parquet', engine='pyarrow')\n",
    "\n",
    "endtime = round(((time.time() - starttime) / 60), 5)\n",
    "print('Reading of the input file completed after '+str(endtime)+' minutes.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Sampling rate of 1 data point per hour - Test for different values in the future - e.g. longer training set\n",
    "TRAIN = 12 # 12 * 1 h = 12 hour training period\n",
    "TEST = 1 # 1 * 1 h = 2 hours testing period\n",
    "STEP = 1 # move 1 * 1 h = 1 hour per step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter for chunks that have sufficient values to be used for training and testing the model\n",
    "all_chunks_value_count = chartevents_resampled.CHUNK_ID_FILLED_TH.value_counts()\n",
    "chunkid_filter = all_chunks_value_count[all_chunks_value_count >= (TRAIN + TEST)].index\n",
    "arima_data = chartevents_resampled[chartevents_resampled.CHUNK_ID_FILLED_TH.isin(chunkid_filter)].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create new HOURS_SINCE_FIRST_RECORD column containing the time difference that has passed since the first timestamp of the measurement series.\n",
    "import numpy as np\n",
    "# arima_data['MINUTES_SINCE_FIRST_RECORD'] = arima_data.groupby('CHUNK_ID_FILLED_TH')#['CHARTTIME'].transform(lambda x: (x - x.min())/np.timedelta64(1,'m'))\n",
    "# Alternative for hours instead of minutes\n",
    "arima_data['HOURS_SINCE_FIRST_RECORD'] = arima_data.groupby('CHUNK_ID_FILLED_TH')['CHARTTIME'].transform(lambda x: (x - x.min())/np.timedelta64(1,'h'))"
   ]
  },
  {
   "source": [
    "## Create dict with each chunk as key\n",
    "Create dict that holds vital parameter series, threshold high and threshold low series for each chunk id (key). The series are all indexed the same way (= dif to first measurement in hours with current sampling rate) so they relate to the same time"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create dictionary with chunk id as key and a dataframe as value.\n",
    "# This dataframe contains of three columns the vital parameter values, the high thresholds and the low thresholds.\n",
    "# As the index of these three list is the same and can be referenced back to the \"HOURS_SINCE_FIRST_RECORD\", we keep the time related information.\n",
    "# Example:\n",
    "# dict_of_chunk_series = {\n",
    "#     \"<chunkid_A>\" : { | vital_parameter_series | threshold_high_series | threshold_low_series\n",
    "#                     0 |                   95.0 |                   120 |                   60\n",
    "#                     1 |                   90.5 |                   120 |                   60\n",
    "#                     2 |                   91.0 |                   120 |                   60\n",
    "#                    },\n",
    "#     \"<chunkid_B>\" : { | vital_parameter_series | threshold_high_series | threshold_low_series\n",
    "#                     0 |                   88.0 |                   110 |                   50\n",
    "#                     1 |                   78.5 |                   110 |                   50\n",
    "#                     2 |                   73.0 |                   120 |                   60\n",
    "#                    }\n",
    "#  }\n",
    "#\n",
    "# Example with additional vital parameter series which can be used as an exogenous variable for ARIMAX\n",
    "# dict_of_chunk_series_with_test_and_train = {\n",
    "#     \"<chunkid_A>\" : | vital_parameter_series_median | vital_parameter_series_mean | vital_parameter_series_min |threshold_high_series | threshold_low_series (+max)\n",
    "#                   0 |                          95.0 |                        98.0 |                          80|                  120 |                   60\n",
    "#                   1 |                          90.5 |                        96.0 |                          79|                  120 |                   60\n",
    "#                   2 |                          91.0 |                        94.0 |                          83|                  120 |                   60\n",
    "#  }\n",
    "\n",
    "runningtime = round(((time.time() - starttime) / 60), 5)\n",
    "print('Starting setting up chunk based dictionary (First). Running time '+str(runningtime)+' min.')\n",
    "\n",
    "dict_of_chunk_series = {}\n",
    "\n",
    "for chunkid in chunkid_filter:\n",
    "    \n",
    "    chunk_data = arima_data[arima_data.CHUNK_ID_FILLED_TH == chunkid].copy()\n",
    "\n",
    "    # vital parameter series - median resampling\n",
    "    chunk_value_series_median = pd.Series(chunk_data['VITAL_PARAMTER_VALUE_MEDIAN_RESAMPLING'],name=\"vital_parameter_series_median\")\n",
    "    chunk_value_series_median = chunk_value_series_median.reset_index(drop=True)\n",
    "    chunk_value_series_median.index = list(chunk_value_series_median.index)\n",
    "\n",
    "    # vital parameter series - mean resampling\n",
    "    chunk_value_series_mean = pd.Series(chunk_data['VITAL_PARAMTER_VALUE_MEAN_RESAMPLING'],name=\"vital_parameter_series_mean\")\n",
    "    chunk_value_series_mean = chunk_value_series_mean.reset_index(drop=True)\n",
    "    chunk_value_series_mean.index = list(chunk_value_series_mean.index)\n",
    "\n",
    "    # vital parameter series - min resampling\n",
    "    chunk_value_series_min = pd.Series(chunk_data['VITAL_PARAMTER_VALUE_MIN_RESAMPLING'],name=\"vital_parameter_series_min\")\n",
    "    chunk_value_series_min = chunk_value_series_min.reset_index(drop=True)\n",
    "    chunk_value_series_min.index = list(chunk_value_series_min.index)\n",
    "\n",
    "    # vital parameter series - max resampling\n",
    "    chunk_value_series_max = pd.Series(chunk_data['VITAL_PARAMTER_VALUE_MAX_RESAMPLING'],name=\"vital_parameter_series_max\")\n",
    "    chunk_value_series_max = chunk_value_series_max.reset_index(drop=True)\n",
    "    chunk_value_series_max.index = list(chunk_value_series_max.index)  \n",
    "    \n",
    "\n",
    "    # threshold series high\n",
    "    chunk_threshold_high_series = pd.Series(chunk_data['THRESHOLD_VALUE_HIGH'],name=\"threshold_high_series\")\n",
    "    chunk_threshold_high_series = chunk_threshold_high_series.reset_index(drop=True)\n",
    "    chunk_threshold_high_series.index = list(chunk_threshold_high_series.index)\n",
    "\n",
    "    # threshold series low\n",
    "    chunk_threshold_low_series = pd.Series(chunk_data['THRESHOLD_VALUE_LOW'],name=\"threshold_low_series\")\n",
    "    chunk_threshold_low_series = chunk_threshold_low_series.reset_index(drop=True)\n",
    "    chunk_threshold_low_series.index = list(chunk_threshold_low_series.index)\n",
    "\n",
    "    # Append series with key (CHUNK_ID) into dictionary\n",
    "    vital_parameter_and_thresholds_for_chunkid = pd.concat([chunk_value_series_median,chunk_value_series_mean,chunk_value_series_min,chunk_value_series_max,chunk_threshold_high_series,chunk_threshold_low_series],axis=1)\n",
    "    dict_of_chunk_series[chunkid] = vital_parameter_and_thresholds_for_chunkid\n",
    "\n",
    "runningtime = round(((time.time() - starttime) / 60), 5)\n",
    "print('Finished setting up chunk based dictionary (First). Running time '+str(runningtime)+' min.')"
   ]
  },
  {
   "source": [
    "## Create dict with each chunk and chunk iteration as key\n",
    "Steady TRAIN  - > Train Set size is of size TRAIN"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create multiple test & train sets for each chunk to iteratively predict the next x measurements\n",
    "# Create nested dictionary that holds the CHUNK_ID as first key.\n",
    "# This key holds one dictionary for each iteration over this chunk. This depends on the TEST, TRAIN, and STEP.\n",
    "# For each iteration we create another dictionary, whereby the last index of the train list acts as key.\n",
    "# This key holds again one dictionary for the train list and one for the test list.\n",
    "# Example:\n",
    "# dict_of_chunk_series_with_test_and_train = {\n",
    "#     \"<chunkid_A>\" : {\n",
    "#         \"<last_index_of_training_list_of_first_chunkid_A_iteration>\" : {\n",
    "#             \"TRAIN_LIST_MEDIAN\" : train_list_median,\n",
    "#             \"TEST_LIST_MEDIAN\" : test_list_median,\n",
    "#             \"TRAIN_LIST_MEAN\" : train_list_mean,\n",
    "#             \"TEST_LIST_MEAN\" : test_list_mean,\n",
    "#             \"TRAIN_LIST_MIN\" : train_list_min,\n",
    "#             \"TEST_LIST_MIN\" : test_list_min,\n",
    "#             \"TRAIN_LIST_MAX\" : train_list_max,\n",
    "#             \"TEST_LIST_MAX\" : test_list_max,\n",
    "#             \"THRESHOLD_HIGH_FOR_TRAIN_LIST\" : threshold_high_for_train_list ,\n",
    "#             \"THRESHOLD_LOW_FOR_TEST_LIST\" : threshold_low_for_test_list\n",
    "#         },\n",
    "#         \"<last_index_of_training_list_of_second_chunkid_A_iteration>\" : {\n",
    "#             \"TRAIN_LIST_MEDIAN\" : train_list_median,\n",
    "#             \"TEST_LIST_MEDIAN\" : test_list_median,\n",
    "#             \"TRAIN_LIST_MEAN\" : train_list_mean,\n",
    "#             \"TEST_LIST_MEAN\" : test_list_mean,\n",
    "#             \"TRAIN_LIST_MIN\" : train_list_min,\n",
    "#             \"TEST_LIST_MIN\" : test_list_min,\n",
    "#             \"TRAIN_LIST_MAX\" : train_list_max,\n",
    "#             \"TEST_LIST_MAX\" : test_list_max,\n",
    "#             \"THRESHOLD_HIGH_FOR_TRAIN_LIST\" : threshold_high_for_train_list ,\n",
    "#             \"THRESHOLD_LOW_FOR_TEST_LIST\" : threshold_low_for_test_list\n",
    "#         },\n",
    "#     }\n",
    "# }\n",
    "import copy\n",
    "\n",
    "runningtime = round(((time.time() - starttime) / 60), 5)\n",
    "print('Starting setting up chunk iteration based dictionary for steady train size (Second). Running time '+str(runningtime)+' min.')\n",
    "\n",
    "dict_of_chunk_series_with_test_and_train = {}\n",
    "\n",
    "for i, chunk in enumerate(dict_of_chunk_series):\n",
    "    # acces dataframe of current chunk\n",
    "    chunk_series_for_chunk = copy.deepcopy(dict_of_chunk_series[chunk])\n",
    "\n",
    "    # access vital_parameter_series_median of current chunk\n",
    "    chunk_value_series_for_chunk_median = chunk_series_for_chunk[\"vital_parameter_series_median\"]\n",
    "    # access vital_parameter_series_mean of current chunk\n",
    "    chunk_value_series_for_chunk_mean = chunk_series_for_chunk[\"vital_parameter_series_mean\"]\n",
    "    # access vital_parameter_series_min of current chunk\n",
    "    chunk_value_series_for_chunk_min = chunk_series_for_chunk[\"vital_parameter_series_min\"]\n",
    "    # access vital_parameter_series_max of current chunk\n",
    "    chunk_value_series_for_chunk_max = chunk_series_for_chunk[\"vital_parameter_series_max\"]\n",
    "        \n",
    "    # access threshold_high_series of current chunk\n",
    "    chunk_threshold_high_series_for_chunk = chunk_series_for_chunk[\"threshold_high_series\"]\n",
    "    # access threshold_low_series of current chunk\n",
    "    chunk_threshold_low_series_for_chunk = chunk_series_for_chunk[\"threshold_low_series\"]\n",
    "\n",
    "    # create an empty dictionary for the key of the current chunk\n",
    "    dict_of_chunk_series_with_test_and_train[chunk] = {}\n",
    "\n",
    "    # create multiple test and train lists for that chunk\n",
    "    for start in range(0, len(chunk_value_series_for_chunk_median) - (TRAIN + TEST)+1, STEP):\n",
    "        \n",
    "        # vital_parameter_series_median\n",
    "        train_list_median = pd.Series(chunk_value_series_for_chunk_median[start : start+TRAIN], name=\"train_list_median\")\n",
    "        test_list_median = pd.Series(chunk_value_series_for_chunk_median[start+TRAIN : start+TRAIN+TEST], name=\"test_list_median\")\n",
    "        # vital_parameter_series_mean\n",
    "        train_list_mean = pd.Series(chunk_value_series_for_chunk_mean[start : start+TRAIN], name=\"train_list_mean\")\n",
    "        test_list_mean = pd.Series(chunk_value_series_for_chunk_mean[start+TRAIN : start+TRAIN+TEST], name=\"test_list_mean\")\n",
    "        # vital_parameter_series_min\n",
    "        train_list_min = pd.Series(chunk_value_series_for_chunk_min[start : start+TRAIN], name=\"train_list_min\")\n",
    "        test_list_min = pd.Series(chunk_value_series_for_chunk_min[start+TRAIN : start+TRAIN+TEST], name=\"test_list_min\")\n",
    "        # vital_parameter_series_max\n",
    "        train_list_max = pd.Series(chunk_value_series_for_chunk_max[start : start+TRAIN], name=\"train_list_max\")\n",
    "        test_list_max = pd.Series(chunk_value_series_for_chunk_max[start+TRAIN : start+TRAIN+TEST], name=\"test_list_max\")\n",
    "        \n",
    "        #threshold series high & low\n",
    "        threshold_high_for_test_list = pd.Series(chunk_threshold_high_series_for_chunk[start+TRAIN : start+TRAIN+TEST],name=\"threshold_high_for_test_list\")\n",
    "        threshold_low_for_test_list = pd.Series(chunk_threshold_low_series_for_chunk[start+TRAIN : start+TRAIN+TEST],name=\"threshold_low_for_test_list\")\n",
    "        \n",
    "        # For each iteration over the current chunk, we will create a dictionary that holds again the test and train list as dictionary\n",
    "        # We use the last index of the current train list (which currently refers to the difference to first measurement) as second key\n",
    "        second_key = train_list_median.index.max()\n",
    "        dict_of_chunk_series_with_test_and_train[chunk][second_key] = {}\n",
    "        # Assign the train and test list to the current chunk iteration      \n",
    "        dict_of_chunk_series_with_test_and_train[chunk][second_key][\"TRAIN_LIST_MEDIAN\"] = train_list_median\n",
    "        dict_of_chunk_series_with_test_and_train[chunk][second_key][\"TRAIN_LIST_MEAN\"] = train_list_mean\n",
    "        dict_of_chunk_series_with_test_and_train[chunk][second_key][\"TRAIN_LIST_MIN\"] = train_list_min\n",
    "        dict_of_chunk_series_with_test_and_train[chunk][second_key][\"TRAIN_LIST_MAX\"] = train_list_max\n",
    "\n",
    "        dict_of_chunk_series_with_test_and_train[chunk][second_key][\"TEST_LIST_MEDIAN\"] = test_list_median\n",
    "        dict_of_chunk_series_with_test_and_train[chunk][second_key][\"TEST_LIST_MEAN\"] = test_list_median\n",
    "        dict_of_chunk_series_with_test_and_train[chunk][second_key][\"TEST_LIST_MIN\"] = test_list_min\n",
    "        dict_of_chunk_series_with_test_and_train[chunk][second_key][\"TEST_LIST_MAX\"] = test_list_max\n",
    "\n",
    "        dict_of_chunk_series_with_test_and_train[chunk][second_key][\"THRESHOLD_HIGH_FOR_TEST_LIST\"] = threshold_high_for_test_list\n",
    "        dict_of_chunk_series_with_test_and_train[chunk][second_key][\"THRESHOLD_LOW_FOR_TEST_LIST\"] = threshold_low_for_test_list\n",
    "\n",
    "runningtime = round(((time.time() - starttime) / 60), 5)\n",
    "print('Finished setting up chunk iteration based dictionary for steady train size (Second). Running time '+str(runningtime)+' min.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "runningtime = round(((time.time() - starttime) / 60), 5)\n",
    "print('Started writing chunk iteration based dictionary for steady train size (Second) to pickle. Running time '+str(runningtime)+' min.')\n",
    "\n",
    "# Write dictionaries with expanding train and fix test size to pickle file\n",
    "\n",
    "#output_file = open(str(path_to_data)+'arima_preprocessing/dict_of_chunk_iterations_with_steady_train_'+str(TRAIN)+'_hr.pickle', 'wb')\n",
    "#output_file = open(str(path_to_data)+'arima_preprocessing/dict_of_chunk_iterations_with_steady_train_'+str(TRAIN)+'_bp.pickle', 'wb')\n",
    "#output_file = open(str(path_to_data)+'arima_preprocessing/dict_of_chunk_iterations_with_steady_train_'+str(TRAIN)+'_o2.pickle', 'wb')\n",
    "#output_file = open(str(path_to_data)+'arima_preprocessing/dict_of_chunk_iterations_with_steady_train_'+str(TRAIN)+'_hr_first1000.pickle', 'wb')\n",
    "#output_file = open(str(path_to_data)+'arima_preprocessing/dict_of_chunk_iterations_with_steady_train_'+str(TRAIN)+'_bp_first1000.pickle', 'wb')\n",
    "#output_file = open(str(path_to_data)+'arima_preprocessing/dict_of_chunk_iterations_with_steady_train_'+str(TRAIN)+'_o2_first1000.pickle', 'wb')\n",
    "\n",
    "pickle.dump(dict_of_chunk_series_with_test_and_train,output_file)\n",
    "output_file.close()\n",
    "\n",
    "runningtime = round(((time.time() - starttime) / 60), 5)\n",
    "print('Finished writing chunk iteration based dictionary for steady train size (Second) to pickle. Running time '+str(runningtime)+' min.')\n",
    "endtime = round(((time.time() - starttime) / 60), 5)\n",
    "print('DONE')\n",
    "print('Completed in '+str(endtime)+' minutes.')\n"
   ]
  }
 ]
}