mimic_prediction_arima/prediction_arimax_steady_train.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.9.0 64-bit"
  },
  "interpreter": {
   "hash": "e2bfb1b1dd0bcdebdb315279aa118b1f834444d4ba3ba6d660e9f6ce7703f6a2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "# ARIMAX"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# path_to_data = '/hpi/fs00/share/MPSS2021BA1/data/'\n",
    "path_to_data = '../data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import time\n",
    "\n",
    "starttime = time.time()\n",
    "print('Start reading the input file.')\n",
    "\n",
    "TRAIN=12\n",
    "# Run for the one or the other\n",
    "# read dict where train size is TRAIN\n",
    "input_file = open(str(path_to_data) + 'dict_of_chunk_series_with_test_and_train_' + str(TRAIN) + '.pickle', 'rb')\n",
    "dict_of_chunk_series_with_test_and_train = pickle.load(input_file)\n",
    "input_file.close()\n",
    "\n",
    "endtime = round(((time.time() - starttime) / 60), 5)\n",
    "print('Reading of the input file completed after '+str(endtime)+' minutes.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pmdarima as pm\n",
    "import numpy as np\n",
    "import copy\n",
    "import time\n",
    "import warnings\n",
    "import pyarrow as pa\n",
    "\n",
    "starttime = time.time()\n",
    "\n",
    "# Expand the previously created dictionary (dict_of_chunk_series_with_test_and_train) to also hold the prediction series next to the train and the test series (and threshold values for test)\n",
    "\n",
    "runningtime = round(((time.time() - starttime) / 60), 5)\n",
    "print('Starting setting up dictionaries. Running time '+str(runningtime)+' min.')\n",
    "\n",
    "dict_of_chunk_series_with_test_and_train_and_forecast = copy.deepcopy(dict_of_chunk_series_with_test_and_train)\n",
    "dict_of_chunk_series_with_forecast_df = {}\n",
    "accuracy_dict_for_chunk_iterations = {}\n",
    "chunk_iterations_with_runtime_warning = pd.DataFrame(columns=[\"CHUNK_ID_FILLED_TH\",\"ITERATION\",\"WARNING_MSG\"])\n",
    "\n",
    "# Convert warnings to exceptions\n",
    "warnings.filterwarnings('error', category=RuntimeWarning)\n",
    "np.seterr(all='warn')\n",
    "\n",
    "runningtime = round(((time.time() - starttime) / 60), 5)\n",
    "print('Completed setting up dictionaries. Running time '+str(runningtime)+' min.')\n",
    "\n",
    "for j, chunk in enumerate(dict_of_chunk_series_with_test_and_train_and_forecast):\n",
    "    dict_of_chunk_series_with_forecast_df[chunk] = {}\n",
    "    accuracy_dict_for_chunk_iterations[chunk] = {}\n",
    "\n",
    "    runningtime = round(((time.time() - starttime) / 60), 5)\n",
    "    print('Chunk '+str(j)+' (ID: '+str(chunk)+'): START. Running time '+str(runningtime)+' min.')\n",
    "    \n",
    "    for i, chunk_iteration in enumerate(dict_of_chunk_series_with_test_and_train_and_forecast[chunk]):\n",
    "        \n",
    "        TEST = dict_of_chunk_series_with_test_and_train_and_forecast[chunk][chunk_iteration][\"TEST_LIST_MEDIAN\"].size\n",
    "        tp, tn, fp, fn = 0, 0, 0, 0\n",
    "        accurracy_matrix_df_for_chunk_iteration = pd.DataFrame(columns=[\"TP\",\"FN\",\"FP\",\"TN\"])\n",
    "\n",
    "        ########################\n",
    "        # ARIMAX for High Alarms\n",
    "        ########################\n",
    "\n",
    "        current_train_list_high = dict_of_chunk_series_with_test_and_train_and_forecast[chunk][chunk_iteration][\"TRAIN_LIST_MAX\"] \n",
    "        current_test_list_high = dict_of_chunk_series_with_test_and_train_and_forecast[chunk][chunk_iteration][\"TEST_LIST_MAX\"]\n",
    "        current_train_list_exog_high = dict_of_chunk_series_with_test_and_train_and_forecast[chunk][chunk_iteration][\"TRAIN_LIST_MEDIAN\"].values.reshape(-1, 1)\n",
    "        current_test_list_exog_high = dict_of_chunk_series_with_test_and_train_and_forecast[chunk][chunk_iteration][\"TEST_LIST_MEDIAN\"].values.reshape(-1, 1)\n",
    "\n",
    "        try:\n",
    "            arimax_high = pm.auto_arima(current_train_list_high, X=current_train_list_exog_high, seasonal=False, suppress_warnings=True, error_action='ignore')\n",
    "            forecast_arimax_high = pd.Series(arimax_high.predict(TEST, X=current_test_list_exog_high), index=[*range(i+TRAIN,i+TRAIN+TEST,1)], name=\"forecast_list_arimax_high\")\n",
    "            dict_of_chunk_series_with_test_and_train_and_forecast[chunk][chunk_iteration][\"FORECAST_LIST_ARIMAX_HIGH\"] = forecast_arimax_high\n",
    "            \n",
    "            runningtime = round(((time.time() - starttime) / 60), 5)\n",
    "            print('Chunk '+str(j)+' (ID: '+str(chunk)+') iteration '+str(chunk_iteration)+': Completed ARIMAX - High Alarms. Running time '+str(runningtime)+' min.')\n",
    "\n",
    "            ########################\n",
    "            # ARIMAX for Low Alarms\n",
    "            ########################\n",
    "            \n",
    "            current_train_list_low = dict_of_chunk_series_with_test_and_train_and_forecast[chunk][chunk_iteration][\"TRAIN_LIST_MIN\"] \n",
    "            current_test_list_low = dict_of_chunk_series_with_test_and_train_and_forecast[chunk][chunk_iteration][\"TEST_LIST_MIN\"] \n",
    "            current_train_list_exog_low = dict_of_chunk_series_with_test_and_train_and_forecast[chunk][chunk_iteration][\"TRAIN_LIST_MEDIAN\"].values.reshape(-1, 1)\n",
    "            current_test_list_exog_low = dict_of_chunk_series_with_test_and_train_and_forecast[chunk][chunk_iteration][\"TEST_LIST_MEDIAN\"].values.reshape(-1, 1)\n",
    "            arimax_low = pm.auto_arima(current_train_list_low, X=current_train_list_exog_low, seasonal=False, suppress_warnings=True, error_action='ignore')\n",
    "            forecast_arimax_low = pd.Series(arimax_low.predict(TEST, X=current_test_list_exog_low), index=[*range(i+TRAIN,i+TRAIN+TEST,1)], name=\"forecast_list_arimax_low\")\n",
    "            dict_of_chunk_series_with_test_and_train_and_forecast[chunk][chunk_iteration][\"FORECAST_LIST_ARIMAX_LOW\"] = forecast_arimax_low\n",
    "            \n",
    "            runningtime = round(((time.time() - starttime) / 60), 5)\n",
    "            print('Chunk '+str(j)+' (ID: '+str(chunk)+') iteration '+str(chunk_iteration)+': Completed ARIMAX - Low Alarms. Running time '+str(runningtime)+' min.')\n",
    "\n",
    "            # extract threshold series \n",
    "            threshold_high_for_test_list = dict_of_chunk_series_with_test_and_train_and_forecast[chunk][chunk_iteration][\"THRESHOLD_HIGH_FOR_TEST_LIST\"]\n",
    "            threshold_low_for_test_list = dict_of_chunk_series_with_test_and_train_and_forecast[chunk][chunk_iteration][\"THRESHOLD_LOW_FOR_TEST_LIST\"]\n",
    "            \n",
    "            # write to dict_of_chunk_series_with_forecast_df dataframe\n",
    "            all_dict_lists_as_df = pd.concat([current_test_list_high,forecast_arimax_high,threshold_high_for_test_list,current_test_list_low,forecast_arimax_low,threshold_low_for_test_list],axis=1)\n",
    "            dict_of_chunk_series_with_forecast_df[chunk][chunk_iteration] = all_dict_lists_as_df\n",
    "\n",
    "            ##############################################\n",
    "            # Add information whether alarm was triggered\n",
    "            ##############################################\n",
    "\n",
    "            df_for_chunk_iteration = dict_of_chunk_series_with_forecast_df[chunk][chunk_iteration]\n",
    "            \n",
    "            # True alarms\n",
    "            df_for_chunk_iteration['high_alarm_triggered'] = np.where(df_for_chunk_iteration['test_list_max'] > df_for_chunk_iteration['threshold_high_for_test_list'] ,1,0)\n",
    "            df_for_chunk_iteration['low_alarm_triggered'] = np.where(df_for_chunk_iteration['test_list_min'] < df_for_chunk_iteration['threshold_low_for_test_list'] ,1,0)\n",
    "                    \n",
    "            # ARIMAX forecast\n",
    "            df_for_chunk_iteration['high_alarm_triggered_forecast_arimax'] = np.where(df_for_chunk_iteration['forecast_list_arimax_high'] > df_for_chunk_iteration['threshold_high_for_test_list'],1,0)\n",
    "            df_for_chunk_iteration['low_alarm_triggered_forecast_arimax'] = np.where(df_for_chunk_iteration['forecast_list_arimax_low'] < df_for_chunk_iteration['threshold_low_for_test_list'],1,0)\n",
    "            #write to dict_of_chunk_series_with_forecast_and_alarm_df dataframe\n",
    "            dict_of_chunk_series_with_forecast_df[chunk][chunk_iteration] = df_for_chunk_iteration\n",
    "            \n",
    "            runningtime = round(((time.time() - starttime) / 60), 5)\n",
    "            print('Chunk '+str(j)+' (ID: '+str(chunk)+') iteration '+str(chunk_iteration)+': Completed Alarm Identification. Running time '+str(runningtime)+' min.')\n",
    "\n",
    "            ##########################################\n",
    "            # Calculate Confusion Matrix - High Alarms\n",
    "            ##########################################\n",
    "\n",
    "            # select true high alarms triggered\n",
    "            column_index_of_high_alarm_triggered = df_for_chunk_iteration.columns.get_loc(\"high_alarm_triggered\")\n",
    "\n",
    "            # select predicted high alarms\n",
    "            column_index_of_high_alarm_triggered_forecast_arimax = df_for_chunk_iteration.columns.get_loc(\"high_alarm_triggered_forecast_arimax\")\n",
    "            \n",
    "            # create df with bot as column\n",
    "            high_alarms = df_for_chunk_iteration.iloc[0:,[column_index_of_high_alarm_triggered,column_index_of_high_alarm_triggered_forecast_arimax]]\n",
    "            \n",
    "            for row_in_high_alarms in high_alarms.iterrows():\n",
    "\n",
    "                if row_in_high_alarms[1][0] and row_in_high_alarms[1][1]:\n",
    "                    tp +=1\n",
    "                    # print(\"tp\", tp)\n",
    "                if row_in_high_alarms[1][0] and not row_in_high_alarms[1][1]:\n",
    "                    fn +=1\n",
    "                    # print(\"fn\", fn)\n",
    "                if not row_in_high_alarms[1][0] and row_in_high_alarms[1][1]:\n",
    "                    fp +=1\n",
    "                    # print(\"fp\", fp)\n",
    "                if not row_in_high_alarms[1][0] and not row_in_high_alarms[1][1]:\n",
    "                    tn +=1\n",
    "                    # print(\"tn\",tn)\n",
    "            \n",
    "            a_new_row = {\"TP\":tp,\"FN\":fn,\"FP\":fp,\"TN\":tn}\n",
    "            a_new_row_series = pd.Series(a_new_row,name=\"accuracy_high_alarms_arimax\")\n",
    "\n",
    "            accurracy_matrix_df_for_chunk_iteration = accurracy_matrix_df_for_chunk_iteration.append(a_new_row_series)\n",
    "\n",
    "            runningtime = round(((time.time() - starttime) / 60), 5)\n",
    "            print('Chunk '+str(j)+' (ID: '+str(chunk)+') iteration '+str(chunk_iteration)+': Completed Confusion Matrix - High Alarms. Running time '+str(runningtime)+' min.')\n",
    "\n",
    "            #########################################\n",
    "            # Calculate Confusion Matrix - Low Alarms\n",
    "            ##########################################\n",
    "\n",
    "            # Reset tp, tn, fp, fn\n",
    "            tp, tn, fp, fn = 0, 0, 0, 0\n",
    "\n",
    "            # select true low alarms triggered\n",
    "            column_index_of_low_alarm_triggered = df_for_chunk_iteration.columns.get_loc(\"low_alarm_triggered\")\n",
    "            \n",
    "            # select predicted low alarms\n",
    "            column_index_of_low_alarm_triggered_forecast_arimax = df_for_chunk_iteration.columns.get_loc(\"low_alarm_triggered_forecast_arimax\")\n",
    "            \n",
    "            # create df with bot as column \n",
    "            low_alarms = df_for_chunk_iteration.iloc[0:,[column_index_of_low_alarm_triggered,column_index_of_low_alarm_triggered_forecast_arimax]]\n",
    "            \n",
    "            for row_in_low_alarms in low_alarms.iterrows():\n",
    "\n",
    "                if row_in_low_alarms[1][0] and row_in_low_alarms[1][1]:\n",
    "                    tp +=1\n",
    "                    # print(\"tp\", tp)\n",
    "                if row_in_low_alarms[1][0] and not row_in_low_alarms[1][1]:\n",
    "                    fn +=1\n",
    "                    # print(\"fn\", fn)\n",
    "                if not row_in_low_alarms[1][0] and row_in_low_alarms[1][1]:\n",
    "                    fp +=1\n",
    "                    # print(\"fp\", fp)\n",
    "                if not row_in_low_alarms[1][0] and not row_in_low_alarms[1][1]:\n",
    "                    tn +=1\n",
    "                    # print(\"tn\",tn)\n",
    "            \n",
    "            a_new_row = {\"TP\":tp,\"FN\":fn,\"FP\":fp,\"TN\":tn}\n",
    "            a_new_row_series = pd.Series(a_new_row,name=\"accuracy_low_alarms_arimax\")\n",
    "            \n",
    "            accurracy_matrix_df_for_chunk_iteration = accurracy_matrix_df_for_chunk_iteration.append(a_new_row_series)\n",
    "\n",
    "            runningtime = round(((time.time() - starttime) / 60), 5)\n",
    "            print('Chunk '+str(j)+' (ID: '+str(chunk)+') iteration '+str(chunk_iteration)+': Completed Confusion Matrix - Low Alarms. Running time '+str(runningtime)+' min.')\n",
    "            \n",
    "            # Write confusion matrix into dictionary\n",
    "            accuracy_dict_for_chunk_iterations[chunk][chunk_iteration] = accurracy_matrix_df_for_chunk_iteration\n",
    "\n",
    "        except RuntimeWarning as rw:\n",
    "\n",
    "            rw_string = str(rw)\n",
    "            a_new_row = {\"CHUNK_ID_FILLED_TH\":chunk,\"ITERATION\":chunk_iteration,\"WARNING_MSG\":rw_string}\n",
    "            a_new_row_series = pd.Series(a_new_row)\n",
    "            chunk_iterations_with_runtime_warning = chunk_iterations_with_runtime_warning.append(a_new_row_series, ignore_index = True)\n",
    "            chunk_iterations_with_runtime_warning.to_parquet(str(path_to_data)+'chunk_iterations_with_runtime_warning_for_arimax_'+str(TRAIN)+'.parquet', engine='pyarrow')\n",
    "            print(\"RUNTIME WARNING DETECTED:\")\n",
    "            print(a_new_row_series)\n",
    "        \n",
    "\n",
    "\n",
    "    runningtime = round(((time.time() - starttime) / 60), 5)\n",
    "    print('Chunk '+str(j)+' (ID: '+str(chunk)+' ) : Completed chunk. Running time '+str(runningtime)+' min.')\n",
    "    print('--------------------')\n",
    "        \n",
    "endtime = round(((time.time() - starttime) / 60), 5)\n",
    "print('DONE')\n",
    "print('Completed in '+str(endtime)+' minutes.')\n",
    "\n",
    "\n",
    "print('Starting saving dictionary.')\n",
    "output_file = open(str(path_to_data)+'accuracy_dict_for_chunk_iterations_arimax_'+str(TRAIN)+'.pickle', 'wb')\n",
    "pickle.dump(accuracy_dict_for_chunk_iterations, output_file)\n",
    "output_file.close()\n",
    "print('Completed saving dictionary.')\n"
   ]
  }
 ]
}