mimic_prediction_rnn/analyze_chunk_level_predictions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Analysis of Predictions Produced on Chunk Level with RNNModel Class by Darts\n",
    "\n",
    "This script analyzes all pickle files in `./data/{approach}/{n_chunks}_chunks/{style}/`, starting with `confusion_matrix_chunks`, i.e. all chunk level results. At the moment, the paths are adapted for local execution."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Merge and Adjust Chunk Level Results\n",
    "\n",
    "The following analysis steps are only performed for one chunk-specific matrix file.\n",
    "\n",
    "### Define Variables to Adjust"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Adjust variables defining path\n",
    "approach = 'RNNModel'\n",
    "n_chunks = 2000\n",
    "style = 'all'\n",
    "\n",
    "# Adjust variables defining model\n",
    "version = 'normal'\n",
    "model_type = 'GRU'\n",
    "parameter = 'bp'\n",
    "endogenous_input = 'Median'\n",
    "window_idx = 0\n",
    "\n",
    "# Defines which results should be read\n",
    "# '': read all results\n",
    "# '_n': read non-scaled results\n",
    "# '_s1': read results produced with series scaled using standard score (same mean and same standard deviation)\n",
    "# '_s2': read results produced with series scaled separately with several MinMaxScalers\n",
    "suffix = ''\n",
    "\n",
    "# Adjust variable for correlation plot\n",
    "input_length = 12"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Read One Chunk Level Matrix"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from IPython.display import display\n",
    "import pickle5 as pickle\n",
    "\n",
    "# Define path to all chunk level matrices produced by prediction\n",
    "path_to_chunk_matrices = f'../../data/{approach}/{n_chunks}_chunks/{style}'\n",
    "\n",
    "# Read chunk-specific matrix\n",
    "chunks_matrix_f = open(f'{path_to_chunk_matrices}/confusion_matrix_chunks_{model_type}_{parameter}_{endogenous_input}_'\n",
    "                       f'{version}_window{window_idx}{suffix}.pickle', 'rb')\n",
    "chunks_matrix = pickle.load(chunks_matrix_f)\n",
    "chunks_matrix_f.close()\n",
    "\n",
    "# Show chunk-specific matrix\n",
    "display(chunks_matrix)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Read Chunk Level Matrices of All Windows & Parameters"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from IPython.display import display\n",
    "import pandas as pd\n",
    "import pickle5 as pickle\n",
    "\n",
    "# Define path to all chunk level matrices produced by prediction\n",
    "path_to_chunk_matrices = f'../../data/{approach}/{n_chunks}_chunks/{style}'\n",
    "\n",
    "chunks_matrix = pd.DataFrame(\n",
    "    columns=['CHUNK_ID', 'SCALED', 'PARAMETER', 'MODEL', 'ENDOGENOUS', 'EXOGENOUS', 'FIRST_FORECAST',\n",
    "             'ALARM_TYPE', 'FP', 'TP', 'FN', 'TN', 'N_HIGH_ALARMS', 'N_LOW_ALARMS', 'N_ITERATIONS'])\n",
    "\n",
    "for param in ['bp', 'hr', 'o2']:\n",
    "    for win in range(5):\n",
    "        # Read chunk-specific matrix\n",
    "        current_chunks_matrix_f = open(f'{path_to_chunk_matrices}/confusion_matrix_chunks_{model_type}_{param}_'\n",
    "                                       f'{endogenous_input}_{version}_window{win}{suffix}.pickle', 'rb')\n",
    "        current_chunks_matrix = pickle.load(current_chunks_matrix_f)\n",
    "        current_chunks_matrix_f.close()\n",
    "\n",
    "        # Append current matrix to result matrix\n",
    "        chunks_matrix = pd.concat([chunks_matrix, current_chunks_matrix])\n",
    "\n",
    "# Show chunk-specific matrix\n",
    "display(chunks_matrix)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Add Metrics to Chunk Level Matrix\n",
    "\n",
    "See https://en.wikipedia.org/wiki/Sensitivity_and_specificity for more information."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "# Note: To avoid dividing by zero, zeros are converted to NaN before division (since any value divided by NaN gives NaN)\n",
    "\n",
    "chunks_matrix['fp_tn_divisor'] = chunks_matrix['FP'] + chunks_matrix['TN']\n",
    "chunks_matrix['TNR'] = chunks_matrix.TN.div(chunks_matrix.fp_tn_divisor.where(chunks_matrix.fp_tn_divisor != 0, np.nan))\n",
    "chunks_matrix['FPR'] = chunks_matrix.FP.div(chunks_matrix.fp_tn_divisor.where(chunks_matrix.fp_tn_divisor != 0, np.nan)) # 1 - TNR\n",
    "\n",
    "chunks_matrix['fn_tp_divisor'] = chunks_matrix['FN'] + chunks_matrix['TP']\n",
    "chunks_matrix['TPR'] = chunks_matrix.TP.div(chunks_matrix.fn_tp_divisor.where(chunks_matrix.fn_tp_divisor != 0, np.nan))\n",
    "chunks_matrix['FNR'] = chunks_matrix.FN.div(chunks_matrix.fn_tp_divisor.where(chunks_matrix.fn_tp_divisor != 0, np.nan)) # 1 - TPR\n",
    "\n",
    "chunks_matrix['ACC_dividend'] = chunks_matrix['TN'] + chunks_matrix['TP']\n",
    "chunks_matrix['ACC_divisor'] = chunks_matrix['fp_tn_divisor'] + chunks_matrix['fn_tp_divisor']\n",
    "chunks_matrix['ACC'] = chunks_matrix.ACC_dividend.div(chunks_matrix.ACC_divisor.where(chunks_matrix.ACC_divisor != 0,\n",
    "                                                                                      np.nan))\n",
    "\n",
    "chunks_matrix['F1S_divisor'] = chunks_matrix['TP'] + 0.5 * (chunks_matrix['FP'] + chunks_matrix['FN'])\n",
    "chunks_matrix['F1S'] = chunks_matrix.TP.div(chunks_matrix.F1S_divisor.where(chunks_matrix.F1S_divisor != 0, np.nan))\n",
    "\n",
    "# Add weighted score from https://physionet.org/content/challenge-2015/1.0.0/\n",
    "chunks_matrix['WEIGHTED_SCORE_AA_divisor'] = chunks_matrix['TP'] + chunks_matrix['FN'] + (5 * chunks_matrix['FP'])\n",
    "chunks_matrix['WEIGHTED_SCORE_AA'] = chunks_matrix.TP.div(chunks_matrix.WEIGHTED_SCORE_AA_divisor\n",
    "                                                          .where(chunks_matrix.WEIGHTED_SCORE_AA_divisor != 0, np.nan))\n",
    "\n",
    "# Round all floats to 4 decimal places\n",
    "# Note: round() does not work for floats with many decimal places\n",
    "decimals = 4\n",
    "for col in ['FPR', 'TPR', 'FNR', 'TNR', 'ACC', 'F1S', 'WEIGHTED_SCORE_AA']:\n",
    "    chunks_matrix[col] = chunks_matrix[col].apply(lambda x: round(x, decimals))\n",
    "\n",
    "# Sort and remove helper columns for similarity with model level matrices\n",
    "chunks_matrix = chunks_matrix[['CHUNK_ID', 'SCALED', 'PARAMETER', 'MODEL', 'ENDOGENOUS', 'EXOGENOUS', 'FIRST_FORECAST',\n",
    "                               'ALARM_TYPE', 'FP', 'TP', 'FN', 'TN', 'FPR', 'TPR', 'FNR', 'TNR', 'ACC', 'F1S',\n",
    "                               'WEIGHTED_SCORE_AA', 'N_HIGH_ALARMS', 'N_LOW_ALARMS', 'N_ITERATIONS']]\n",
    "\n",
    "# Show complemented chunk level matrix for one chunk\n",
    "display(chunks_matrix)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Visualization of Chunk Level Results\n",
    "\n",
    "### Plot Correlation Between Chunk Length and Selected Metric"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# Add column for chunk length to all chunks of matrix\n",
    "chunks_matrix['LENGTH'] = chunks_matrix['N_ITERATIONS'] + input_length\n",
    "\n",
    "for param in pd.unique(chunks_matrix.PARAMETER):\n",
    "    print(f'## {param} ##')\n",
    "\n",
    "    plotdata = chunks_matrix[chunks_matrix['PARAMETER'] == param]\n",
    "\n",
    "    for metric in ['WEIGHTED_SCORE_AA']:\n",
    "\n",
    "        if metric == 'WEIGHTED_SCORE_AA':\n",
    "            ylabel = 'Evaluation Score'\n",
    "        else:\n",
    "            ylabel = metric\n",
    "\n",
    "        # Define background color, subplots and suptitle\n",
    "        sns.set_style('whitegrid')\n",
    "        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 4))\n",
    "        #fig.suptitle(f'Correlation of Chunk Length and {metric} of Chunk ({style.replace(\"_\", \" \").upper()})', fontsize=14)\n",
    "\n",
    "        if endogenous_input == 'MIN':\n",
    "            ax1.set_visible(False)\n",
    "            ax2.set_position([1, 0.2, 0.05, 0.2])\n",
    "        else:\n",
    "            # Extract chunks for high and low analysis plot\n",
    "            high_chunks = plotdata[plotdata['ALARM_TYPE'] == 'High'][[metric, 'LENGTH']]\n",
    "\n",
    "            # Introduce mean value for each length\n",
    "            # Note: If mean value of metric is used, lines can be drawn again (with default of linestyle parameter)\n",
    "            #high_chunks = high_chunks.astype(float)\n",
    "            #high_chunks = high_chunks.groupby('LENGTH').mean()\n",
    "\n",
    "            # Reset indices to make access via column names possible again\n",
    "            #high_chunks.reset_index(level=0, inplace=True, drop=True)\n",
    "            high_chunks['LENGTH'] = high_chunks.index\n",
    "\n",
    "            print(high_chunks)\n",
    "\n",
    "            # Add left plot (high threshold analysis)\n",
    "            ax1.plot('LENGTH',\n",
    "                     metric,\n",
    "                     data=high_chunks,\n",
    "                     marker='o',\n",
    "                     color=sns.color_palette('colorblind')[0],\n",
    "                     linestyle='None')\n",
    "            ax1.set_title(f'{ylabel} Regarding High Thresholds', fontsize=10)\n",
    "            ax1.set_xlabel('Chunk Length', fontsize=8)\n",
    "            ax1.set_ylabel(f'{ylabel} of Chunk', fontsize=8)\n",
    "            ax1.set_ylim(bottom=0, top=1.1)\n",
    "\n",
    "        if endogenous_input == 'MAX':\n",
    "            ax2.set_visible(False)\n",
    "            ax1.set_position([0, 0.2, 0.05, 0.2])\n",
    "        else:\n",
    "            # Extract chunks for high and low analysis plot\n",
    "            low_chunks = plotdata[plotdata['ALARM_TYPE'] == 'Low'][[metric, 'LENGTH']]\n",
    "\n",
    "            # Introduce mean value for each length\n",
    "            # Note: If mean value of metric is used, lines can be drawn again (with default of linestyle parameter)\n",
    "            #low_chunks = low_chunks.astype(float)\n",
    "            #low_chunks = low_chunks.groupby('LENGTH').mean()\n",
    "\n",
    "            # Reset indices to make access via column names possible again\n",
    "            #low_chunks.reset_index(level=0, inplace=True, drop=True)\n",
    "            low_chunks['LENGTH'] = low_chunks.index\n",
    "\n",
    "            # Add right plot (low threshold analysis)\n",
    "            ax2.plot('LENGTH',\n",
    "                     metric,\n",
    "                     data=low_chunks,\n",
    "                     marker='o',\n",
    "                     color=sns.color_palette('colorblind')[1],\n",
    "                     linestyle='None')\n",
    "            ax2.set_title(f'{ylabel} Regarding Low Thresholds', fontsize=10)\n",
    "            ax2.set_xlabel('Chunk Length', fontsize=8)\n",
    "            ax2.set_ylabel(f'{ylabel} of Chunk', fontsize=8)\n",
    "            ax2.set_ylim(bottom=0, top=1.1)\n",
    "\n",
    "        # Improve layout and save figure\n",
    "        fig.tight_layout()\n",
    "        fig.show()\n",
    "        fig.savefig(f'../../plots/{approach}/{n_chunks}_chunks/{style}/rnn_results_correlation_chunk_length_and_{metric}_'\n",
    "                    f'{model_type}_{param}_{endogenous_input}_{version}{suffix}_curve.png', dpi=72)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Time-Series Plot of Chunk with Prediction\n",
    "\n",
    "Note: `chunk_ids_plotting` have to be adjusted manually."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "print(f'Original amount of chunks: {len(chunks_matrix)}\\n')\n",
    "\n",
    "interesting_chunks = chunks_matrix[chunks_matrix.FPR.notnull() & chunks_matrix.F1S.notnull()]\n",
    "print(f'Amount of interesting chunks: {len(interesting_chunks)}\\n')\n",
    "\n",
    "print(interesting_chunks[['CHUNK_ID', 'FPR', 'TPR', 'FNR', 'TNR', 'ACC', 'F1S', 'N_HIGH_ALARMS', 'N_LOW_ALARMS']])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from darts import TimeSeries\n",
    "import pandas as pd\n",
    "\n",
    "chunk_ids_plotting = ['200098.0_220277.0_2136-03-27 12:00:00', '200061.0_220277.0_2134-01-24 14:15:00']\n",
    "\n",
    "for chunk_id in chunk_ids_plotting:\n",
    "\n",
    "    # Format chunk IDs into Windows format that have to be used when loading from or saving to Windows machine\n",
    "    chunk_id_win10 = chunk_id.replace(':', '%3A')\n",
    "\n",
    "    # Extract predicted series of chunk\n",
    "    prediction_chunk_f = open(f'../../data/{approach}/{n_chunks}_chunks/{style}/{model_type}/{parameter}/{endogenous_input}/'\n",
    "                              f'05_prediction_{chunk_id_win10}_{version}{suffix}_window{window_idx}.pickle', 'rb')\n",
    "    prediction_chunk = pickle.load(prediction_chunk_f)\n",
    "    prediction_chunk_f.close()\n",
    "\n",
    "    # Convert predicted series of chunk to TimeSeries object\n",
    "    prediction_chunk = TimeSeries.from_dataframe(\n",
    "        df=prediction_chunk,\n",
    "        time_col='Time',\n",
    "        value_cols=['Value'],\n",
    "        freq='H')\n",
    "\n",
    "    # Extract original series of chunk\n",
    "    resampled_chunks = pd.read_parquet(f'../../data/resampling/resample_output_{parameter}_first{n_chunks}.parquet',\n",
    "                                       engine='pyarrow')\n",
    "    original_chunk = resampled_chunks[resampled_chunks['CHUNK_ID_FILLED_TH'] == chunk_id]\n",
    "\n",
    "    # Convert original series of chunk to TimeSeries object\n",
    "    original_chunk = TimeSeries.from_dataframe(\n",
    "        df=original_chunk,\n",
    "        time_col='CHARTTIME',\n",
    "        value_cols=[f'VITAL_PARAMTER_VALUE_{endogenous_input}_RESAMPLING'],\n",
    "        freq='H')\n",
    "\n",
    "    # Actual plot\n",
    "    sns.set_style('whitegrid')\n",
    "    plt.figure(figsize=(8, 5))\n",
    "    original_chunk.plot(label=f'{parameter.upper()} - actual')\n",
    "    prediction_chunk.plot(label=f'{parameter.upper()} - predicted')\n",
    "\n",
    "    # Adjust texts of plot\n",
    "    plt.legend()\n",
    "    plt.suptitle(f'Prediction of {parameter.upper()} with {n_chunks} Chunks, {endogenous_input} Input, and {model_type} '\n",
    "                 f'\\nModel ({style.replace(\"_\", \" \").upper()})', fontweight='bold')\n",
    "    plt.xlabel('Time')\n",
    "    plt.ylabel('Value')\n",
    "\n",
    "    plt.show()\n",
    "    plt.savefig(f'../../plots/{approach}/{n_chunks}_chunks/{style}/rnn_results_prediction_{model_type}_{parameter}_{endogenous_input}_'\n",
    "                f'{chunk_id_win10}_{version}{suffix}.png', dpi=72)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}