model_selection-classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from xgboost.sklearn import XGBClassifier \n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import GridSearchCV, StratifiedKFold\n",
    "from sklearn import metrics \n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "from utils import data_handler, plotter\n",
    "import time\n",
    "\n",
    "\n",
    "#handle warnings\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(clf, X_test,y_test):\n",
    "    y_pred = clf.predict(X_test)\n",
    "    c_mat = metrics.confusion_matrix(y_test,y_pred)\n",
    "    accuracy = (c_mat[0,0] + c_mat[1,1])/c_mat.sum()\n",
    "    sensitivity = c_mat[1,1]/(c_mat[1,0]+c_mat[1,1])  \n",
    "    specificity = c_mat[0,0]/(c_mat[0,0]+c_mat[0,1])  \n",
    "    try:\n",
    "        auc = metrics.roc_auc_score(y_true=y_test, y_score=clf.predict_proba(X_test))\n",
    "    except:\n",
    "        auc = metrics.roc_auc_score(y_true=y_test, y_score=clf.predict_proba(X_test)[:,1])\n",
    "\n",
    "    return np.array([auc,accuracy,sensitivity,specificity])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Handling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data Analysis - There are 183 Can and 117 Cannot, single-class guess accuracy is 61%\n",
    "\n",
    "Samples of two classes are separated such that latter can perform stratefied outer cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_df,Y_df = data_handler.load_XY()\n",
    "X = X_df.values\n",
    "Y = Y_df.values\n",
    "feature_list = X_df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generalization - Nested Cross Validatoin\n",
    "1. Outer CV: 10-fold cross validation (10 repetitions)\n",
    "2. Inner CV: 10-fold cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setup\n",
    "save_csv= True\n",
    "verbose = False\n",
    "n_jobs=4\n",
    "\n",
    "# cross validation settup\n",
    "Ntrials = 10\n",
    "outter_nsplit = 10\n",
    "inner_nsplit = 10\n",
    "\n",
    "print('start  ',str(Ntrials),' trials...')\n",
    "tot_count = Ntrials * outter_nsplit\n",
    "\n",
    "# Results store\n",
    "mlp_mat = np.zeros((tot_count,4))\n",
    "nb_mat = np.zeros((tot_count,4))\n",
    "svm_mat = np.zeros((tot_count,4))\n",
    "xgb_mat = np.zeros((tot_count,4))\n",
    "\n",
    "\n",
    "for i in range(Ntrials):\n",
    "    init_time = time.time()\n",
    "    print(\"trial = \",i)\n",
    "    train_index = []  \n",
    "    test_index = []  \n",
    "    \n",
    "    outer_cv = StratifiedKFold(n_splits=outter_nsplit, shuffle=True, random_state=i)\n",
    "    for train_ind,test_ind in outer_cv.split(X,Y):\n",
    "        train_index.append(train_ind.tolist())\n",
    "        test_index.append(test_ind.tolist())\n",
    "\n",
    "        \n",
    "    for j in range(outter_nsplit):#outter_nsplit\n",
    "        count = i * outter_nsplit + j\n",
    "        print(str(count), \"  / \",str(tot_count))\n",
    "        X_train = X[train_index[j]]\n",
    "        Y_train = Y[train_index[j]]\n",
    "        \n",
    "        X_test = X[test_index[j]]\n",
    "        Y_test = Y[test_index[j]]\n",
    "        \n",
    "         \n",
    "        inner_cv = StratifiedKFold(n_splits=inner_nsplit, shuffle=False, random_state=j)\n",
    "        \n",
    "        \n",
    "        #NB\n",
    "        nb_clf = GaussianNB()\n",
    "        tuned_parameters = dict(var_smoothing=[1e-12,1e-11,1e-10,1e-9,1e-8,1e-7,1e-6])\n",
    "        nb_cv = GridSearchCV(nb_clf, tuned_parameters, cv=inner_cv,scoring='roc_auc',verbose=0,n_jobs=n_jobs)\n",
    "        nb_cv.fit(X_train, Y_train)\n",
    "        nb_mat[count] = test(nb_cv,X_test,Y_test)\n",
    "\n",
    "\n",
    "        # MLP\n",
    "        mlp_clf = Pipeline([            \n",
    "                ('sc', StandardScaler()), \n",
    "                ('clf',  MLPClassifier())\n",
    "                ])\n",
    "        tuned_parameters = dict(clf__hidden_layer_sizes=[[5],[10],[20],[5,5],[10,10],[20,20],[5,5,5],[10,10,10],[20,20,20]],\n",
    "                          clf__alpha=[1e-4,1e-3, 1e-2, 1e-1,1], #L2 penalty (regularization term) parameter.\n",
    "                          clf__early_stopping=[True],\n",
    "                         clf__solver= ['lbfgs'])\n",
    "        mlp_cv = GridSearchCV(mlp_clf, tuned_parameters, cv=inner_cv,scoring='roc_auc',verbose=verbose,n_jobs=n_jobs)\n",
    "        mlp_cv.fit(X_train, Y_train)\n",
    "        mlp_mat[count] = test(mlp_cv,X_test,Y_test)\n",
    "\n",
    "        \n",
    "        # SVM - rbf\n",
    "        svm_clf = Pipeline([            \n",
    "                ('sc', StandardScaler()), \n",
    "                ('clf',  SVC(probability=True))\n",
    "                ])\n",
    "        tuned_parameters = dict(clf__kernel=['rbf'],\n",
    "                              clf__gamma=[1e-2,1e-1,'auto', 1, 1e1,1e2],\n",
    "                                clf__C=[1e-2,1e-1,1,1e1,1e2,1e3,1e4])\n",
    "        svm_cv = GridSearchCV(svm_clf, tuned_parameters, cv=inner_cv,scoring='roc_auc',verbose=verbose,n_jobs=n_jobs)\n",
    "        svm_cv.fit(X_train, Y_train)\n",
    "        svm_mat[count] = test(svm_cv,X_test,Y_test)\n",
    "        \n",
    "        \n",
    "        # XGBoost\n",
    "        xgb_clf = XGBClassifier(objective=\"binary:logistic\",min_child_weight=1,**{'tree_method':'exact'},\n",
    "                                 silent=True,n_jobs=4,random_state=3,seed=3);\n",
    "        tuned_parameters = dict(learning_rate=[0.01,0.1],\n",
    "                  n_estimators=[100, 300, 500],\n",
    "                  colsample_bylevel = [0.5,0.7,0.9],\n",
    "                  gamma=[0,0.2,0.4],\n",
    "                  max_depth =[3,5,7],\n",
    "                  reg_lambda = [0.1,1,10],\n",
    "                  subsample=[0.4,0.7,1])\n",
    "        xgb_cv = GridSearchCV(xgb_clf,tuned_parameters, cv=inner_cv,scoring='roc_auc',verbose=verbose,n_jobs=n_jobs)\n",
    "        xgb_cv.fit(X_train, Y_train)\n",
    "        xgb_mat[count] = test(xgb_cv,X_test,Y_test)\n",
    "        \n",
    "        if(verbose):\n",
    "            print(nb_mat[count])\n",
    "            print(mlp_mat[count])\n",
    "            print(svm_mat[count])\n",
    "            print(xgb_mat[count])\n",
    "  \n",
    "\n",
    "   \n",
    "    print((time.time()-init_time)/60, ' min')\n",
    "\n",
    "        \n",
    "# Results store\n",
    "svm_results = pd.DataFrame(data=svm_mat,columns=['AUROC','Accuracy','Sensitivity','Specificity'])\n",
    "nb_results = pd.DataFrame(data=nb_mat,columns=['AUROC','Accuracy','Sensitivity','Specificity'])\n",
    "mlp_results = pd.DataFrame(data=mlp_mat,columns=['AUROC','Accuracy','Sensitivity','Specificity'])\n",
    "xgb_results = pd.DataFrame(data=xgb_mat,columns=['AUROC','Accuracy','Sensitivity','Specificity'])        \n",
    "\n",
    "if(save_csv):\n",
    "    data_handler.save_csv(svm_results,title='[model_selection_clf]mos2_svm_results')\n",
    "    data_handler.save_csv(nb_results,title='[model_selection_clf]mos2_nb_results')\n",
    "    data_handler.save_csv(mlp_results,title='[model_selection_clf]mos2_mlp_results')\n",
    "    data_handler.save_csv(xgb_results,title='[model_selection_clf]mos2_xgb_results')\n",
    "\n",
    "\n",
    "print('end')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Determining Best Model\n",
    "1. Comparing 4 performance metrics \n",
    "    - visualizing by boxplots\n",
    "2. Bayesian correlated t-test\n",
    "    - to verify recognizable difference between the best model versus other three candidates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(44)\n",
    "\n",
    "print('->>>XGBoost_mean : \\n',xgb_results.mean(axis=0), '\\n  std = \\n', xgb_results.std(axis=0))\n",
    "print('->>>SVM_mean : \\n',svm_results.mean(axis=0),' \\n std =\\n',svm_results.std(axis=0) )\n",
    "print('->>>NB_mean : \\n',nb_results.mean(axis=0), ' \\n std =\\n',nb_results.std(axis=0))\n",
    "print('->>>MLP_mean : \\n',mlp_results.mean(axis=0), '\\n  std =\\n',mlp_results.std(axis=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Boxplot - AUROC/acc/sen/spe of 4 candidate classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "data = []\n",
    "data.append(xgb_results)\n",
    "data.append(mlp_results)\n",
    "data.append(svm_results)\n",
    "data.append(nb_results)\n",
    "\n",
    "plotter.plot_boxplots(data=data, ylabels = ['XGBoost-C','MLP-C','SVM-C','NB-C'],xmin=-0.025, toSaveFig=True,title='[model_selection_clf]mos2_')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bayesian correlated t-test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rope=0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = ( \"SVM-C\",\"XGBoost-C\")\n",
    "x=np.zeros((svm_results.shape[0],2),'float')\n",
    "x[:,1]=xgb_results['AUROC']\n",
    "x[:,0]=svm_results['AUROC']\n",
    "title = names[1]+' vs ' +names[0]+' on MoS2 dataset'\n",
    "left, within, right = plotter.plot_ttest(x, rope=rope,names= names, verbose=True,runs=Ntrials,title=title,toSaveFig=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = ( \"MLP-C\",\"XGBoost-C\")\n",
    "x=np.zeros((mlp_results.shape[0],2),'float')\n",
    "x[:,1]=xgb_results['AUROC']\n",
    "x[:,0]=mlp_results['AUROC']\n",
    "title = names[1]+' vs ' +names[0]+' on MoS2 dataset'\n",
    "left, within, right = plotter.plot_ttest(x, rope=rope,runs=Ntrials,verbose=True,names=names,title=title,toSaveFig=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = ( \"NB-C\",\"XGBoost-C\")\n",
    "x=np.zeros((nb_results.shape[0],2),'float')\n",
    "x[:,1]=xgb_results['AUROC']\n",
    "x[:,0]=nb_results['AUROC']\n",
    "title = names[1]+' vs ' +names[0]+' on MoS2 dataset'\n",
    "left, within, right = plotter.plot_ttest(x, rope=rope,runs=Ntrials,verbose=True,names=names,title=title,toSaveFig=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}