LGBM_composition_2_creation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('../input/playground-series-s4e7/train.csv')\n",
    "test = pd.read_csv('../input/playground-series-s4e7/test.csv')\n",
    "ss = pd.read_csv('../input/playground-series-s4e7/sample_submission-2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import joblib\n",
    "dl135 = joblib.load('nn_oofs/denselight_5fold_oof_test_089135.jbl')\n",
    "dl161 = joblib.load('nn_oofs/denselight_5fold_oof_test_089161.jbl')\n",
    "\n",
    "ft150 = joblib.load('nn_oofs/fttransformer_5fold_oof_test_089150.jbl')\n",
    "rn157 = joblib.load('nn_oofs/resnet_5fold_oof_test_089157.jbl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "HOLDOUT = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "if HOLDOUT:\n",
    "    X_train, X_val = train_test_split(train, test_size=0.2, random_state=42, shuffle=True, stratify=train.Response)\n",
    "else:\n",
    "    X_train, X_val = train, test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.loc[:, 'ft150'] = ft150[0]\n",
    "X_train.loc[:, 'rn157'] = rn157[0]\n",
    "X_train.loc[:, 'dl161'] = dl161[0]\n",
    "X_train.loc[:, 'dl135'] = dl135[0]\n",
    "\n",
    "X_val.loc[:, 'ft150'] = ft150[1]\n",
    "X_val.loc[:, 'rn157'] = rn157[1]\n",
    "X_val.loc[:, 'dl161'] = dl161[1]\n",
    "X_val.loc[:, 'dl135'] = dl135[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>Driving_License</th>\n",
       "      <th>Region_Code</th>\n",
       "      <th>Previously_Insured</th>\n",
       "      <th>Vehicle_Age</th>\n",
       "      <th>Vehicle_Damage</th>\n",
       "      <th>Annual_Premium</th>\n",
       "      <th>Policy_Sales_Channel</th>\n",
       "      <th>Vintage</th>\n",
       "      <th>Response</th>\n",
       "      <th>ft150</th>\n",
       "      <th>rn157</th>\n",
       "      <th>dl161</th>\n",
       "      <th>dl135</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Male</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1-2 Year</td>\n",
       "      <td>Yes</td>\n",
       "      <td>65101.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>0.260343</td>\n",
       "      <td>0.330062</td>\n",
       "      <td>0.310721</td>\n",
       "      <td>0.328448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Male</td>\n",
       "      <td>43</td>\n",
       "      <td>1</td>\n",
       "      <td>28.0</td>\n",
       "      <td>0</td>\n",
       "      <td>&gt; 2 Years</td>\n",
       "      <td>Yes</td>\n",
       "      <td>58911.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>288</td>\n",
       "      <td>1</td>\n",
       "      <td>0.466775</td>\n",
       "      <td>0.495850</td>\n",
       "      <td>0.502867</td>\n",
       "      <td>0.501508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Female</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1</td>\n",
       "      <td>&lt; 1 Year</td>\n",
       "      <td>No</td>\n",
       "      <td>38043.0</td>\n",
       "      <td>152.0</td>\n",
       "      <td>254</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000228</td>\n",
       "      <td>0.000112</td>\n",
       "      <td>0.000068</td>\n",
       "      <td>0.000182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Female</td>\n",
       "      <td>35</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1-2 Year</td>\n",
       "      <td>Yes</td>\n",
       "      <td>2630.0</td>\n",
       "      <td>156.0</td>\n",
       "      <td>76</td>\n",
       "      <td>0</td>\n",
       "      <td>0.141679</td>\n",
       "      <td>0.175906</td>\n",
       "      <td>0.159691</td>\n",
       "      <td>0.167480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Female</td>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>15.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1-2 Year</td>\n",
       "      <td>No</td>\n",
       "      <td>31951.0</td>\n",
       "      <td>152.0</td>\n",
       "      <td>294</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000313</td>\n",
       "      <td>0.000266</td>\n",
       "      <td>0.000029</td>\n",
       "      <td>0.000041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11504793</th>\n",
       "      <td>11504793</td>\n",
       "      <td>Male</td>\n",
       "      <td>48</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1-2 Year</td>\n",
       "      <td>Yes</td>\n",
       "      <td>27412.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>218</td>\n",
       "      <td>0</td>\n",
       "      <td>0.474919</td>\n",
       "      <td>0.396422</td>\n",
       "      <td>0.464265</td>\n",
       "      <td>0.410795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11504794</th>\n",
       "      <td>11504794</td>\n",
       "      <td>Female</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt; 1 Year</td>\n",
       "      <td>Yes</td>\n",
       "      <td>29509.0</td>\n",
       "      <td>152.0</td>\n",
       "      <td>115</td>\n",
       "      <td>1</td>\n",
       "      <td>0.214196</td>\n",
       "      <td>0.301742</td>\n",
       "      <td>0.282953</td>\n",
       "      <td>0.337362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11504795</th>\n",
       "      <td>11504795</td>\n",
       "      <td>Female</td>\n",
       "      <td>29</td>\n",
       "      <td>1</td>\n",
       "      <td>32.0</td>\n",
       "      <td>1</td>\n",
       "      <td>&lt; 1 Year</td>\n",
       "      <td>No</td>\n",
       "      <td>2630.0</td>\n",
       "      <td>152.0</td>\n",
       "      <td>189</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000433</td>\n",
       "      <td>0.000215</td>\n",
       "      <td>0.000233</td>\n",
       "      <td>0.000342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11504796</th>\n",
       "      <td>11504796</td>\n",
       "      <td>Female</td>\n",
       "      <td>51</td>\n",
       "      <td>1</td>\n",
       "      <td>28.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1-2 Year</td>\n",
       "      <td>Yes</td>\n",
       "      <td>48443.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>274</td>\n",
       "      <td>1</td>\n",
       "      <td>0.192171</td>\n",
       "      <td>0.182069</td>\n",
       "      <td>0.186144</td>\n",
       "      <td>0.209001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11504797</th>\n",
       "      <td>11504797</td>\n",
       "      <td>Male</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>28.0</td>\n",
       "      <td>1</td>\n",
       "      <td>&lt; 1 Year</td>\n",
       "      <td>No</td>\n",
       "      <td>32855.0</td>\n",
       "      <td>152.0</td>\n",
       "      <td>189</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000490</td>\n",
       "      <td>0.000085</td>\n",
       "      <td>0.000190</td>\n",
       "      <td>0.000192</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11504798 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                id  Gender  Age  Driving_License  Region_Code  \\\n",
       "0                0    Male   21                1         35.0   \n",
       "1                1    Male   43                1         28.0   \n",
       "2                2  Female   25                1         14.0   \n",
       "3                3  Female   35                1          1.0   \n",
       "4                4  Female   36                1         15.0   \n",
       "...            ...     ...  ...              ...          ...   \n",
       "11504793  11504793    Male   48                1          6.0   \n",
       "11504794  11504794  Female   26                1         36.0   \n",
       "11504795  11504795  Female   29                1         32.0   \n",
       "11504796  11504796  Female   51                1         28.0   \n",
       "11504797  11504797    Male   25                1         28.0   \n",
       "\n",
       "          Previously_Insured Vehicle_Age Vehicle_Damage  Annual_Premium  \\\n",
       "0                          0    1-2 Year            Yes         65101.0   \n",
       "1                          0   > 2 Years            Yes         58911.0   \n",
       "2                          1    < 1 Year             No         38043.0   \n",
       "3                          0    1-2 Year            Yes          2630.0   \n",
       "4                          1    1-2 Year             No         31951.0   \n",
       "...                      ...         ...            ...             ...   \n",
       "11504793                   0    1-2 Year            Yes         27412.0   \n",
       "11504794                   0    < 1 Year            Yes         29509.0   \n",
       "11504795                   1    < 1 Year             No          2630.0   \n",
       "11504796                   0    1-2 Year            Yes         48443.0   \n",
       "11504797                   1    < 1 Year             No         32855.0   \n",
       "\n",
       "          Policy_Sales_Channel  Vintage  Response     ft150     rn157  \\\n",
       "0                        124.0      187         0  0.260343  0.330062   \n",
       "1                         26.0      288         1  0.466775  0.495850   \n",
       "2                        152.0      254         0  0.000228  0.000112   \n",
       "3                        156.0       76         0  0.141679  0.175906   \n",
       "4                        152.0      294         0  0.000313  0.000266   \n",
       "...                        ...      ...       ...       ...       ...   \n",
       "11504793                  26.0      218         0  0.474919  0.396422   \n",
       "11504794                 152.0      115         1  0.214196  0.301742   \n",
       "11504795                 152.0      189         0  0.000433  0.000215   \n",
       "11504796                  26.0      274         1  0.192171  0.182069   \n",
       "11504797                 152.0      189         0  0.000490  0.000085   \n",
       "\n",
       "             dl161     dl135  \n",
       "0         0.310721  0.328448  \n",
       "1         0.502867  0.501508  \n",
       "2         0.000068  0.000182  \n",
       "3         0.159691  0.167480  \n",
       "4         0.000029  0.000041  \n",
       "...            ...       ...  \n",
       "11504793  0.464265  0.410795  \n",
       "11504794  0.282953  0.337362  \n",
       "11504795  0.000233  0.000342  \n",
       "11504796  0.186144  0.209001  \n",
       "11504797  0.000190  0.000192  \n",
       "\n",
       "[11504798 rows x 16 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:46:39] Stdout logging level is DEBUG.\n",
      "[23:46:39] Task: binary\n",
      "\n",
      "[23:46:39] Start automl preset with listed constraints:\n",
      "[23:46:39] - time: 2160000.00 seconds\n",
      "[23:46:39] - CPU: 12 cores\n",
      "[23:46:39] - memory: 16 GB\n",
      "\n",
      "[23:46:39] \u001b[1mTrain data shape: (11504798, 16)\u001b[0m\n",
      "\n",
      "[23:46:53] Feats was rejected during automatic roles guess: []\n",
      "[23:46:54] Layer \u001b[1m1\u001b[0m train process start. Time left 2159984.49 secs\n",
      "[23:53:39] Start fitting \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m ...\n",
      "[23:53:39] Training params: {'task': 'train', 'learning_rate': 0.05, 'num_leaves': 244, 'feature_fraction': 0.7, 'bagging_fraction': 0.7, 'bagging_freq': 1, 'max_depth': -1, 'verbosity': -1, 'reg_alpha': 1, 'reg_lambda': 0.0, 'min_split_gain': 0.0, 'zero_as_missing': False, 'num_threads': 12, 'max_bin': 255, 'min_data_in_bin': 3, 'num_trees': 2000, 'early_stopping_rounds': 100, 'random_state': 42}\n",
      "[23:53:40] ===== Start working with \u001b[1mfold 0\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[23:53:43] Training until validation scores don't improve for 100 rounds\n",
      "[23:54:09] [100]\tvalid's auc: 0.892843\n",
      "[23:54:36] [200]\tvalid's auc: 0.892362\n",
      "[23:54:44] Early stopping, best iteration is:\n",
      "[130]\tvalid's auc: 0.892916\n",
      "[23:54:46] ===== Start working with \u001b[1mfold 1\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[23:54:49] Training until validation scores don't improve for 100 rounds\n",
      "[23:55:14] [100]\tvalid's auc: 0.892431\n",
      "[23:55:42] [200]\tvalid's auc: 0.891592\n",
      "[23:55:49] Early stopping, best iteration is:\n",
      "[129]\tvalid's auc: 0.892489\n",
      "[23:55:52] ===== Start working with \u001b[1mfold 2\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[23:55:54] Training until validation scores don't improve for 100 rounds\n",
      "[23:56:21] [100]\tvalid's auc: 0.892583\n",
      "[23:56:49] [200]\tvalid's auc: 0.89159\n",
      "[23:56:55] Early stopping, best iteration is:\n",
      "[120]\tvalid's auc: 0.892641\n",
      "[23:56:57] ===== Start working with \u001b[1mfold 3\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[23:57:00] Training until validation scores don't improve for 100 rounds\n",
      "[23:57:26] [100]\tvalid's auc: 0.892674\n",
      "[23:57:54] [200]\tvalid's auc: 0.892207\n",
      "[23:58:01] Early stopping, best iteration is:\n",
      "[125]\tvalid's auc: 0.892735\n",
      "[23:58:03] ===== Start working with \u001b[1mfold 4\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[23:58:06] Training until validation scores don't improve for 100 rounds\n",
      "[23:58:32] [100]\tvalid's auc: 0.893226\n",
      "[23:59:00] [200]\tvalid's auc: 0.892645\n",
      "[23:59:10] Early stopping, best iteration is:\n",
      "[135]\tvalid's auc: 0.893316\n",
      "[23:59:16] Fitting \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m finished. score = \u001b[1m0.892746022465867\u001b[0m\n",
      "[23:59:16] \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m fitting and predicting completed\n",
      "[23:59:16] Time left 2159243.10 secs\n",
      "\n",
      "[23:59:16] \u001b[1mLayer 1 training completed.\u001b[0m\n",
      "\n",
      "[23:59:16] \u001b[1mAutoml preset training completed in 757.01 seconds\u001b[0m\n",
      "\n",
      "[23:59:16] Model description:\n",
      "Final prediction for new objects (level 0) = \n",
      "\t 1.00000 * (5 averaged models Lvl_0_Pipe_0_Mod_0_LightGBM) \n",
      "\n"
     ]
    }
   ],
   "source": [
    "task = Task('binary') #‘binary’ \n",
    "automl = TabularAutoML(\n",
    "    task = task,\n",
    "    timeout = 600 * 3600,\n",
    "    cpu_limit = 12,\n",
    "    general_params = {'use_algos':[['lgb']]},\n",
    "    selection_params ={'mode' : 0},\n",
    "    tuning_params = {'max_tuning_time': 0},\n",
    "    reader_params = {'n_jobs': 12, 'cv': 5, 'random_state': 42}\n",
    ")\n",
    "\n",
    "out_of_fold_predictions1 = automl.fit_predict(\n",
    "    X_train,\n",
    "    roles = {\n",
    "        'target': 'Response',\n",
    "        'drop': ['id']\n",
    "    }, \n",
    "    verbose = 4\n",
    ")\n",
    "\n",
    "val_predict = automl.predict(X_val)\n",
    "if HOLDOUT:\n",
    "    roc_auc_score(X_val.Response, val_predict.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.892746022465867"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(X_train.Response, out_of_fold_predictions1.data[:, 0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2 binary model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[00:01:39] Stdout logging level is DEBUG.\n",
      "[00:01:39] Task: binary\n",
      "\n",
      "[00:01:39] Start automl preset with listed constraints:\n",
      "[00:01:39] - time: 2160000.00 seconds\n",
      "[00:01:39] - CPU: 12 cores\n",
      "[00:01:39] - memory: 16 GB\n",
      "\n",
      "[00:01:39] \u001b[1mTrain data shape: (6178116, 16)\u001b[0m\n",
      "\n",
      "[00:01:49] Feats was rejected during automatic roles guess: []\n",
      "[00:01:49] Layer \u001b[1m1\u001b[0m train process start. Time left 2159989.54 secs\n",
      "[00:05:22] Start fitting \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m ...\n",
      "[00:05:22] Training params: {'task': 'train', 'learning_rate': 0.05, 'num_leaves': 244, 'feature_fraction': 0.7, 'bagging_fraction': 0.7, 'bagging_freq': 1, 'max_depth': -1, 'verbosity': -1, 'reg_alpha': 1, 'reg_lambda': 0.0, 'min_split_gain': 0.0, 'zero_as_missing': False, 'num_threads': 12, 'max_bin': 255, 'min_data_in_bin': 3, 'num_trees': 2000, 'early_stopping_rounds': 100, 'random_state': 42}\n",
      "[00:05:22] ===== Start working with \u001b[1mfold 0\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:05:24] Training until validation scores don't improve for 100 rounds\n",
      "[00:05:45] [100]\tvalid's auc: 0.775245\n",
      "[00:06:00] [200]\tvalid's auc: 0.775609\n",
      "[00:06:14] [300]\tvalid's auc: 0.775654\n",
      "[00:06:26] [400]\tvalid's auc: 0.775663\n",
      "[00:06:38] Early stopping, best iteration is:\n",
      "[388]\tvalid's auc: 0.775665\n",
      "[00:06:41] ===== Start working with \u001b[1mfold 1\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:06:43] Training until validation scores don't improve for 100 rounds\n",
      "[00:07:03] [100]\tvalid's auc: 0.77544\n",
      "[00:07:19] [200]\tvalid's auc: 0.775781\n",
      "[00:07:32] [300]\tvalid's auc: 0.775846\n",
      "[00:07:45] [400]\tvalid's auc: 0.775868\n",
      "[00:07:53] Early stopping, best iteration is:\n",
      "[363]\tvalid's auc: 0.775886\n",
      "[00:07:57] ===== Start working with \u001b[1mfold 2\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:07:58] Training until validation scores don't improve for 100 rounds\n",
      "[00:08:18] [100]\tvalid's auc: 0.774676\n",
      "[00:08:34] [200]\tvalid's auc: 0.775007\n",
      "[00:08:46] [300]\tvalid's auc: 0.775068\n",
      "[00:08:58] [400]\tvalid's auc: 0.775089\n",
      "[00:09:09] Early stopping, best iteration is:\n",
      "[386]\tvalid's auc: 0.775098\n",
      "[00:09:13] ===== Start working with \u001b[1mfold 3\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:09:14] Training until validation scores don't improve for 100 rounds\n",
      "[00:09:34] [100]\tvalid's auc: 0.775001\n",
      "[00:09:49] [200]\tvalid's auc: 0.775315\n",
      "[00:10:03] [300]\tvalid's auc: 0.775402\n",
      "[00:10:16] [400]\tvalid's auc: 0.775425\n",
      "[00:10:29] [500]\tvalid's auc: 0.775409\n",
      "[00:10:30] Early stopping, best iteration is:\n",
      "[410]\tvalid's auc: 0.77543\n",
      "[00:10:34] ===== Start working with \u001b[1mfold 4\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:10:36] Training until validation scores don't improve for 100 rounds\n",
      "[00:10:53] [100]\tvalid's auc: 0.775396\n",
      "[00:11:07] [200]\tvalid's auc: 0.775686\n",
      "[00:11:20] [300]\tvalid's auc: 0.775702\n",
      "[00:11:26] Early stopping, best iteration is:\n",
      "[246]\tvalid's auc: 0.775728\n",
      "[00:11:31] Fitting \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m finished. score = \u001b[1m0.7755528307265607\u001b[0m\n",
      "[00:11:31] \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m fitting and predicting completed\n",
      "[00:11:31] Time left 2159408.27 secs\n",
      "\n",
      "[00:11:31] \u001b[1mLayer 1 training completed.\u001b[0m\n",
      "\n",
      "[00:11:31] \u001b[1mAutoml preset training completed in 591.79 seconds\u001b[0m\n",
      "\n",
      "[00:11:31] Model description:\n",
      "Final prediction for new objects (level 0) = \n",
      "\t 1.00000 * (5 averaged models Lvl_0_Pipe_0_Mod_0_LightGBM) \n",
      "\n",
      "[00:11:31] Stdout logging level is DEBUG.\n",
      "[00:11:31] Task: binary\n",
      "\n",
      "[00:11:31] Start automl preset with listed constraints:\n",
      "[00:11:31] - time: 2160000.00 seconds\n",
      "[00:11:31] - CPU: 12 cores\n",
      "[00:11:31] - memory: 16 GB\n",
      "\n",
      "[00:11:31] \u001b[1mTrain data shape: (5326682, 16)\u001b[0m\n",
      "\n",
      "[00:11:40] Feats was rejected during automatic roles guess: []\n",
      "[00:11:41] Layer \u001b[1m1\u001b[0m train process start. Time left 2159990.73 secs\n",
      "[00:14:16] Start fitting \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m ...\n",
      "[00:14:16] Training params: {'task': 'train', 'learning_rate': 0.05, 'num_leaves': 244, 'feature_fraction': 0.7, 'bagging_fraction': 0.7, 'bagging_freq': 1, 'max_depth': -1, 'verbosity': -1, 'reg_alpha': 1, 'reg_lambda': 0.0, 'min_split_gain': 0.0, 'zero_as_missing': False, 'num_threads': 12, 'max_bin': 255, 'min_data_in_bin': 3, 'num_trees': 2000, 'early_stopping_rounds': 100, 'random_state': 42}\n",
      "[00:14:16] ===== Start working with \u001b[1mfold 0\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:14:17] Training until validation scores don't improve for 100 rounds\n",
      "[00:14:29] [100]\tvalid's auc: 0.826458\n",
      "[00:14:29] Early stopping, best iteration is:\n",
      "[1]\tvalid's auc: 0.851216\n",
      "[00:14:29] ===== Start working with \u001b[1mfold 1\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:14:30] Training until validation scores don't improve for 100 rounds\n",
      "[00:14:42] [100]\tvalid's auc: 0.855042\n",
      "[00:14:55] [200]\tvalid's auc: 0.853469\n",
      "[00:14:59] Early stopping, best iteration is:\n",
      "[132]\tvalid's auc: 0.856796\n",
      "[00:15:00] ===== Start working with \u001b[1mfold 2\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:15:01] Training until validation scores don't improve for 100 rounds\n",
      "[00:15:12] [100]\tvalid's auc: 0.87144\n",
      "[00:15:26] [200]\tvalid's auc: 0.867549\n",
      "[00:15:29] Early stopping, best iteration is:\n",
      "[127]\tvalid's auc: 0.872808\n",
      "[00:15:31] ===== Start working with \u001b[1mfold 3\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:15:32] Training until validation scores don't improve for 100 rounds\n",
      "[00:15:44] [100]\tvalid's auc: 0.835557\n",
      "[00:15:57] [200]\tvalid's auc: 0.847104\n",
      "[00:16:05] Early stopping, best iteration is:\n",
      "[159]\tvalid's auc: 0.850914\n",
      "[00:16:07] ===== Start working with \u001b[1mfold 4\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:16:08] Training until validation scores don't improve for 100 rounds\n",
      "[00:16:19] [100]\tvalid's auc: 0.85596\n",
      "[00:16:32] [200]\tvalid's auc: 0.863003\n",
      "[00:16:44] Early stopping, best iteration is:\n",
      "[190]\tvalid's auc: 0.864781\n",
      "[00:16:47] Fitting \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m finished. score = \u001b[1m0.8043599417172912\u001b[0m\n",
      "[00:16:47] \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m fitting and predicting completed\n",
      "[00:16:47] Time left 2159684.67 secs\n",
      "\n",
      "[00:16:47] \u001b[1mLayer 1 training completed.\u001b[0m\n",
      "\n",
      "[00:16:47] \u001b[1mAutoml preset training completed in 315.39 seconds\u001b[0m\n",
      "\n",
      "[00:16:47] Model description:\n",
      "Final prediction for new objects (level 0) = \n",
      "\t 1.00000 * (5 averaged models Lvl_0_Pipe_0_Mod_0_LightGBM) \n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "task = Task('binary') #‘binary’ \n",
    "automl1 = TabularAutoML(\n",
    "    task = task,\n",
    "    timeout = 600 * 3600,\n",
    "    cpu_limit = 12,\n",
    "    general_params = {'use_algos':[['lgb']]},\n",
    "    selection_params ={'mode' : 0},\n",
    "    tuning_params = {'max_tuning_time': 0},\n",
    "    reader_params = {'n_jobs': 12, 'cv': 5, 'random_state': 42}\n",
    ")\n",
    "\n",
    "out_of_fold_predictions1_1 = automl1.fit_predict(\n",
    "    X_train[X_train.Previously_Insured==0], \n",
    "    roles = {\n",
    "        'target': 'Response',\n",
    "        'drop': ['id']\n",
    "    }, \n",
    "    verbose = 4\n",
    ")\n",
    "\n",
    "\n",
    "automl2 = TabularAutoML(\n",
    "    task = task,\n",
    "    timeout = 600 * 3600,\n",
    "    cpu_limit = 12,\n",
    "    general_params = {'use_algos':[['lgb']]},\n",
    "    selection_params ={'mode' : 0},\n",
    "    tuning_params = {'max_tuning_time': 0},\n",
    "    reader_params = {'n_jobs': 12, 'cv': 5, 'random_state': 42}\n",
    ")\n",
    "out_of_fold_predictions2_1 = automl2.fit_predict(\n",
    "    X_train[X_train.Previously_Insured==1],\n",
    "    roles = {\n",
    "        'target': 'Response',\n",
    "        'drop': ['id']\n",
    "    }, \n",
    "    verbose = 4\n",
    ")\n",
    "\n",
    "val_predict1 = automl1.predict(X_val[X_val.Previously_Insured==0])\n",
    "val_predict2 = automl2.predict(X_val[X_val.Previously_Insured==1])\n",
    "\n",
    "\n",
    "y_pred = pd.Series(index=X_val.index)\n",
    "y_pred.iloc[X_val.Previously_Insured==0] = val_predict1.data[:, 0]\n",
    "y_pred.iloc[X_val.Previously_Insured==1] = val_predict2.data[:, 0]\n",
    "\n",
    "oof1 = pd.Series(index=X_train.index)\n",
    "oof1.iloc[X_train.Previously_Insured==0] = out_of_fold_predictions1_1.data[:, 0]\n",
    "oof1.iloc[X_train.Previously_Insured==1] = out_of_fold_predictions2_1.data[:, 0]\n",
    "\n",
    "if HOLDOUT:\n",
    "    roc_auc_score(X_val.Response, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8914516316380822"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(X_train.Response, oof1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[00:19:48] Stdout logging level is DEBUG.\n",
      "[00:19:48] Task: binary\n",
      "\n",
      "[00:19:48] Start automl preset with listed constraints:\n",
      "[00:19:48] - time: 2160000.00 seconds\n",
      "[00:19:48] - CPU: 12 cores\n",
      "[00:19:48] - memory: 16 GB\n",
      "\n",
      "[00:19:48] \u001b[1mTrain data shape: (5783229, 16)\u001b[0m\n",
      "\n",
      "[00:19:57] Feats was rejected during automatic roles guess: []\n",
      "[00:19:57] Layer \u001b[1m1\u001b[0m train process start. Time left 2159990.43 secs\n",
      "[00:23:15] Start fitting \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m ...\n",
      "[00:23:15] Training params: {'task': 'train', 'learning_rate': 0.05, 'num_leaves': 244, 'feature_fraction': 0.7, 'bagging_fraction': 0.7, 'bagging_freq': 1, 'max_depth': -1, 'verbosity': -1, 'reg_alpha': 1, 'reg_lambda': 0.0, 'min_split_gain': 0.0, 'zero_as_missing': False, 'num_threads': 12, 'max_bin': 255, 'min_data_in_bin': 3, 'num_trees': 2000, 'early_stopping_rounds': 100, 'random_state': 42}\n",
      "[00:23:15] ===== Start working with \u001b[1mfold 0\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:23:17] Training until validation scores don't improve for 100 rounds\n",
      "[00:23:36] [100]\tvalid's auc: 0.764818\n",
      "[00:23:50] [200]\tvalid's auc: 0.765112\n",
      "[00:24:03] [300]\tvalid's auc: 0.765178\n",
      "[00:24:15] [400]\tvalid's auc: 0.765201\n",
      "[00:24:26] Early stopping, best iteration is:\n",
      "[390]\tvalid's auc: 0.765232\n",
      "[00:24:29] ===== Start working with \u001b[1mfold 1\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:24:31] Training until validation scores don't improve for 100 rounds\n",
      "[00:24:46] [100]\tvalid's auc: 0.765163\n",
      "[00:24:59] [200]\tvalid's auc: 0.765481\n",
      "[00:25:10] [300]\tvalid's auc: 0.765581\n",
      "[00:25:21] [400]\tvalid's auc: 0.76561\n",
      "[00:25:31] Early stopping, best iteration is:\n",
      "[390]\tvalid's auc: 0.765618\n",
      "[00:25:34] ===== Start working with \u001b[1mfold 2\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:25:36] Training until validation scores don't improve for 100 rounds\n",
      "[00:25:55] [100]\tvalid's auc: 0.765517\n",
      "[00:26:09] [200]\tvalid's auc: 0.765804\n",
      "[00:26:21] [300]\tvalid's auc: 0.765848\n",
      "[00:26:33] [400]\tvalid's auc: 0.765869\n",
      "[00:26:43] Early stopping, best iteration is:\n",
      "[385]\tvalid's auc: 0.765878\n",
      "[00:26:46] ===== Start working with \u001b[1mfold 3\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:26:48] Training until validation scores don't improve for 100 rounds\n",
      "[00:27:03] [100]\tvalid's auc: 0.765484\n",
      "[00:27:16] [200]\tvalid's auc: 0.765809\n",
      "[00:27:28] [300]\tvalid's auc: 0.765848\n",
      "[00:27:39] [400]\tvalid's auc: 0.765852\n",
      "[00:27:47] Early stopping, best iteration is:\n",
      "[370]\tvalid's auc: 0.76586\n",
      "[00:27:51] ===== Start working with \u001b[1mfold 4\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:27:52] Training until validation scores don't improve for 100 rounds\n",
      "[00:28:08] [100]\tvalid's auc: 0.764405\n",
      "[00:28:21] [200]\tvalid's auc: 0.76468\n",
      "[00:28:33] [300]\tvalid's auc: 0.764749\n",
      "[00:28:41] Early stopping, best iteration is:\n",
      "[271]\tvalid's auc: 0.76476\n",
      "[00:28:45] Fitting \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m finished. score = \u001b[1m0.7654607888889398\u001b[0m\n",
      "[00:28:45] \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m fitting and predicting completed\n",
      "[00:28:45] Time left 2159462.95 secs\n",
      "\n",
      "[00:28:45] \u001b[1mLayer 1 training completed.\u001b[0m\n",
      "\n",
      "[00:28:45] \u001b[1mAutoml preset training completed in 537.09 seconds\u001b[0m\n",
      "\n",
      "[00:28:45] Model description:\n",
      "Final prediction for new objects (level 0) = \n",
      "\t 1.00000 * (5 averaged models Lvl_0_Pipe_0_Mod_0_LightGBM) \n",
      "\n",
      "[00:28:46] Stdout logging level is DEBUG.\n",
      "[00:28:46] Task: binary\n",
      "\n",
      "[00:28:46] Start automl preset with listed constraints:\n",
      "[00:28:46] - time: 2160000.00 seconds\n",
      "[00:28:46] - CPU: 12 cores\n",
      "[00:28:46] - memory: 16 GB\n",
      "\n",
      "[00:28:46] \u001b[1mTrain data shape: (5721569, 16)\u001b[0m\n",
      "\n",
      "[00:28:55] Feats was rejected during automatic roles guess: []\n",
      "[00:28:55] Layer \u001b[1m1\u001b[0m train process start. Time left 2159990.72 secs\n",
      "[00:30:01] Start fitting \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m ...\n",
      "[00:30:01] Training params: {'task': 'train', 'learning_rate': 0.05, 'num_leaves': 244, 'feature_fraction': 0.7, 'bagging_fraction': 0.7, 'bagging_freq': 1, 'max_depth': -1, 'verbosity': -1, 'reg_alpha': 1, 'reg_lambda': 0.0, 'min_split_gain': 0.0, 'zero_as_missing': False, 'num_threads': 12, 'max_bin': 255, 'min_data_in_bin': 3, 'num_trees': 2000, 'early_stopping_rounds': 100, 'random_state': 42}\n",
      "[00:30:01] ===== Start working with \u001b[1mfold 0\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:30:02] Training until validation scores don't improve for 100 rounds\n",
      "[00:30:13] [100]\tvalid's auc: 0.959543\n",
      "[00:30:18] Early stopping, best iteration is:\n",
      "[40]\tvalid's auc: 0.961111\n",
      "[00:30:19] ===== Start working with \u001b[1mfold 1\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:30:20] Training until validation scores don't improve for 100 rounds\n",
      "[00:30:30] [100]\tvalid's auc: 0.958911\n",
      "[00:30:38] Early stopping, best iteration is:\n",
      "[62]\tvalid's auc: 0.959875\n",
      "[00:30:38] ===== Start working with \u001b[1mfold 2\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:30:39] Training until validation scores don't improve for 100 rounds\n",
      "[00:30:49] [100]\tvalid's auc: 0.955715\n",
      "[00:30:56] Early stopping, best iteration is:\n",
      "[50]\tvalid's auc: 0.957307\n",
      "[00:30:57] ===== Start working with \u001b[1mfold 3\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:30:58] Training until validation scores don't improve for 100 rounds\n",
      "[00:31:08] [100]\tvalid's auc: 0.956478\n",
      "[00:31:12] Early stopping, best iteration is:\n",
      "[28]\tvalid's auc: 0.956944\n",
      "[00:31:12] ===== Start working with \u001b[1mfold 4\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:31:13] Training until validation scores don't improve for 100 rounds\n",
      "[00:31:24] [100]\tvalid's auc: 0.957111\n",
      "[00:31:31] Early stopping, best iteration is:\n",
      "[49]\tvalid's auc: 0.958458\n",
      "[00:31:32] Fitting \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m finished. score = \u001b[1m0.9550321575669853\u001b[0m\n",
      "[00:31:32] \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m fitting and predicting completed\n",
      "[00:31:32] Time left 2159833.66 secs\n",
      "\n",
      "[00:31:32] \u001b[1mLayer 1 training completed.\u001b[0m\n",
      "\n",
      "[00:31:32] \u001b[1mAutoml preset training completed in 166.38 seconds\u001b[0m\n",
      "\n",
      "[00:31:32] Model description:\n",
      "Final prediction for new objects (level 0) = \n",
      "\t 1.00000 * (5 averaged models Lvl_0_Pipe_0_Mod_0_LightGBM) \n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "task = Task('binary') #‘binary’ \n",
    "automl1_1 = TabularAutoML(\n",
    "    task = task,\n",
    "    timeout = 600 * 3600,\n",
    "    cpu_limit = 12,\n",
    "    general_params = {'use_algos':[['lgb']]},\n",
    "    selection_params ={'mode' : 0},\n",
    "    tuning_params = {'max_tuning_time': 0},\n",
    "    reader_params = {'n_jobs': 12, 'cv': 5, 'random_state': 42}\n",
    ")\n",
    "\n",
    "out_of_fold_predictions1_2 = automl1_1.fit_predict(\n",
    "    X_train[X_train.Vehicle_Damage=='Yes'], \n",
    "    roles = {\n",
    "        'target': 'Response',\n",
    "        'drop': ['id']\n",
    "    }, \n",
    "    verbose = 4\n",
    ")\n",
    "\n",
    "\n",
    "automl2_1 = TabularAutoML(\n",
    "    task = task,\n",
    "    timeout = 600 * 3600,\n",
    "    cpu_limit = 12,\n",
    "    general_params = {'use_algos':[['lgb']]},\n",
    "    selection_params ={'mode' : 0},\n",
    "    tuning_params = {'max_tuning_time': 0},\n",
    "    reader_params = {'n_jobs': 12, 'cv': 5, 'random_state': 42}\n",
    ")\n",
    "out_of_fold_predictions2_2 = automl2_1.fit_predict(\n",
    "    X_train[X_train.Vehicle_Damage=='No'],\n",
    "    roles = {\n",
    "        'target': 'Response',\n",
    "        'drop': ['id']\n",
    "    }, \n",
    "    verbose = 4\n",
    ")\n",
    "\n",
    "val_predict1_2 = automl1_1.predict(X_val[X_val.Vehicle_Damage=='Yes'])\n",
    "val_predict2_2 = automl2_1.predict(X_val[X_val.Vehicle_Damage=='No'])\n",
    "\n",
    "\n",
    "y_pred2 = pd.Series(index=X_val.index)\n",
    "y_pred2.iloc[X_val.Vehicle_Damage=='Yes'] = val_predict1_2.data[:, 0]\n",
    "y_pred2.iloc[X_val.Vehicle_Damage=='No'] = val_predict2_2.data[:, 0]\n",
    "\n",
    "oof2 = pd.Series(index=X_train.index)\n",
    "oof2.iloc[X_train.Vehicle_Damage=='Yes'] = out_of_fold_predictions1_2.data[:, 0]\n",
    "oof2.iloc[X_train.Vehicle_Damage=='No'] = out_of_fold_predictions2_2.data[:, 0]\n",
    "\n",
    "if HOLDOUT:\n",
    "    roc_auc_score(X_val.Response, y_pred2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8927603717013254"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(X_train.Response, oof2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "if HOLDOUT:\n",
    "    roc_auc_score(X_val.Response, y_pred+y_pred2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.892033343062512"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(X_train.Response, oof1+oof2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[00:33:57] Stdout logging level is DEBUG.\n",
      "[00:33:57] Task: binary\n",
      "\n",
      "[00:33:57] Start automl preset with listed constraints:\n",
      "[00:33:57] - time: 2160000.00 seconds\n",
      "[00:33:57] - CPU: 12 cores\n",
      "[00:33:57] - memory: 16 GB\n",
      "\n",
      "[00:33:57] \u001b[1mTrain data shape: (5504102, 16)\u001b[0m\n",
      "\n",
      "[00:34:06] Feats was rejected during automatic roles guess: []\n",
      "[00:34:07] Layer \u001b[1m1\u001b[0m train process start. Time left 2159990.26 secs\n",
      "[00:37:23] Start fitting \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m ...\n",
      "[00:37:23] Training params: {'task': 'train', 'learning_rate': 0.05, 'num_leaves': 244, 'feature_fraction': 0.7, 'bagging_fraction': 0.7, 'bagging_freq': 1, 'max_depth': -1, 'verbosity': -1, 'reg_alpha': 1, 'reg_lambda': 0.0, 'min_split_gain': 0.0, 'zero_as_missing': False, 'num_threads': 12, 'max_bin': 255, 'min_data_in_bin': 3, 'num_trees': 2000, 'early_stopping_rounds': 100, 'random_state': 42}\n",
      "[00:37:23] ===== Start working with \u001b[1mfold 0\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:37:25] Training until validation scores don't improve for 100 rounds\n",
      "[00:37:40] [100]\tvalid's auc: 0.751058\n",
      "[00:37:52] [200]\tvalid's auc: 0.751391\n",
      "[00:38:03] [300]\tvalid's auc: 0.751452\n",
      "[00:38:14] [400]\tvalid's auc: 0.751461\n",
      "[00:38:24] [500]\tvalid's auc: 0.751407\n",
      "[00:38:27] Early stopping, best iteration is:\n",
      "[428]\tvalid's auc: 0.751464\n",
      "[00:38:31] ===== Start working with \u001b[1mfold 1\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:38:32] Training until validation scores don't improve for 100 rounds\n",
      "[00:38:47] [100]\tvalid's auc: 0.74998\n",
      "[00:38:59] [200]\tvalid's auc: 0.750306\n",
      "[00:39:09] [300]\tvalid's auc: 0.750334\n",
      "[00:39:20] [400]\tvalid's auc: 0.750162\n",
      "[00:39:22] Early stopping, best iteration is:\n",
      "[320]\tvalid's auc: 0.750341\n",
      "[00:39:25] ===== Start working with \u001b[1mfold 2\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:39:27] Training until validation scores don't improve for 100 rounds\n",
      "[00:39:42] [100]\tvalid's auc: 0.749398\n",
      "[00:39:54] [200]\tvalid's auc: 0.74972\n",
      "[00:40:05] [300]\tvalid's auc: 0.749791\n",
      "[00:40:15] [400]\tvalid's auc: 0.749801\n",
      "[00:40:19] Early stopping, best iteration is:\n",
      "[338]\tvalid's auc: 0.749827\n",
      "[00:40:22] ===== Start working with \u001b[1mfold 3\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:40:23] Training until validation scores don't improve for 100 rounds\n",
      "[00:40:38] [100]\tvalid's auc: 0.749897\n",
      "[00:40:50] [200]\tvalid's auc: 0.750211\n",
      "[00:41:01] [300]\tvalid's auc: 0.750303\n",
      "[00:41:11] [400]\tvalid's auc: 0.750272\n",
      "[00:41:14] Early stopping, best iteration is:\n",
      "[329]\tvalid's auc: 0.750314\n",
      "[00:41:17] ===== Start working with \u001b[1mfold 4\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:41:18] Training until validation scores don't improve for 100 rounds\n",
      "[00:41:32] [100]\tvalid's auc: 0.750749\n",
      "[00:41:44] [200]\tvalid's auc: 0.751132\n",
      "[00:41:54] [300]\tvalid's auc: 0.75122\n",
      "[00:42:04] [400]\tvalid's auc: 0.751203\n",
      "[00:42:10] Early stopping, best iteration is:\n",
      "[357]\tvalid's auc: 0.751232\n",
      "[00:42:14] Fitting \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m finished. score = \u001b[1m0.7506324845971126\u001b[0m\n",
      "[00:42:14] \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m fitting and predicting completed\n",
      "[00:42:14] Time left 2159502.46 secs\n",
      "\n",
      "[00:42:14] \u001b[1mLayer 1 training completed.\u001b[0m\n",
      "\n",
      "[00:42:14] \u001b[1mAutoml preset training completed in 497.57 seconds\u001b[0m\n",
      "\n",
      "[00:42:14] Model description:\n",
      "Final prediction for new objects (level 0) = \n",
      "\t 1.00000 * (5 averaged models Lvl_0_Pipe_0_Mod_0_LightGBM) \n",
      "\n",
      "[00:42:15] Stdout logging level is DEBUG.\n",
      "[00:42:15] Task: binary\n",
      "\n",
      "[00:42:15] Start automl preset with listed constraints:\n",
      "[00:42:15] - time: 2160000.00 seconds\n",
      "[00:42:15] - CPU: 12 cores\n",
      "[00:42:15] - memory: 16 GB\n",
      "\n",
      "[00:42:15] \u001b[1mTrain data shape: (674014, 16)\u001b[0m\n",
      "\n",
      "[00:42:20] Feats was rejected during automatic roles guess: []\n",
      "[00:42:20] Layer \u001b[1m1\u001b[0m train process start. Time left 2159994.39 secs\n",
      "[00:42:42] Start fitting \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m ...\n",
      "[00:42:42] Training params: {'task': 'train', 'learning_rate': 0.05, 'num_leaves': 244, 'feature_fraction': 0.7, 'bagging_fraction': 0.7, 'bagging_freq': 1, 'max_depth': -1, 'verbosity': -1, 'reg_alpha': 1, 'reg_lambda': 0.0, 'min_split_gain': 0.0, 'zero_as_missing': False, 'num_threads': 12, 'max_bin': 255, 'min_data_in_bin': 3, 'num_trees': 2000, 'early_stopping_rounds': 100, 'random_state': 42}\n",
      "[00:42:42] ===== Start working with \u001b[1mfold 0\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:42:43] Training until validation scores don't improve for 100 rounds\n",
      "[00:42:45] [100]\tvalid's auc: 0.777141\n",
      "[00:42:45] Early stopping, best iteration is:\n",
      "[29]\tvalid's auc: 0.778798\n",
      "[00:42:45] ===== Start working with \u001b[1mfold 1\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:42:46] Training until validation scores don't improve for 100 rounds\n",
      "[00:42:48] [100]\tvalid's auc: 0.787186\n",
      "[00:42:50] [200]\tvalid's auc: 0.787396\n",
      "[00:42:50] Early stopping, best iteration is:\n",
      "[110]\tvalid's auc: 0.787551\n",
      "[00:42:50] ===== Start working with \u001b[1mfold 2\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:42:50] Training until validation scores don't improve for 100 rounds\n",
      "[00:42:53] [100]\tvalid's auc: 0.781972\n",
      "[00:42:53] Early stopping, best iteration is:\n",
      "[19]\tvalid's auc: 0.78372\n",
      "[00:42:53] ===== Start working with \u001b[1mfold 3\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:42:53] Training until validation scores don't improve for 100 rounds\n",
      "[00:42:56] [100]\tvalid's auc: 0.784345\n",
      "[00:42:58] [200]\tvalid's auc: 0.784237\n",
      "[00:42:59] Early stopping, best iteration is:\n",
      "[149]\tvalid's auc: 0.784625\n",
      "[00:43:00] ===== Start working with \u001b[1mfold 4\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:43:00] Training until validation scores don't improve for 100 rounds\n",
      "[00:43:02] [100]\tvalid's auc: 0.791371\n",
      "[00:43:03] Early stopping, best iteration is:\n",
      "[38]\tvalid's auc: 0.793466\n",
      "[00:43:03] Fitting \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m finished. score = \u001b[1m0.78082296299612\u001b[0m\n",
      "[00:43:03] \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m fitting and predicting completed\n",
      "[00:43:03] Time left 2159951.66 secs\n",
      "\n",
      "[00:43:03] \u001b[1mLayer 1 training completed.\u001b[0m\n",
      "\n",
      "[00:43:03] \u001b[1mAutoml preset training completed in 48.34 seconds\u001b[0m\n",
      "\n",
      "[00:43:03] Model description:\n",
      "Final prediction for new objects (level 0) = \n",
      "\t 1.00000 * (5 averaged models Lvl_0_Pipe_0_Mod_0_LightGBM) \n",
      "\n",
      "[00:43:03] Stdout logging level is DEBUG.\n",
      "[00:43:03] Task: binary\n",
      "\n",
      "[00:43:03] Start automl preset with listed constraints:\n",
      "[00:43:03] - time: 2160000.00 seconds\n",
      "[00:43:03] - CPU: 12 cores\n",
      "[00:43:03] - memory: 16 GB\n",
      "\n",
      "[00:43:03] \u001b[1mTrain data shape: (279127, 16)\u001b[0m\n",
      "\n",
      "[00:43:09] Feats was rejected during automatic roles guess: []\n",
      "[00:43:09] Layer \u001b[1m1\u001b[0m train process start. Time left 2159994.05 secs\n",
      "[00:43:18] Start fitting \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m ...\n",
      "[00:43:18] Training params: {'task': 'train', 'learning_rate': 0.04, 'num_leaves': 128, 'feature_fraction': 0.7, 'bagging_fraction': 0.7, 'bagging_freq': 1, 'max_depth': -1, 'verbosity': -1, 'reg_alpha': 1, 'reg_lambda': 0.0, 'min_split_gain': 0.0, 'zero_as_missing': False, 'num_threads': 12, 'max_bin': 255, 'min_data_in_bin': 3, 'num_trees': 2000, 'early_stopping_rounds': 100, 'random_state': 42}\n",
      "[00:43:18] ===== Start working with \u001b[1mfold 0\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:43:18] Training until validation scores don't improve for 100 rounds\n",
      "[00:43:19] [100]\tvalid's auc: 0.746403\n",
      "[00:43:20] [200]\tvalid's auc: 0.747363\n",
      "[00:43:20] Early stopping, best iteration is:\n",
      "[131]\tvalid's auc: 0.748537\n",
      "[00:43:21] ===== Start working with \u001b[1mfold 1\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:43:21] Training until validation scores don't improve for 100 rounds\n",
      "[00:43:22] [100]\tvalid's auc: 0.765994\n",
      "[00:43:23] [200]\tvalid's auc: 0.768856\n",
      "[00:43:24] [300]\tvalid's auc: 0.767446\n",
      "[00:43:24] Early stopping, best iteration is:\n",
      "[238]\tvalid's auc: 0.770255\n",
      "[00:43:24] ===== Start working with \u001b[1mfold 2\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:43:24] Training until validation scores don't improve for 100 rounds\n",
      "[00:43:25] [100]\tvalid's auc: 0.755749\n",
      "[00:43:26] Early stopping, best iteration is:\n",
      "[12]\tvalid's auc: 0.761482\n",
      "[00:43:26] ===== Start working with \u001b[1mfold 3\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:43:26] Training until validation scores don't improve for 100 rounds\n",
      "[00:43:27] [100]\tvalid's auc: 0.75813\n",
      "[00:43:28] [200]\tvalid's auc: 0.760602\n",
      "[00:43:28] Early stopping, best iteration is:\n",
      "[166]\tvalid's auc: 0.761527\n",
      "[00:43:29] ===== Start working with \u001b[1mfold 4\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:43:29] Training until validation scores don't improve for 100 rounds\n",
      "[00:43:30] [100]\tvalid's auc: 0.731506\n",
      "[00:43:31] [200]\tvalid's auc: 0.736298\n",
      "[00:43:31] Early stopping, best iteration is:\n",
      "[160]\tvalid's auc: 0.736741\n",
      "[00:43:31] Fitting \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m finished. score = \u001b[1m0.7326746266452022\u001b[0m\n",
      "[00:43:31] \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m fitting and predicting completed\n",
      "[00:43:31] Time left 2159971.52 secs\n",
      "\n",
      "[00:43:31] \u001b[1mLayer 1 training completed.\u001b[0m\n",
      "\n",
      "[00:43:31] \u001b[1mAutoml preset training completed in 28.48 seconds\u001b[0m\n",
      "\n",
      "[00:43:31] Model description:\n",
      "Final prediction for new objects (level 0) = \n",
      "\t 1.00000 * (5 averaged models Lvl_0_Pipe_0_Mod_0_LightGBM) \n",
      "\n",
      "[00:43:31] Stdout logging level is DEBUG.\n",
      "[00:43:31] Task: binary\n",
      "\n",
      "[00:43:31] Start automl preset with listed constraints:\n",
      "[00:43:31] - time: 2160000.00 seconds\n",
      "[00:43:31] - CPU: 12 cores\n",
      "[00:43:31] - memory: 16 GB\n",
      "\n",
      "[00:43:31] \u001b[1mTrain data shape: (5047555, 16)\u001b[0m\n",
      "\n",
      "[00:43:40] Feats was rejected during automatic roles guess: []\n",
      "[00:43:40] Layer \u001b[1m1\u001b[0m train process start. Time left 2159991.14 secs\n",
      "[00:46:30] Start fitting \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m ...\n",
      "[00:46:30] Training params: {'task': 'train', 'learning_rate': 0.05, 'num_leaves': 244, 'feature_fraction': 0.7, 'bagging_fraction': 0.7, 'bagging_freq': 1, 'max_depth': -1, 'verbosity': -1, 'reg_alpha': 1, 'reg_lambda': 0.0, 'min_split_gain': 0.0, 'zero_as_missing': False, 'num_threads': 12, 'max_bin': 255, 'min_data_in_bin': 3, 'num_trees': 2000, 'early_stopping_rounds': 100, 'random_state': 42}\n",
      "[00:46:30] ===== Start working with \u001b[1mfold 0\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:46:31] Training until validation scores don't improve for 100 rounds\n",
      "[00:46:44] [100]\tvalid's auc: 0.647529\n",
      "[00:46:44] Early stopping, best iteration is:\n",
      "[1]\tvalid's auc: 0.776666\n",
      "[00:46:44] ===== Start working with \u001b[1mfold 1\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:46:45] Training until validation scores don't improve for 100 rounds\n",
      "[00:46:59] [100]\tvalid's auc: 0.698804\n",
      "[00:46:59] Early stopping, best iteration is:\n",
      "[1]\tvalid's auc: 0.778186\n",
      "[00:46:59] ===== Start working with \u001b[1mfold 2\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:47:00] Training until validation scores don't improve for 100 rounds\n",
      "[00:47:12] [100]\tvalid's auc: 0.615849\n",
      "[00:47:13] Early stopping, best iteration is:\n",
      "[1]\tvalid's auc: 0.746283\n",
      "[00:47:13] ===== Start working with \u001b[1mfold 3\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:47:14] Training until validation scores don't improve for 100 rounds\n",
      "[00:47:27] [100]\tvalid's auc: 0.654589\n",
      "[00:47:27] Early stopping, best iteration is:\n",
      "[1]\tvalid's auc: 0.763972\n",
      "[00:47:27] ===== Start working with \u001b[1mfold 4\u001b[0m for \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m =====\n",
      "[00:47:29] Training until validation scores don't improve for 100 rounds\n",
      "[00:47:40] [100]\tvalid's auc: 0.746927\n",
      "[00:47:52] [200]\tvalid's auc: 0.740617\n",
      "[00:47:54] Early stopping, best iteration is:\n",
      "[118]\tvalid's auc: 0.776835\n",
      "[00:47:56] Fitting \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m finished. score = \u001b[1m0.6853023193229876\u001b[0m\n",
      "[00:47:56] \u001b[1mLvl_0_Pipe_0_Mod_0_LightGBM\u001b[0m fitting and predicting completed\n",
      "[00:47:56] Time left 2159735.01 secs\n",
      "\n",
      "[00:47:56] \u001b[1mLayer 1 training completed.\u001b[0m\n",
      "\n",
      "[00:47:56] \u001b[1mAutoml preset training completed in 265.02 seconds\u001b[0m\n",
      "\n",
      "[00:47:56] Model description:\n",
      "Final prediction for new objects (level 0) = \n",
      "\t 1.00000 * (5 averaged models Lvl_0_Pipe_0_Mod_0_LightGBM) \n",
      "\n"
     ]
    }
   ],
   "source": [
    "Xt1 = X_train[X_train.Previously_Insured==0]\n",
    "Xt1 = Xt1[Xt1.Vehicle_Damage=='Yes']\n",
    "\n",
    "Xt2 = X_train[X_train.Previously_Insured==0]\n",
    "Xt2 = Xt2[Xt2.Vehicle_Damage=='No']\n",
    "\n",
    "Xt3 = X_train[X_train.Previously_Insured==1]\n",
    "Xt3 = Xt3[Xt3.Vehicle_Damage=='Yes']\n",
    "\n",
    "Xt4 = X_train[X_train.Previously_Insured==1]\n",
    "Xt4 = Xt4[Xt4.Vehicle_Damage=='No']\n",
    "\n",
    "Xv1 = X_val[X_val.Previously_Insured==0]\n",
    "Xv1 = Xv1[Xv1.Vehicle_Damage=='Yes']\n",
    "\n",
    "Xv2 = X_val[X_val.Previously_Insured==0]\n",
    "Xv2 = Xv2[Xv2.Vehicle_Damage=='No']\n",
    "\n",
    "Xv3 = X_val[X_val.Previously_Insured==1]\n",
    "Xv3 = Xv3[Xv3.Vehicle_Damage=='Yes']\n",
    "\n",
    "Xv4 = X_val[X_val.Previously_Insured==1]\n",
    "Xv4 = Xv4[Xv4.Vehicle_Damage=='No']\n",
    "\n",
    "task = Task('binary') #‘binary’ \n",
    "\n",
    "automl1_3 = TabularAutoML(\n",
    "    task = task,\n",
    "    timeout = 600 * 3600,\n",
    "    cpu_limit = 12,\n",
    "    general_params = {'use_algos':[['lgb']]},\n",
    "    selection_params ={'mode' : 0},\n",
    "    tuning_params = {'max_tuning_time': 0},\n",
    "    reader_params = {'n_jobs': 12, 'cv': 5, 'random_state': 42}\n",
    ")\n",
    "\n",
    "out_of_fold_predictions1_3 = automl1_3.fit_predict(\n",
    "    Xt1, \n",
    "    roles = {\n",
    "        'target': 'Response',\n",
    "        'drop': ['id']\n",
    "    }, \n",
    "    verbose = 4\n",
    ")\n",
    "\n",
    "automl2_3 = TabularAutoML(\n",
    "    task = task,\n",
    "    timeout = 600 * 3600,\n",
    "    cpu_limit = 12,\n",
    "    general_params = {'use_algos':[['lgb']]},\n",
    "    selection_params ={'mode' : 0},\n",
    "    tuning_params = {'max_tuning_time': 0},\n",
    "    reader_params = {'n_jobs': 12, 'cv': 5, 'random_state': 42}\n",
    ")\n",
    "\n",
    "out_of_fold_predictions2_3 = automl2_3.fit_predict(\n",
    "    Xt2, \n",
    "    roles = {\n",
    "        'target': 'Response',\n",
    "        'drop': ['id']\n",
    "    }, \n",
    "    verbose = 4\n",
    ")\n",
    "\n",
    "automl3_3 = TabularAutoML(\n",
    "    task = task,\n",
    "    timeout = 600 * 3600,\n",
    "    cpu_limit = 12,\n",
    "    general_params = {'use_algos':[['lgb']]},\n",
    "    selection_params ={'mode' : 0},\n",
    "    tuning_params = {'max_tuning_time': 0},\n",
    "    reader_params = {'n_jobs': 12, 'cv': 5, 'random_state': 42}\n",
    ")\n",
    "\n",
    "out_of_fold_predictions3_3 = automl3_3.fit_predict(\n",
    "    Xt3,\n",
    "    roles = {\n",
    "        'target': 'Response',\n",
    "        'drop': ['id']\n",
    "    }, \n",
    "    verbose = 4\n",
    ")\n",
    "\n",
    "automl4_3 = TabularAutoML(\n",
    "    task = task,\n",
    "    timeout = 600 * 3600,\n",
    "    cpu_limit = 12,\n",
    "    general_params = {'use_algos':[['lgb']]},\n",
    "    selection_params ={'mode' : 0},\n",
    "    tuning_params = {'max_tuning_time': 0},\n",
    "    reader_params = {'n_jobs': 12, 'cv': 5, 'random_state': 42}\n",
    ")\n",
    "\n",
    "out_of_fold_predictions4_3 = automl4_3.fit_predict(\n",
    "    Xt4,\n",
    "    roles = {\n",
    "        'target': 'Response',\n",
    "        'drop': ['id']\n",
    "    }, \n",
    "    verbose = 4\n",
    ")\n",
    "\n",
    "val_predict1_3 = automl1_3.predict(Xv1)\n",
    "val_predict2_3 = automl2_3.predict(Xv2)\n",
    "val_predict3_3 = automl3_3.predict(Xv3)\n",
    "val_predict4_3 = automl4_3.predict(Xv4)\n",
    "\n",
    "y_pred3 = pd.Series(index=X_val.index)\n",
    "y_pred3.loc[Xv1.index] = val_predict1_3.data[:, 0]\n",
    "y_pred3.loc[Xv2.index] = val_predict2_3.data[:, 0]\n",
    "y_pred3.loc[Xv3.index] = val_predict3_3.data[:, 0]\n",
    "y_pred3.loc[Xv4.index] = val_predict4_3.data[:, 0]\n",
    "\n",
    "oof3 = pd.Series(index=X_train.index)\n",
    "oof3.loc[Xt1.index] = out_of_fold_predictions1_3.data[:, 0]\n",
    "oof3.loc[Xt2.index] = out_of_fold_predictions2_3.data[:, 0]\n",
    "oof3.loc[Xt3.index] = out_of_fold_predictions3_3.data[:, 0]\n",
    "oof3.loc[Xt4.index] = out_of_fold_predictions4_3.data[:, 0]\n",
    "\n",
    "\n",
    "if HOLDOUT:\n",
    "    roc_auc_score(X_val.Response, y_pred3.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8922743333965447"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(X_train.Response, oof3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "if HOLDOUT:\n",
    "    roc_auc_score(X_val.Response, y_pred.values + y_pred2.values + y_pred3.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['gbdts_stack_with_nn.jbl']"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(\n",
    "    {'m1_full_train': automl,\n",
    "    'm2': {'Previously_Insured0': automl1, 'Previously_Insured1': automl2},\n",
    "    'm3': {'Vehicle_DamageY': automl1_1, 'Vehicle_DamageN':automl2_1},\n",
    "    'm4': {'PI0VDY': automl1_3, 'PI0VDN':automl2_3, 'PI1VDY':automl3_3, 'PI1VDN':automl4_3}},\n",
    "     'gbdts_stack_with_nn.jbl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:root] *",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}