BayesianCAT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import RandomizedSearchCV, GridSearchCV\n",
    "from datetime import datetime\n",
    "from sklearn.ensemble import StackingClassifier, RandomForestClassifier\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.metrics import classification_report\n",
    "from lightgbm import LGBMClassifier\n",
    "from scipy.stats import randint as sp_randint\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from scipy.stats import uniform as sp_uniform\n",
    "from sklearn.model_selection import cross_val_score\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from skopt import BayesSearchCV\n",
    "from skopt.space import Real, Categorical, Integer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def average(df):\n",
    "    sum = 0\n",
    "    count = 0\n",
    "    for x in df['f1']:\n",
    "        if x!=-1:\n",
    "            sum+=x\n",
    "            count+=1\n",
    "    avg = sum/count\n",
    "    return avg\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def results(model, X_train, X_valid,y_train, y_valid):\n",
    "    soft_probs = model.predict_proba(X_valid)\n",
    "    valid_auc = roc_auc_score(y_valid, soft_probs[:,1])\n",
    "    print(valid_auc)\n",
    "    train_preds = model.predict(X_train)\n",
    "    acc = accuracy_score(y_train, train_preds)\n",
    "    print(acc)\n",
    "    preds = model.predict(X_valid)\n",
    "    acc = accuracy_score(y_valid, preds)\n",
    "    print(acc)\n",
    "    print(classification_report(y_valid,preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Y</th>\n",
       "      <th>f1</th>\n",
       "      <th>f2</th>\n",
       "      <th>f3</th>\n",
       "      <th>f4</th>\n",
       "      <th>f5</th>\n",
       "      <th>f6</th>\n",
       "      <th>f7</th>\n",
       "      <th>f8</th>\n",
       "      <th>...</th>\n",
       "      <th>f15</th>\n",
       "      <th>f16</th>\n",
       "      <th>f17</th>\n",
       "      <th>f18</th>\n",
       "      <th>f19</th>\n",
       "      <th>f20</th>\n",
       "      <th>f21</th>\n",
       "      <th>f22</th>\n",
       "      <th>f23</th>\n",
       "      <th>f24</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>1.638300e+04</td>\n",
       "      <td>16383.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>8192.000000</td>\n",
       "      <td>0.942135</td>\n",
       "      <td>43031.415720</td>\n",
       "      <td>1.044375</td>\n",
       "      <td>11.770938</td>\n",
       "      <td>118323.581456</td>\n",
       "      <td>1.044436</td>\n",
       "      <td>0.050052</td>\n",
       "      <td>117089.674113</td>\n",
       "      <td>169730.178600</td>\n",
       "      <td>...</td>\n",
       "      <td>25894.316914</td>\n",
       "      <td>119045.099005</td>\n",
       "      <td>184622.040835</td>\n",
       "      <td>1.047305</td>\n",
       "      <td>125959.667765</td>\n",
       "      <td>1.044558</td>\n",
       "      <td>1.045718</td>\n",
       "      <td>1.041934</td>\n",
       "      <td>3.271890e+04</td>\n",
       "      <td>1.043948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>4729.509065</td>\n",
       "      <td>0.233495</td>\n",
       "      <td>33596.053696</td>\n",
       "      <td>0.264806</td>\n",
       "      <td>353.187115</td>\n",
       "      <td>4518.059755</td>\n",
       "      <td>0.265601</td>\n",
       "      <td>0.293892</td>\n",
       "      <td>10261.292970</td>\n",
       "      <td>69396.677853</td>\n",
       "      <td>...</td>\n",
       "      <td>36086.993946</td>\n",
       "      <td>18321.987129</td>\n",
       "      <td>100590.811845</td>\n",
       "      <td>0.306239</td>\n",
       "      <td>31091.344158</td>\n",
       "      <td>0.262576</td>\n",
       "      <td>0.266874</td>\n",
       "      <td>0.246597</td>\n",
       "      <td>3.184929e+06</td>\n",
       "      <td>0.259640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>37.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.770000</td>\n",
       "      <td>23779.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4292.000000</td>\n",
       "      <td>4673.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>4674.000000</td>\n",
       "      <td>3130.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>117879.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>4096.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>20331.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.770000</td>\n",
       "      <td>118096.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>117961.000000</td>\n",
       "      <td>117906.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>4554.000000</td>\n",
       "      <td>118395.000000</td>\n",
       "      <td>118398.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>118274.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>8192.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>35530.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.770000</td>\n",
       "      <td>118300.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>117961.000000</td>\n",
       "      <td>128130.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>13234.000000</td>\n",
       "      <td>118929.000000</td>\n",
       "      <td>119095.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>118568.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>12287.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>74240.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.540000</td>\n",
       "      <td>118386.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>117961.000000</td>\n",
       "      <td>234498.500000</td>\n",
       "      <td>...</td>\n",
       "      <td>38902.000000</td>\n",
       "      <td>120539.000000</td>\n",
       "      <td>290919.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>120006.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>9.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>16383.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>312152.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>43910.160000</td>\n",
       "      <td>286791.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>311178.000000</td>\n",
       "      <td>311867.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>311696.000000</td>\n",
       "      <td>286792.000000</td>\n",
       "      <td>308574.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>311867.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>4.042886e+08</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Id             Y             f1            f2            f3  \\\n",
       "count  16383.000000  16383.000000   16383.000000  16383.000000  16383.000000   \n",
       "mean    8192.000000      0.942135   43031.415720      1.044375     11.770938   \n",
       "std     4729.509065      0.233495   33596.053696      0.264806    353.187115   \n",
       "min        1.000000      0.000000      37.000000      1.000000      1.770000   \n",
       "25%     4096.500000      1.000000   20331.000000      1.000000      1.770000   \n",
       "50%     8192.000000      1.000000   35530.000000      1.000000      1.770000   \n",
       "75%    12287.500000      1.000000   74240.500000      1.000000      3.540000   \n",
       "max    16383.000000      1.000000  312152.000000      7.000000  43910.160000   \n",
       "\n",
       "                  f4            f5            f6             f7  \\\n",
       "count   16383.000000  16383.000000  16383.000000   16383.000000   \n",
       "mean   118323.581456      1.044436      0.050052  117089.674113   \n",
       "std      4518.059755      0.265601      0.293892   10261.292970   \n",
       "min     23779.000000      1.000000      0.000000    4292.000000   \n",
       "25%    118096.000000      1.000000      0.000000  117961.000000   \n",
       "50%    118300.000000      1.000000      0.000000  117961.000000   \n",
       "75%    118386.000000      1.000000      0.000000  117961.000000   \n",
       "max    286791.000000      9.000000     10.000000  311178.000000   \n",
       "\n",
       "                  f8  ...            f15            f16            f17  \\\n",
       "count   16383.000000  ...   16383.000000   16383.000000   16383.000000   \n",
       "mean   169730.178600  ...   25894.316914  119045.099005  184622.040835   \n",
       "std     69396.677853  ...   36086.993946   18321.987129  100590.811845   \n",
       "min      4673.000000  ...      25.000000    4674.000000    3130.000000   \n",
       "25%    117906.000000  ...    4554.000000  118395.000000  118398.000000   \n",
       "50%    128130.000000  ...   13234.000000  118929.000000  119095.000000   \n",
       "75%    234498.500000  ...   38902.000000  120539.000000  290919.000000   \n",
       "max    311867.000000  ...  311696.000000  286792.000000  308574.000000   \n",
       "\n",
       "                f18            f19           f20           f21           f22  \\\n",
       "count  16383.000000   16383.000000  16383.000000  16383.000000  16383.000000   \n",
       "mean       1.047305  125959.667765      1.044558      1.045718      1.041934   \n",
       "std        0.306239   31091.344158      0.262576      0.266874      0.246597   \n",
       "min        1.000000  117879.000000      1.000000      1.000000      1.000000   \n",
       "25%        1.000000  118274.000000      1.000000      1.000000      1.000000   \n",
       "50%        1.000000  118568.000000      1.000000      1.000000      1.000000   \n",
       "75%        1.000000  120006.000000      1.000000      1.000000      1.000000   \n",
       "max       18.000000  311867.000000      8.000000      8.000000      7.000000   \n",
       "\n",
       "                f23           f24  \n",
       "count  1.638300e+04  16383.000000  \n",
       "mean   3.271890e+04      1.043948  \n",
       "std    3.184929e+06      0.259640  \n",
       "min    1.000000e+00      1.000000  \n",
       "25%    1.000000e+00      1.000000  \n",
       "50%    2.000000e+00      1.000000  \n",
       "75%    9.000000e+00      1.000000  \n",
       "max    4.042886e+08      8.000000  \n",
       "\n",
       "[8 rows x 26 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"train_final.csv\")\n",
    "df.describe()\n",
    "f1_avg = average(df)\n",
    "df['f1'].replace(-1,f1_avg, inplace = True)\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df.loc[:,'Y']\n",
    "X = df.loc[:,'f1':'f24']\n",
    "X['f3']=100*X['f3']\n",
    "X['f3'] = X['f3'].astype('int')\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X,y,train_size = 0.75, test_size = 0.25,random_state = 42, shuffle = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from catboost import CatBoostClassifier, Pool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\ttotal: 74.7ms\tremaining: 50.7s\n",
      "100:\ttotal: 5.3s\tremaining: 30.4s\n",
      "200:\ttotal: 13.2s\tremaining: 31.5s\n",
      "300:\ttotal: 20.8s\tremaining: 26.2s\n",
      "400:\ttotal: 30.8s\tremaining: 21.4s\n",
      "500:\ttotal: 38.5s\tremaining: 13.8s\n",
      "600:\ttotal: 45.1s\tremaining: 5.93s\n",
      "679:\ttotal: 49.4s\tremaining: 0us\n",
      "0.9005485813902007\n",
      "0.9724912509156018\n",
      "0.96484375\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.40      0.56       224\n",
      "           1       0.97      1.00      0.98      3872\n",
      "\n",
      "    accuracy                           0.96      4096\n",
      "   macro avg       0.93      0.70      0.77      4096\n",
      "weighted avg       0.96      0.96      0.96      4096\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# cat_indices = [1,3,4,5,6,7,8,9,10,12,14,15,16,17,18,19,20,21,22,23]\n",
    "cat_indices = [1,3,4,5,6,7,8,9,10,11,12,14,15,16,17,18,19,20,21,22,23]\n",
    "model_cat = CatBoostClassifier(cat_features=cat_indices,\n",
    "                           n_estimators=680,                         \n",
    "                           learning_rate =0.0169,\n",
    "                           random_state = 42,\n",
    "                           subsample = 0.66322,\n",
    "                           early_stopping_rounds = 30,\n",
    "                           max_depth = 8,\n",
    "                           one_hot_max_size = 10,\n",
    "                           verbose=100,\n",
    "                            bagging_temperature = 1.0,\n",
    "#                            scale_pos_weight = 0.05785,\n",
    "                           eval_metric='AUC')\n",
    "model_cat.fit(X_train,y_train)\n",
    "results(model_cat,X_train, X_valid, y_train, y_valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run Bayesian search CV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "params ={    'n_estimators':Integer(700,1500),\n",
    "             'learning_rate':Real(0.01,1.0,'log-uniform'),\n",
    "             'subsample': Real(0.4,0.8),\n",
    "             'max_depth': Integer(5,9),\n",
    "            'colsample_bylevel': Real(0.1, 0.8),\n",
    "           'bagging_temperature':Real(0.0, 1.0),\n",
    "         'l2_leaf_reg':Integer(2,30),\n",
    "         'random_strength': Real(1e-9, 10, 'log-uniform'),\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(n_splits=3, shuffle = True, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_indices = [1,3,4,5,6,7,8,9,10,11,12,14,15,16,17,18,19,20,21,22,23]\n",
    "model_cat_search = CatBoostClassifier(cat_features=cat_indices,\n",
    "#                            n_estimators=680,                         \n",
    "#                            learning_rate =0.0169,\n",
    "                           random_state = 42,\n",
    "#                            subsample = 0.66322,\n",
    "                           early_stopping_rounds = 30,\n",
    "#                            max_depth = 8,\n",
    "                           one_hot_max_size = 10,\n",
    "                           verbose=100,\n",
    "#                             bagging_temperature = 1.0,\n",
    "#                            scale_pos_weight = 0.05785,\n",
    "                           eval_metric='AUC')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "bayes_cv_tuner = BayesSearchCV(estimator = model_cat_search, scoring='roc_auc',\n",
    "                              search_spaces = params, cv = skf, n_jobs=-1,n_iter=100,\n",
    "                              refit=True, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def status_print(optim_result):\n",
    "    \"\"\"Status callback durring bayesian hyperparameter search\"\"\"\n",
    "\n",
    "    # Get all the models tested so far in DataFrame format\n",
    "    all_models = pd.DataFrame(bayes_cv_tuner.cv_results_)    \n",
    "\n",
    "    # Get current parameters and the best parameters    \n",
    "    best_params = pd.Series(bayes_cv_tuner.best_params_)\n",
    "    print('Model #{}\\nBest ROC-AUC: {}\\nBest params: {}\\n'.format(\n",
    "        len(all_models),\n",
    "        np.round(bayes_cv_tuner.best_score_, 4),\n",
    "        bayes_cv_tuner.best_params_\n",
    "    ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model #1\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #2\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #3\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #4\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #5\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #6\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #7\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #8\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #9\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #10\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #11\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #12\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #13\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #14\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #15\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #16\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #17\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #18\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #19\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #20\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n",
      "Model #21\n",
      "Best ROC-AUC: 0.8983\n",
      "Best params: OrderedDict([('bagging_temperature', 0.41010395885331385), ('colsample_bylevel', 0.6094080202241275), ('l2_leaf_reg', 28), ('learning_rate', 0.042815319280763466), ('max_depth', 8), ('n_estimators', 1031), ('random_strength', 3.230824361824754e-06), ('subsample', 0.6958016936761682)])\n",
      "\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-39-f8a22dcba040>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mresults_bayes\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbayes_cv_tuner\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mstatus_print\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\anaconda3\\envs\\dslab\\lib\\site-packages\\skopt\\searchcv.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, groups, callback)\u001b[0m\n\u001b[0;32m    690\u001b[0m                 \u001b[0mn_points_adjusted\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn_iter\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_points\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    691\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 692\u001b[1;33m                 optim_result = self._step(\n\u001b[0m\u001b[0;32m    693\u001b[0m                     \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msearch_space\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    694\u001b[0m                     \u001b[0mgroups\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mgroups\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_points\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mn_points_adjusted\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda3\\envs\\dslab\\lib\\site-packages\\skopt\\searchcv.py\u001b[0m in \u001b[0;36m_step\u001b[1;34m(self, X, y, search_space, optimizer, groups, n_points)\u001b[0m\n\u001b[0;32m    577\u001b[0m         \u001b[0mrefit\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrefit\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    578\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrefit\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 579\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgroups\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams_dict\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    580\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrefit\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrefit\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    581\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda3\\envs\\dslab\\lib\\site-packages\\skopt\\searchcv.py\u001b[0m in \u001b[0;36m_fit\u001b[1;34m(self, X, y, groups, parameter_iterable)\u001b[0m\n\u001b[0;32m    408\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    409\u001b[0m         \u001b[0mcv_iter\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgroups\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 410\u001b[1;33m         out = Parallel(\n\u001b[0m\u001b[0;32m    411\u001b[0m             \u001b[0mn_jobs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mn_jobs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mverbose\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    412\u001b[0m             \u001b[0mpre_dispatch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpre_dispatch\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda3\\envs\\dslab\\lib\\site-packages\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, iterable)\u001b[0m\n\u001b[0;32m   1040\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1041\u001b[0m             \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mretrieval_context\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1042\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1043\u001b[0m             \u001b[1;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1044\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda3\\envs\\dslab\\lib\\site-packages\\joblib\\parallel.py\u001b[0m in \u001b[0;36mretrieve\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    919\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    920\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'supports_timeout'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 921\u001b[1;33m                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    922\u001b[0m                 \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    923\u001b[0m                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda3\\envs\\dslab\\lib\\site-packages\\joblib\\_parallel_backends.py\u001b[0m in \u001b[0;36mwrap_future_result\u001b[1;34m(future, timeout)\u001b[0m\n\u001b[0;32m    540\u001b[0m         AsyncResults.get from multiprocessing.\"\"\"\n\u001b[0;32m    541\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 542\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    543\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mCfTimeoutError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    544\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda3\\envs\\dslab\\lib\\concurrent\\futures\\_base.py\u001b[0m in \u001b[0;36mresult\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    432\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__get_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    433\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 434\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_condition\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    435\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    436\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_state\u001b[0m \u001b[1;32min\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mCANCELLED\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mCANCELLED_AND_NOTIFIED\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda3\\envs\\dslab\\lib\\threading.py\u001b[0m in \u001b[0;36mwait\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    300\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m    \u001b[1;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    301\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 302\u001b[1;33m                 \u001b[0mwaiter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    303\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    304\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "results_bayes = bayes_cv_tuner.fit(X_train, y_train, callback=status_print)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Based on 40 iteration performances on colab gpu training, bayes search cv isnt much of a help. Move on to Optuna/hyperopt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}