notebooks/6_lightgbm_bayesian_optimization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-20T12:28:32.643440Z",
     "start_time": "2018-05-20T12:28:32.606527Z"
    }
   },
   "source": [
    "### LightGBM and Bayesian Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-28T19:40:33.809284Z",
     "start_time": "2019-06-28T19:40:32.703440Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import warnings\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from datetime import datetime\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "import lightgbm as lgb\n",
    "from bayes_opt import BayesianOptimization\n",
    "from bayes_opt.observer import JSONLogger\n",
    "from bayes_opt.event import Events\n",
    "\n",
    "import utils\n",
    "from constants import DATA_DIR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-28T19:40:33.818849Z",
     "start_time": "2019-06-28T19:40:33.811927Z"
    }
   },
   "outputs": [],
   "source": [
    "np.random.seed(23)\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LightGBM and Hyperparameter Tuning with Bayesian Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-28T19:40:33.851224Z",
     "start_time": "2019-06-28T19:40:33.826148Z"
    }
   },
   "outputs": [],
   "source": [
    "def bayes_parameter_opt_lgb(X, y, init_round, opt_round, n_folds, random_seed):\n",
    "    train_data = lgb.Dataset(data=X, label=y)\n",
    "\n",
    "    def lgb_eval(\n",
    "        feature_fraction,\n",
    "        bagging_fraction,\n",
    "        lambda_l1,\n",
    "        lambda_l2,\n",
    "        max_depth,\n",
    "        num_leaves,\n",
    "        min_split_gain,\n",
    "        min_child_weight,\n",
    "        learning_rate,\n",
    "        n_estimators,\n",
    "    ):\n",
    "        params = {\n",
    "            \"objective\": \"regression\",\n",
    "            \"max_bin\": 255,\n",
    "            \"bagging_freq\": 1,\n",
    "            \"min_child_samples\": 20,\n",
    "            \"boosting\": \"gbdt\",\n",
    "            \"verbosity\": 1,\n",
    "            \"early_stopping_round\": 200,\n",
    "            \"metric\": \"rmse\",\n",
    "        }\n",
    "\n",
    "        params[\"feature_fraction\"] = max(min(feature_fraction, 1), 0)\n",
    "        params[\"bagging_fraction\"] = max(min(bagging_fraction, 1), 0)\n",
    "        params[\"lambda_l1\"] = max(lambda_l1, 0)\n",
    "        params[\"lambda_l2\"] = max(lambda_l2, 0)\n",
    "        params[\"max_depth\"] = int(round(max_depth))\n",
    "        params[\"num_leaves\"] = int(round(num_leaves))\n",
    "        params[\"min_split_gain\"] = min_split_gain\n",
    "        params[\"min_child_weight\"] = min_child_weight\n",
    "        params[\"learning_rate\"] = learning_rate\n",
    "        params[\"n_estimators\"] = int(round(n_estimators))\n",
    "\n",
    "        cv_result = lgb.cv(\n",
    "            params,\n",
    "            train_data,\n",
    "            nfold=n_folds,\n",
    "            seed=random_seed,\n",
    "            verbose_eval=None,\n",
    "            stratified=False,\n",
    "        )\n",
    "\n",
    "        # Print RMSE for each round of lgbBO for rough tracking of the optimization process\n",
    "        min_index = cv_result[\"rmse-mean\"].index(min(cv_result[\"rmse-mean\"]))\n",
    "        print(\n",
    "            \"RMSE: {} +- {}\".format(\n",
    "                round(cv_result[\"rmse-mean\"][min_index], 5),\n",
    "                round(cv_result[\"rmse-stdv\"][min_index], 5),\n",
    "            )\n",
    "        )\n",
    "\n",
    "        return (-1.0 * np.array(cv_result[\"rmse-mean\"])).max()\n",
    "\n",
    "    lgbBO = BayesianOptimization(\n",
    "        lgb_eval,\n",
    "        {\n",
    "            \"feature_fraction\": (0.6, 0.9),\n",
    "            \"bagging_fraction\": (0.8, 1),\n",
    "            \"lambda_l1\": (0, 3),\n",
    "            \"lambda_l2\": (0, 3),\n",
    "            \"max_depth\": (5, 100),\n",
    "            \"num_leaves\": (10, 300),\n",
    "            \"min_split_gain\": (0.001, 0.1),\n",
    "            \"min_child_weight\": (0, 1),\n",
    "            \"learning_rate\": (0.001, 0.1),\n",
    "            \"n_estimators\": (50, 5000),\n",
    "        },\n",
    "        random_state=random_seed,\n",
    "    )\n",
    "\n",
    "    # Save progress for each round into a JSON file which can be monitored on a editor (i.e. VSCode)\n",
    "    # This somehow suppresses the terminal output (https://github.com/fmfn/BayesianOptimization/issues/167)\n",
    "    logger = JSONLogger(\n",
    "        path=DATA_DIR\n",
    "        + \"/Models/LightGBM/Params/{}.json\".format(\n",
    "            pd.Timestamp.now().strftime(\"%Y%m%d-%Hh%Mm\")\n",
    "        )\n",
    "    )\n",
    "    lgbBO.subscribe(Events.OPTMIZATION_STEP, logger)\n",
    "\n",
    "    lgbBO.maximize(init_points=init_round, n_iter=opt_round, acq=\"ei\")\n",
    "\n",
    "    return lgbBO.max[\"params\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-28T19:40:36.621718Z",
     "start_time": "2019-06-28T19:40:33.853934Z"
    }
   },
   "outputs": [],
   "source": [
    "X, y = utils.load_full_dataset(\"quad\")\n",
    "opt_params = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-28T19:40:36.632886Z",
     "start_time": "2019-06-28T19:40:36.626686Z"
    }
   },
   "outputs": [],
   "source": [
    "# Check DATA_DIR/Logs/{}.json for tuned parameters while optimising\n",
    "# Takes ~ 2 hrs for 30 rounds (15 init + 15 opt) on CPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-28T19:40:36.645290Z",
     "start_time": "2019-06-28T19:40:36.635416Z"
    },
    "code_folding": [],
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# opt_params = bayes_parameter_opt_lgb(X, y,\n",
    "#                                      init_round=15,\n",
    "#                                      opt_round=15,\n",
    "#                                      n_folds=5,\n",
    "#                                      random_seed=23)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-28T19:40:36.683991Z",
     "start_time": "2019-06-28T19:40:36.650245Z"
    }
   },
   "outputs": [],
   "source": [
    "# Load the newest file in the params directory, otherwise set manually\n",
    "path_params = sorted(glob.glob(DATA_DIR + \"/Models/LightGBM/Params/*.json\"))[-1]\n",
    "df_params = pd.read_json(path_params, lines=True)\n",
    "df_params = (\n",
    "    df_params.loc[:, [\"target\", \"params\"]]\n",
    "    .sort_values(by=\"target\", ascending=False)\n",
    "    .reset_index()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-28T19:40:36.727535Z",
     "start_time": "2019-06-28T19:40:36.689484Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>target</th>\n",
       "      <th>params</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17</td>\n",
       "      <td>-9.080672</td>\n",
       "      <td>{'bagging_fraction': 0.9397558879953231, 'feat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13</td>\n",
       "      <td>-9.080783</td>\n",
       "      <td>{'bagging_fraction': 0.8967783942656771, 'feat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>-9.082169</td>\n",
       "      <td>{'bagging_fraction': 0.965750296864551, 'featu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>19</td>\n",
       "      <td>-9.082335</td>\n",
       "      <td>{'bagging_fraction': 0.8, 'feature_fraction': ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>-9.083474</td>\n",
       "      <td>{'bagging_fraction': 0.9803204385614761, 'feat...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index    target                                             params\n",
       "0     17 -9.080672  {'bagging_fraction': 0.9397558879953231, 'feat...\n",
       "1     13 -9.080783  {'bagging_fraction': 0.8967783942656771, 'feat...\n",
       "2      3 -9.082169  {'bagging_fraction': 0.965750296864551, 'featu...\n",
       "3     19 -9.082335  {'bagging_fraction': 0.8, 'feature_fraction': ...\n",
       "4      5 -9.083474  {'bagging_fraction': 0.9803204385614761, 'feat..."
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_params.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-28T19:40:36.745729Z",
     "start_time": "2019-06-28T19:40:36.730131Z"
    }
   },
   "outputs": [],
   "source": [
    "if opt_params == None:\n",
    "    opt_params = df_params.loc[0, \"params\"]\n",
    "else:\n",
    "    print(\"opt_params is already defined\")\n",
    "\n",
    "for key in opt_params.keys():\n",
    "    if key in [\"max_depth\", \"num_leaves\", \"n_estimators\"]:\n",
    "        opt_params[key] = int(round(opt_params[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-06-28T19:50:36.744767Z",
     "start_time": "2019-06-28T19:40:36.751405Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e2b87b83e684cf7abb1844994a292d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "   <--- Validation Errors --->\n",
      "MAE  | Mean: 6.97997, SD: 0.02451\n",
      "RMSE | Mean: 9.09184, SD: 0.038\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model = lgb.LGBMModel()\n",
    "errors = utils.cross_val(model, X, y, isLightGBM=True, params=opt_params, verbose=0)\n",
    "utils.summarize_errors(errors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LightGBM with Hyperparameter Tuning\n",
    "- MAE  | Mean: 6.97997, SD: 0.02451\n",
    "- RMSE | Mean: 9.09184, SD: 0.038"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}