1_BO_NonDP.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "871ffe8b",
   "metadata": {},
   "source": [
    "## Initial parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d3d15409",
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 42\n",
    "dataset = 'LSAC' # select one in ['adult', 'ACSCoverage', 'LSAC']\n",
    "mechanism = \"non_private\"\n",
    "folder_name = 'results/' + dataset + '/' + mechanism\n",
    "\n",
    "# for ML\n",
    "if dataset == 'adult':\n",
    "    target = 'income'\n",
    "    protected_attribute = 'gender'\n",
    "    \n",
    "elif dataset == 'ACSCoverage':\n",
    "    target = 'PUBCOV'\n",
    "    protected_attribute = 'DIS'\n",
    "    \n",
    "elif dataset == 'LSAC':\n",
    "    target = 'pass_bar'\n",
    "    protected_attribute = 'race1' \n",
    "    \n",
    "test_size = 0.2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1895b9d1",
   "metadata": {},
   "source": [
    "## Write function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "662cbe9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def write(folder_name, values):\n",
    "    with open(folder_name + \"/LGBM_BO_results.csv\", mode='a', newline='') as scores_file:\n",
    "        scores_writer = csv.writer(scores_file, delimiter=',', quotechar='\"', quoting=csv.QUOTE_MINIMAL)\n",
    "        scores_writer.writerow(values)\n",
    "    scores_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8d68cb2",
   "metadata": {},
   "source": [
    "## Importing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ede42761",
   "metadata": {},
   "outputs": [],
   "source": [
    "# General imports\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import copy\n",
    "import csv\n",
    "\n",
    "# sklearn imports\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, f1_score, roc_auc_score, recall_score\n",
    "\n",
    "# hyper-params opti\n",
    "import hyperopt\n",
    "from hyperopt import fmin, tpe, hp, STATUS_OK, Trials\n",
    "\n",
    "# designed functions\n",
    "from functions import fairness_metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e35d356a",
   "metadata": {},
   "source": [
    "## Reading dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "80b48384",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fam_inc</th>\n",
       "      <th>gender</th>\n",
       "      <th>fulltime</th>\n",
       "      <th>race1</th>\n",
       "      <th>lsat</th>\n",
       "      <th>ugpa</th>\n",
       "      <th>pass_bar</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20422</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20423</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20424</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20425</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20426</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20427 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       fam_inc  gender  fulltime  race1  lsat  ugpa  pass_bar\n",
       "0            4       0         0      1    32     4         1\n",
       "1            3       0         0      1    17     4         1\n",
       "2            0       1         0      1    24     4         1\n",
       "3            3       1         0      1    27     4         1\n",
       "4            3       1         0      1    36     4         1\n",
       "...        ...     ...       ...    ...   ...   ...       ...\n",
       "20422        1       1         0      0    14     1         0\n",
       "20423        2       1         0      0     8     1         0\n",
       "20424        2       1         1      0    24     1         1\n",
       "20425        2       1         1      1    32     0         1\n",
       "20426        2       1         0      1    18     0         1\n",
       "\n",
       "[20427 rows x 7 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if dataset == 'adult':\n",
    "    df = pd.read_csv('datasets/db_adult_processed_26k.csv')\n",
    "    \n",
    "elif dataset == 'ACSCoverage':\n",
    "    df = pd.read_csv('datasets/db_ACSCoverage.csv')\n",
    "\n",
    "elif dataset == 'LSAC':\n",
    "    df = pd.read_csv('datasets/db_LSAC.csv')\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3b02848",
   "metadata": {},
   "source": [
    "## Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fee6aefd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fam_inc_0</th>\n",
       "      <th>fam_inc_1</th>\n",
       "      <th>fam_inc_2</th>\n",
       "      <th>fam_inc_3</th>\n",
       "      <th>fam_inc_4</th>\n",
       "      <th>gender_0</th>\n",
       "      <th>gender_1</th>\n",
       "      <th>fulltime_0</th>\n",
       "      <th>fulltime_1</th>\n",
       "      <th>race1_0</th>\n",
       "      <th>...</th>\n",
       "      <th>lsat_34</th>\n",
       "      <th>lsat_35</th>\n",
       "      <th>lsat_36</th>\n",
       "      <th>ugpa_0</th>\n",
       "      <th>ugpa_1</th>\n",
       "      <th>ugpa_2</th>\n",
       "      <th>ugpa_3</th>\n",
       "      <th>ugpa_4</th>\n",
       "      <th>ugpa_5</th>\n",
       "      <th>pass_bar</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20422</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20423</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20424</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20425</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20426</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20427 rows × 55 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       fam_inc_0  fam_inc_1  fam_inc_2  fam_inc_3  fam_inc_4  gender_0  \\\n",
       "0            0.0        0.0        0.0        0.0        1.0       1.0   \n",
       "1            0.0        0.0        0.0        1.0        0.0       1.0   \n",
       "2            1.0        0.0        0.0        0.0        0.0       0.0   \n",
       "3            0.0        0.0        0.0        1.0        0.0       0.0   \n",
       "4            0.0        0.0        0.0        1.0        0.0       0.0   \n",
       "...          ...        ...        ...        ...        ...       ...   \n",
       "20422        0.0        1.0        0.0        0.0        0.0       0.0   \n",
       "20423        0.0        0.0        1.0        0.0        0.0       0.0   \n",
       "20424        0.0        0.0        1.0        0.0        0.0       0.0   \n",
       "20425        0.0        0.0        1.0        0.0        0.0       0.0   \n",
       "20426        0.0        0.0        1.0        0.0        0.0       0.0   \n",
       "\n",
       "       gender_1  fulltime_0  fulltime_1  race1_0  ...  lsat_34  lsat_35  \\\n",
       "0           0.0         1.0         0.0      0.0  ...      0.0      0.0   \n",
       "1           0.0         1.0         0.0      0.0  ...      0.0      0.0   \n",
       "2           1.0         1.0         0.0      0.0  ...      0.0      0.0   \n",
       "3           1.0         1.0         0.0      0.0  ...      0.0      0.0   \n",
       "4           1.0         1.0         0.0      0.0  ...      0.0      0.0   \n",
       "...         ...         ...         ...      ...  ...      ...      ...   \n",
       "20422       1.0         1.0         0.0      1.0  ...      0.0      0.0   \n",
       "20423       1.0         1.0         0.0      1.0  ...      0.0      0.0   \n",
       "20424       1.0         0.0         1.0      1.0  ...      0.0      0.0   \n",
       "20425       1.0         0.0         1.0      0.0  ...      0.0      0.0   \n",
       "20426       1.0         1.0         0.0      0.0  ...      0.0      0.0   \n",
       "\n",
       "       lsat_36  ugpa_0  ugpa_1  ugpa_2  ugpa_3  ugpa_4  ugpa_5  pass_bar  \n",
       "0          0.0     0.0     0.0     0.0     0.0     1.0     0.0         1  \n",
       "1          0.0     0.0     0.0     0.0     0.0     1.0     0.0         1  \n",
       "2          0.0     0.0     0.0     0.0     0.0     1.0     0.0         1  \n",
       "3          0.0     0.0     0.0     0.0     0.0     1.0     0.0         1  \n",
       "4          1.0     0.0     0.0     0.0     0.0     1.0     0.0         1  \n",
       "...        ...     ...     ...     ...     ...     ...     ...       ...  \n",
       "20422      0.0     0.0     1.0     0.0     0.0     0.0     0.0         0  \n",
       "20423      0.0     0.0     1.0     0.0     0.0     0.0     0.0         0  \n",
       "20424      0.0     0.0     1.0     0.0     0.0     0.0     0.0         1  \n",
       "20425      0.0     1.0     0.0     0.0     0.0     0.0     0.0         1  \n",
       "20426      0.0     1.0     0.0     0.0     0.0     0.0     0.0         1  \n",
       "\n",
       "[20427 rows x 55 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lst_df = []\n",
    "for col in df.columns:\n",
    "    \n",
    "    if col != target:\n",
    "        lst_col_name = [col+\"_{}\".format(val) for val in range(len(set(df[col])))]\n",
    "\n",
    "        k = len(set(df[col]))\n",
    "\n",
    "        OHE = np.eye(k)\n",
    "\n",
    "        df_ohe = pd.DataFrame([OHE[val] for val in df[col]], columns=lst_col_name)\n",
    "        lst_df.append(df_ohe)\n",
    "df = pd.concat([pd.concat(lst_df, axis=1), df[target]], axis=1)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "736ddd11",
   "metadata": {},
   "source": [
    "## Splitting train and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e670d45e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((16341, 54), (16341,), (4086, 54), (4086,))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = copy.deepcopy(df.drop(target, axis=1))\n",
    "y = copy.deepcopy(df[target])\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, shuffle=True, stratify=y, random_state=seed)\n",
    "X_train.shape, y_train.shape, X_test.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c90ca45d",
   "metadata": {},
   "source": [
    "## Single Run of Non-private LGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ddf5ce9d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance Metrics\n",
      "acc: 0.947136563876652\n",
      "f1: 0.9728028204482498\n",
      "auc: 0.5148929364858968\n",
      "recall: 0.9961320268179474\n",
      "\n",
      "Fairness Metrics\n",
      "SP_a_1: 0.9984399375975039\n",
      "SP_a_0: 0.9333333333333333\n",
      "DI: 0.9347916666666667\n",
      "SPD: 0.06510660426417059\n",
      "EO_a_1: 0.9986438839164633\n",
      "EO_a_0: 0.9476439790575916\n",
      "EOD: 0.050999904858871736\n",
      "OA_a_1: 0.9576183047321893\n",
      "OA_a_0: 0.7791666666666667\n",
      "OAD: 0.17845163806552267\n"
     ]
    }
   ],
   "source": [
    "model = LGBMClassifier(random_state=seed, n_jobs=-1, objective=\"binary\")\n",
    "model.fit(X_train, y_train)\n",
    "y_pred = model.predict(X_test)\n",
    "\n",
    "# performance metrics\n",
    "acc = accuracy_score(y_test, y_pred)\n",
    "f1 = f1_score(y_test, y_pred)\n",
    "auc = roc_auc_score(y_test, y_pred)\n",
    "recall = recall_score(y_test, y_pred)\n",
    "\n",
    "print('Performance Metrics')\n",
    "print(\"acc:\", acc)\n",
    "print(\"f1:\", f1)\n",
    "print(\"auc:\", auc)\n",
    "print(\"recall:\", recall)\n",
    "\n",
    "# prepare dataset for fairness metrics\n",
    "df_fm = pd.concat([X_test, y_test], axis=1)\n",
    "df_fm['y_pred'] = y_pred\n",
    "\n",
    "print('\\nFairness Metrics')\n",
    "\n",
    "fair_met = fairness_metrics(df_fm, protected_attribute, target)\n",
    "\n",
    "for key in fair_met.keys():\n",
    "    print(key+\":\", fair_met[key])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "355d0a9a",
   "metadata": {},
   "source": [
    "## Objective function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "59b6feab",
   "metadata": {},
   "outputs": [],
   "source": [
    "def objective_function(space):\n",
    "    \n",
    "    global seed, ITER, X_train, y_train, X_test, y_test\n",
    "    \n",
    "    ITER += 1\n",
    "    \n",
    "    params = {'max_depth': int(space['max_depth']), \n",
    "              'learning_rate': space['learning_rate'],\n",
    "              'n_estimators': int(space['n_estimators']),\n",
    "             }\n",
    "    \n",
    "    print(\"------------------------------------------------------------------------------------\")\n",
    "    print(ITER, \":: \", params)\n",
    "    \n",
    "    # Initialize and fit model\n",
    "    model = LGBMClassifier(random_state=seed, n_jobs=-1, objective=\"binary\")\n",
    "    model.set_params(**params)\n",
    "    model.fit(X_train, y_train)\n",
    "    \n",
    "    # Predictions\n",
    "    y_pred = model.predict(X_test)\n",
    "    \n",
    "    # Performance metrics\n",
    "    acc = accuracy_score(y_test, y_pred)\n",
    "    cm = confusion_matrix(y_test, y_pred)\n",
    "    f1 = f1_score(y_test, y_pred)\n",
    "    auc = roc_auc_score(y_test, y_pred)\n",
    "    recall = recall_score(y_test, y_pred)\n",
    "        \n",
    "    # write [\"iter\", \"acc\", \"f1-score\", \"auc\", \"recall\", \"cm\", \"params\"]\n",
    "    write(folder_name, [str(ITER),\n",
    "           acc,\n",
    "           f1,\n",
    "           auc,\n",
    "           recall,\n",
    "           cm,\n",
    "           params])\n",
    "\n",
    "    print(\"loss:\", auc)\n",
    "    \n",
    "    # maximize AUC metric\n",
    "    return {'loss':-auc, 'status': STATUS_OK}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a3321d2",
   "metadata": {},
   "source": [
    "## Bayesian Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "366cd73a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------------------------------------\n",
      "1 ::  {'max_depth': 31, 'learning_rate': 0.2221423901134277, 'n_estimators': 1650}\n",
      "loss: 0.5533916669972627\n",
      "------------------------------------------------------------------------------------\n",
      "2 ::  {'max_depth': 30, 'learning_rate': 0.21662331565922432, 'n_estimators': 650}\n",
      "loss: 0.5418881659856389\n",
      "------------------------------------------------------------------------------------\n",
      "3 ::  {'max_depth': 3, 'learning_rate': 0.15823860430552075, 'n_estimators': 350}\n",
      "loss: 0.5098273792994009\n",
      "------------------------------------------------------------------------------------\n",
      "4 ::  {'max_depth': 10, 'learning_rate': 0.1521888191981938, 'n_estimators': 850}\n",
      "loss: 0.5421460308644424\n",
      "------------------------------------------------------------------------------------\n",
      "5 ::  {'max_depth': 4, 'learning_rate': 0.07896551676477612, 'n_estimators': 1050}\n",
      "loss: 0.5138614769706827\n",
      "------------------------------------------------------------------------------------\n",
      "6 ::  {'max_depth': 7, 'learning_rate': 0.11606641560458879, 'n_estimators': 1500}\n",
      "loss: 0.5448077418970921\n",
      "------------------------------------------------------------------------------------\n",
      "7 ::  {'max_depth': 50, 'learning_rate': 0.0296192660685174, 'n_estimators': 1400}\n",
      "loss: 0.5273820268179474\n",
      "------------------------------------------------------------------------------------\n",
      "8 ::  {'max_depth': 41, 'learning_rate': 0.1895285855104962, 'n_estimators': 200}\n",
      "loss: 0.5280266890149562\n",
      "------------------------------------------------------------------------------------\n",
      "9 ::  {'max_depth': 38, 'learning_rate': 0.02297868986660416, 'n_estimators': 1450}\n",
      "loss: 0.5253649779823065\n",
      "------------------------------------------------------------------------------------\n",
      "10 ::  {'max_depth': 24, 'learning_rate': 0.1143995088761935, 'n_estimators': 700}\n",
      "loss: 0.538498730511366\n",
      "------------------------------------------------------------------------------------\n",
      "11 ::  {'max_depth': 1, 'learning_rate': 0.07439794089052901, 'n_estimators': 100}\n",
      "loss: 0.5\n",
      "------------------------------------------------------------------------------------\n",
      "12 ::  {'max_depth': 30, 'learning_rate': 0.035159348091152454, 'n_estimators': 1550}\n",
      "loss: 0.5272530943785456\n",
      "------------------------------------------------------------------------------------\n",
      "13 ::  {'max_depth': 36, 'learning_rate': 0.15618499578119666, 'n_estimators': 600}\n",
      "loss: 0.532917443567263\n",
      "------------------------------------------------------------------------------------\n",
      "14 ::  {'max_depth': 23, 'learning_rate': 0.03132444353580099, 'n_estimators': 200}\n",
      "loss: 0.5104720414964098\n",
      "------------------------------------------------------------------------------------\n",
      "15 ::  {'max_depth': 45, 'learning_rate': 0.014103205587458678, 'n_estimators': 500}\n",
      "loss: 0.5102141766176062\n",
      "------------------------------------------------------------------------------------\n",
      "16 ::  {'max_depth': 49, 'learning_rate': 0.05902822048243893, 'n_estimators': 1750}\n",
      "loss: 0.540386846907605\n",
      "------------------------------------------------------------------------------------\n",
      "17 ::  {'max_depth': 41, 'learning_rate': 0.2060787434112076, 'n_estimators': 1750}\n",
      "loss: 0.5559244455905106\n",
      "------------------------------------------------------------------------------------\n",
      "18 ::  {'max_depth': 4, 'learning_rate': 0.09416399038559989, 'n_estimators': 700}\n",
      "loss: 0.5114576308168366\n",
      "------------------------------------------------------------------------------------\n",
      "19 ::  {'max_depth': 42, 'learning_rate': 0.04073845502232924, 'n_estimators': 550}\n",
      "loss: 0.5209440829928195\n",
      "------------------------------------------------------------------------------------\n",
      "20 ::  {'max_depth': 35, 'learning_rate': 0.24423997545734005, 'n_estimators': 900}\n",
      "loss: 0.5557955131511088\n",
      "------------------------------------------------------------------------------------\n",
      "21 ::  {'max_depth': 15, 'learning_rate': 0.24317648329630664, 'n_estimators': 1200}\n",
      "loss: 0.5535205994366644\n",
      "------------------------------------------------------------------------------------\n",
      "22 ::  {'max_depth': 35, 'learning_rate': 0.2443406489168825, 'n_estimators': 2000}\n",
      "loss: 0.5557955131511088\n",
      "------------------------------------------------------------------------------------\n",
      "23 ::  {'max_depth': 46, 'learning_rate': 0.1997440484540963, 'n_estimators': 1950}\n",
      "loss: 0.5485839746895703\n",
      "------------------------------------------------------------------------------------\n",
      "24 ::  {'max_depth': 18, 'learning_rate': 0.18388673797884497, 'n_estimators': 1200}\n",
      "loss: 0.5489707720077757\n",
      "------------------------------------------------------------------------------------\n",
      "25 ::  {'max_depth': 33, 'learning_rate': 0.2484133565354124, 'n_estimators': 1950}\n",
      "loss: 0.550729955964613\n",
      "------------------------------------------------------------------------------------\n",
      "26 ::  {'max_depth': 40, 'learning_rate': 0.22480102785533, 'n_estimators': 1850}\n",
      "loss: 0.5509878208434165\n",
      "------------------------------------------------------------------------------------\n",
      "27 ::  {'max_depth': 27, 'learning_rate': 0.16933564318984734, 'n_estimators': 1750}\n",
      "loss: 0.5508588884040148\n",
      "------------------------------------------------------------------------------------\n",
      "28 ::  {'max_depth': 45, 'learning_rate': 0.20889802914112637, 'n_estimators': 1300}\n",
      "loss: 0.5508588884040148\n",
      "------------------------------------------------------------------------------------\n",
      "29 ::  {'max_depth': 20, 'learning_rate': 0.2283441901955025, 'n_estimators': 1600}\n",
      "loss: 0.5509878208434165\n",
      "------------------------------------------------------------------------------------\n",
      "30 ::  {'max_depth': 28, 'learning_rate': 0.12963461720707695, 'n_estimators': 2000}\n",
      "loss: 0.55124568572222\n",
      "------------------------------------------------------------------------------------\n",
      "31 ::  {'max_depth': 33, 'learning_rate': 0.17853830641553559, 'n_estimators': 1700}\n",
      "loss: 0.5508588884040148\n",
      "------------------------------------------------------------------------------------\n",
      "32 ::  {'max_depth': 48, 'learning_rate': 0.2354997728660511, 'n_estimators': 1850}\n",
      "loss: 0.5508588884040148\n",
      "------------------------------------------------------------------------------------\n",
      "33 ::  {'max_depth': 38, 'learning_rate': 0.21659987052905233, 'n_estimators': 1900}\n",
      "loss: 0.5489707720077757\n",
      "------------------------------------------------------------------------------------\n",
      "34 ::  {'max_depth': 42, 'learning_rate': 0.1436828020538154, 'n_estimators': 1000}\n",
      "loss: 0.5469537231721348\n",
      "------------------------------------------------------------------------------------\n",
      "35 ::  {'max_depth': 31, 'learning_rate': 0.19871716170577236, 'n_estimators': 1650}\n",
      "loss: 0.5508588884040148\n",
      "------------------------------------------------------------------------------------\n",
      "36 ::  {'max_depth': 14, 'learning_rate': 0.17093383798687037, 'n_estimators': 1300}\n",
      "loss: 0.5533916669972627\n",
      "------------------------------------------------------------------------------------\n",
      "37 ::  {'max_depth': 26, 'learning_rate': 0.1389529647144032, 'n_estimators': 1400}\n",
      "loss: 0.5492286368865792\n",
      "------------------------------------------------------------------------------------\n",
      "38 ::  {'max_depth': 36, 'learning_rate': 0.2090382279988366, 'n_estimators': 1800}\n",
      "loss: 0.5559244455905106\n",
      "------------------------------------------------------------------------------------\n",
      "39 ::  {'max_depth': 39, 'learning_rate': 0.20785773861599408, 'n_estimators': 1050}\n",
      "loss: 0.5509878208434165\n",
      "------------------------------------------------------------------------------------\n",
      "40 ::  {'max_depth': 43, 'learning_rate': 0.11704419196765839, 'n_estimators': 1800}\n",
      "loss: 0.5436473499424762\n",
      "------------------------------------------------------------------------------------\n",
      "41 ::  {'max_depth': 50, 'learning_rate': 0.1660311142103707, 'n_estimators': 1550}\n",
      "loss: 0.5508588884040148\n",
      "------------------------------------------------------------------------------------\n",
      "42 ::  {'max_depth': 22, 'learning_rate': 0.09543436866826346, 'n_estimators': 1200}\n",
      "loss: 0.5468247907327329\n",
      "------------------------------------------------------------------------------------\n",
      "43 ::  {'max_depth': 32, 'learning_rate': 0.19334082212324052, 'n_estimators': 800}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.5490997044471774\n",
      "------------------------------------------------------------------------------------\n",
      "44 ::  {'max_depth': 29, 'learning_rate': 0.21402280026452858, 'n_estimators': 1450}\n",
      "loss: 0.5511167532828183\n",
      "------------------------------------------------------------------------------------\n",
      "45 ::  {'max_depth': 37, 'learning_rate': 0.23162334091291253, 'n_estimators': 1300}\n",
      "loss: 0.5504720910858095\n",
      "------------------------------------------------------------------------------------\n",
      "46 ::  {'max_depth': 46, 'learning_rate': 0.15566079242964784, 'n_estimators': 1650}\n",
      "loss: 0.5511167532828183\n",
      "------------------------------------------------------------------------------------\n",
      "47 ::  {'max_depth': 9, 'learning_rate': 0.18177994587365698, 'n_estimators': 1100}\n",
      "loss: 0.5563112429087158\n",
      "------------------------------------------------------------------------------------\n",
      "48 ::  {'max_depth': 8, 'learning_rate': 0.18083714122765138, 'n_estimators': 950}\n",
      "loss: 0.5492286368865792\n",
      "------------------------------------------------------------------------------------\n",
      "49 ::  {'max_depth': 1, 'learning_rate': 0.1284363239455824, 'n_estimators': 400}\n",
      "loss: 0.5085839251001706\n",
      "------------------------------------------------------------------------------------\n",
      "50 ::  {'max_depth': 12, 'learning_rate': 0.14537445465691085, 'n_estimators': 750}\n",
      "loss: 0.5418881659856389\n",
      "------------------------------------------------------------------------------------\n",
      "51 ::  {'max_depth': 6, 'learning_rate': 0.10263688221856349, 'n_estimators': 1100}\n",
      "loss: 0.5336910382036736\n",
      "------------------------------------------------------------------------------------\n",
      "52 ::  {'max_depth': 17, 'learning_rate': 0.06859839518842904, 'n_estimators': 50}\n",
      "loss: 0.5107299063752132\n",
      "------------------------------------------------------------------------------------\n",
      "53 ::  {'max_depth': 0, 'learning_rate': 0.19007554353895026, 'n_estimators': 250}\n",
      "loss: 0.5319318542468362\n",
      "------------------------------------------------------------------------------------\n",
      "54 ::  {'max_depth': 10, 'learning_rate': 0.11946636343602896, 'n_estimators': 450}\n",
      "loss: 0.5266084321815369\n",
      "------------------------------------------------------------------------------------\n",
      "55 ::  {'max_depth': 20, 'learning_rate': 0.16323243446041744, 'n_estimators': 300}\n",
      "loss: 0.5319318542468362\n",
      "------------------------------------------------------------------------------------\n",
      "56 ::  {'max_depth': 11, 'learning_rate': 0.05155245796520268, 'n_estimators': 850}\n",
      "loss: 0.5272530943785456\n",
      "------------------------------------------------------------------------------------\n",
      "57 ::  {'max_depth': 6, 'learning_rate': 0.1769683763088024, 'n_estimators': 600}\n",
      "loss: 0.5336910382036736\n",
      "------------------------------------------------------------------------------------\n",
      "58 ::  {'max_depth': 26, 'learning_rate': 0.22137645700133457, 'n_estimators': 1100}\n",
      "loss: 0.5535205994366644\n",
      "------------------------------------------------------------------------------------\n",
      "59 ::  {'max_depth': 3, 'learning_rate': 0.1492189701956374, 'n_estimators': 1500}\n",
      "loss: 0.5137325445312809\n",
      "------------------------------------------------------------------------------------\n",
      "60 ::  {'max_depth': 24, 'learning_rate': 0.23637942790074493, 'n_estimators': 1400}\n",
      "loss: 0.5536495318760661\n",
      "------------------------------------------------------------------------------------\n",
      "61 ::  {'max_depth': 16, 'learning_rate': 0.13706762920362248, 'n_estimators': 650}\n",
      "loss: 0.5379830007537589\n",
      "------------------------------------------------------------------------------------\n",
      "62 ::  {'max_depth': 18, 'learning_rate': 0.10298156792126445, 'n_estimators': 150}\n",
      "loss: 0.5141193418494863\n",
      "------------------------------------------------------------------------------------\n",
      "63 ::  {'max_depth': 13, 'learning_rate': 0.16103317887015983, 'n_estimators': 1150}\n",
      "loss: 0.5465669258539294\n",
      "------------------------------------------------------------------------------------\n",
      "64 ::  {'max_depth': 8, 'learning_rate': 0.18619509495334835, 'n_estimators': 1350}\n",
      "loss: 0.55124568572222\n",
      "------------------------------------------------------------------------------------\n",
      "65 ::  {'max_depth': 21, 'learning_rate': 0.19545902342975907, 'n_estimators': 1550}\n",
      "loss: 0.5511167532828183\n",
      "------------------------------------------------------------------------------------\n",
      "66 ::  {'max_depth': 35, 'learning_rate': 0.20659378496182035, 'n_estimators': 1750}\n",
      "loss: 0.555666580711707\n",
      "------------------------------------------------------------------------------------\n",
      "67 ::  {'max_depth': 40, 'learning_rate': 0.20396017912054507, 'n_estimators': 1900}\n",
      "loss: 0.5511167532828183\n",
      "------------------------------------------------------------------------------------\n",
      "68 ::  {'max_depth': 44, 'learning_rate': 0.22181095473363227, 'n_estimators': 1700}\n",
      "loss: 0.550729955964613\n",
      "------------------------------------------------------------------------------------\n",
      "69 ::  {'max_depth': 47, 'learning_rate': 0.17353031401945157, 'n_estimators': 2000}\n",
      "loss: 0.5511167532828183\n",
      "------------------------------------------------------------------------------------\n",
      "70 ::  {'max_depth': 33, 'learning_rate': 0.24304714770856548, 'n_estimators': 1600}\n",
      "loss: 0.5509878208434165\n",
      "------------------------------------------------------------------------------------\n",
      "71 ::  {'max_depth': 30, 'learning_rate': 0.21387751862367047, 'n_estimators': 1800}\n",
      "loss: 0.5536495318760661\n",
      "------------------------------------------------------------------------------------\n",
      "72 ::  {'max_depth': 35, 'learning_rate': 0.24943034330948988, 'n_estimators': 1250}\n",
      "loss: 0.5559244455905106\n",
      "------------------------------------------------------------------------------------\n",
      "73 ::  {'max_depth': 35, 'learning_rate': 0.23971998931643457, 'n_estimators': 1250}\n",
      "loss: 0.5530048696790574\n",
      "------------------------------------------------------------------------------------\n",
      "74 ::  {'max_depth': 39, 'learning_rate': 0.23058022802013203, 'n_estimators': 950}\n",
      "loss: 0.5488418395683738\n",
      "------------------------------------------------------------------------------------\n",
      "75 ::  {'max_depth': 37, 'learning_rate': 0.24760351973759295, 'n_estimators': 850}\n",
      "loss: 0.5492286368865792\n",
      "------------------------------------------------------------------------------------\n",
      "76 ::  {'max_depth': 28, 'learning_rate': 0.19911773239902852, 'n_estimators': 1150}\n",
      "loss: 0.5508588884040148\n",
      "------------------------------------------------------------------------------------\n",
      "77 ::  {'max_depth': 26, 'learning_rate': 0.22311375136372402, 'n_estimators': 1000}\n",
      "loss: 0.546309060975126\n",
      "------------------------------------------------------------------------------------\n",
      "78 ::  {'max_depth': 42, 'learning_rate': 0.24917171733266466, 'n_estimators': 1450}\n",
      "loss: 0.5484550422501686\n",
      "------------------------------------------------------------------------------------\n",
      "79 ::  {'max_depth': 49, 'learning_rate': 0.18765581094604844, 'n_estimators': 1250}\n",
      "loss: 0.550729955964613\n",
      "------------------------------------------------------------------------------------\n",
      "80 ::  {'max_depth': 41, 'learning_rate': 0.18250832083749383, 'n_estimators': 1950}\n",
      "loss: 0.5509878208434165\n",
      "------------------------------------------------------------------------------------\n",
      "81 ::  {'max_depth': 31, 'learning_rate': 0.15480958524706342, 'n_estimators': 1350}\n",
      "loss: 0.5489707720077757\n",
      "------------------------------------------------------------------------------------\n",
      "82 ::  {'max_depth': 23, 'learning_rate': 0.21054190901286335, 'n_estimators': 750}\n",
      "loss: 0.5489707720077757\n",
      "------------------------------------------------------------------------------------\n",
      "83 ::  {'max_depth': 34, 'learning_rate': 0.12476933305521085, 'n_estimators': 900}\n",
      "loss: 0.5422749633038442\n",
      "------------------------------------------------------------------------------------\n",
      "84 ::  {'max_depth': 44, 'learning_rate': 0.16792928182614944, 'n_estimators': 1850}\n",
      "loss: 0.550729955964613\n",
      "------------------------------------------------------------------------------------\n",
      "85 ::  {'max_depth': 37, 'learning_rate': 0.227108873968337, 'n_estimators': 1100}\n",
      "loss: 0.550729955964613\n",
      "------------------------------------------------------------------------------------\n",
      "86 ::  {'max_depth': 27, 'learning_rate': 0.018913202437585402, 'n_estimators': 550}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 0.5100852441782046\n",
      "------------------------------------------------------------------------------------\n",
      "87 ::  {'max_depth': 19, 'learning_rate': 0.08296302993864231, 'n_estimators': 1500}\n",
      "loss: 0.5474694529297417\n",
      "------------------------------------------------------------------------------------\n",
      "88 ::  {'max_depth': 48, 'learning_rate': 0.13729378349311658, 'n_estimators': 1250}\n",
      "loss: 0.5513746181616218\n",
      "------------------------------------------------------------------------------------\n",
      "89 ::  {'max_depth': 32, 'learning_rate': 0.19333882733400584, 'n_estimators': 950}\n",
      "loss: 0.5489707720077757\n",
      "------------------------------------------------------------------------------------\n",
      "90 ::  {'max_depth': 29, 'learning_rate': 0.2377207841407428, 'n_estimators': 1050}\n",
      "loss: 0.5459222636569206\n",
      "------------------------------------------------------------------------------------\n",
      "91 ::  {'max_depth': 15, 'learning_rate': 0.17450486905882553, 'n_estimators': 1650}\n",
      "loss: 0.5532627345578609\n",
      "------------------------------------------------------------------------------------\n",
      "92 ::  {'max_depth': 25, 'learning_rate': 0.2182169763051358, 'n_estimators': 1350}\n",
      "loss: 0.5508588884040148\n",
      "------------------------------------------------------------------------------------\n",
      "93 ::  {'max_depth': 39, 'learning_rate': 0.20409734189806383, 'n_estimators': 1450}\n",
      "loss: 0.5484550422501686\n",
      "------------------------------------------------------------------------------------\n",
      "94 ::  {'max_depth': 3, 'learning_rate': 0.14864691482002423, 'n_estimators': 750}\n",
      "loss: 0.5117154956956401\n",
      "------------------------------------------------------------------------------------\n",
      "95 ::  {'max_depth': 36, 'learning_rate': 0.2344077200947505, 'n_estimators': 1150}\n",
      "loss: 0.55124568572222\n",
      "------------------------------------------------------------------------------------\n",
      "96 ::  {'max_depth': 30, 'learning_rate': 0.15844610729784428, 'n_estimators': 650}\n",
      "loss: 0.540386846907605\n",
      "------------------------------------------------------------------------------------\n",
      "97 ::  {'max_depth': 43, 'learning_rate': 0.20179221568061767, 'n_estimators': 1900}\n",
      "loss: 0.5511167532828183\n",
      "------------------------------------------------------------------------------------\n",
      "98 ::  {'max_depth': 22, 'learning_rate': 0.10886193157248494, 'n_estimators': 500}\n",
      "loss: 0.5319318542468362\n",
      "------------------------------------------------------------------------------------\n",
      "99 ::  {'max_depth': 36, 'learning_rate': 0.21675317754808207, 'n_estimators': 900}\n",
      "loss: 0.5511167532828183\n",
      "------------------------------------------------------------------------------------\n",
      "100 ::  {'max_depth': 34, 'learning_rate': 0.1427324945248647, 'n_estimators': 1600}\n",
      "loss: 0.5485839746895703\n",
      "{'learning_rate': 0.18177994587365698, 'max_depth': 9.0, 'n_estimators': 1100.0}\n"
     ]
    }
   ],
   "source": [
    "space = {'max_depth': hp.quniform('max_depth', -1, 50, 1),\n",
    "         'n_estimators': hp.quniform('n_estimators', 50, 2000, 50),\n",
    "         'learning_rate': hp.uniform('learning_rate', 0.01, 0.25),\n",
    "        }\n",
    "\n",
    "header = [\"iter\", \"acc\", \"f1-score\", \"auc\", \"recall\", \"cm\", \"params\"]\n",
    "write(folder_name, header)\n",
    "\n",
    "ITER = 0\n",
    "trials = Trials()\n",
    "best = fmin(fn=objective_function,\n",
    "            space=space,\n",
    "            algo=tpe.suggest,\n",
    "            rstate= np.random.default_rng(seed),\n",
    "            max_evals=100,\n",
    "            verbose=False,\n",
    "            trials=trials)\n",
    "print(best)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b246ce4b",
   "metadata": {},
   "source": [
    "## Save hyperparameters dictionary to numpy file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "19bddd3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ensure data types are correct\n",
    "best = {\n",
    "        'max_depth': int(best['max_depth']), \n",
    "        'learning_rate': best['learning_rate'],\n",
    "        'n_estimators': int(best['n_estimators']),\n",
    "        }\n",
    "np.save(folder_name + '/LGBM_hyperparameters.npy', best) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c406d3fe",
   "metadata": {},
   "source": [
    "## Single Run of Best Non-Private LGBM Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ba39a394",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance Metrics\n",
      "acc: 0.9437102300538424\n",
      "f1: 0.9708491761723701\n",
      "auc: 0.5563112429087158\n",
      "recall: 0.9876224858174316\n",
      "\n",
      "Fairness Metrics\n",
      "SP_a_1: 0.9927197087883516\n",
      "SP_a_0: 0.8083333333333333\n",
      "DI: 0.8142613933996856\n",
      "SPD: 0.18438637545501824\n",
      "EO_a_1: 0.9940330892324383\n",
      "EO_a_0: 0.8638743455497382\n",
      "EOD: 0.13015874368270008\n",
      "OA_a_1: 0.9544981799271971\n",
      "OA_a_0: 0.7708333333333334\n",
      "OAD: 0.18366484659386373\n"
     ]
    }
   ],
   "source": [
    "params = np.load(folder_name + '/LGBM_hyperparameters.npy', allow_pickle='TRUE').item()\n",
    "\n",
    "model = LGBMClassifier(random_state=seed, n_jobs=-1, objective=\"binary\")\n",
    "model.set_params(**params)\n",
    "model.fit(X_train, y_train)\n",
    "y_pred = model.predict(X_test)\n",
    "\n",
    "# performance metrics\n",
    "acc = accuracy_score(y_test, y_pred)\n",
    "f1 = f1_score(y_test, y_pred)\n",
    "auc = roc_auc_score(y_test, y_pred)\n",
    "recall = recall_score(y_test, y_pred)\n",
    "\n",
    "print('Performance Metrics')\n",
    "print(\"acc:\", acc)\n",
    "print(\"f1:\", f1)\n",
    "print(\"auc:\", auc)\n",
    "print(\"recall:\", recall)\n",
    "\n",
    "# prepare dataset for fairness metrics\n",
    "df_fm = pd.concat([X_test, y_test], axis=1)\n",
    "df_fm['y_pred'] = y_pred\n",
    "\n",
    "print('\\nFairness Metrics')\n",
    "\n",
    "fair_met = fairness_metrics(df_fm, protected_attribute, target)\n",
    "\n",
    "for key in fair_met.keys():\n",
    "    print(key+\":\", fair_met[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa80bbb1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}