tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import RandomizedSearchCV, GridSearchCV\n",
    "from datetime import datetime\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def average(df):\n",
    "    sum = 0\n",
    "    count = 0\n",
    "    for x in df['f1']:\n",
    "        if x!=-1:\n",
    "            sum+=x\n",
    "            count+=1\n",
    "    avg = sum/count\n",
    "    return avg\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def results(model, X_train, X_valid,y_train, y_valid):\n",
    "    soft_probs = model.predict_proba(X_valid)\n",
    "    valid_auc = roc_auc_score(y_valid, soft_probs[:,1])\n",
    "    print(valid_auc)\n",
    "    train_preds = model.predict(X_train)\n",
    "    acc = accuracy_score(y_train, train_preds)\n",
    "    print(acc)\n",
    "    preds = model.predict(X_valid)\n",
    "    acc = accuracy_score(y_valid, preds)\n",
    "    print(acc)\n",
    "    print(classification_report(y_valid,preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Timer function\n",
    "def timer(start_time=None):\n",
    "    if not start_time:\n",
    "        start_time = datetime.now()\n",
    "        return start_time\n",
    "    elif start_time:\n",
    "        thour, temp_sec = divmod((datetime.now() - start_time).total_seconds(), 3600)\n",
    "        tmin, tsec = divmod(temp_sec, 60)\n",
    "        print('\\n Time taken: %i hours %i minutes and %s seconds.' % (thour, tmin, round(tsec, 2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Y</th>\n",
       "      <th>f1</th>\n",
       "      <th>f2</th>\n",
       "      <th>f3</th>\n",
       "      <th>f4</th>\n",
       "      <th>f5</th>\n",
       "      <th>f6</th>\n",
       "      <th>f7</th>\n",
       "      <th>f8</th>\n",
       "      <th>...</th>\n",
       "      <th>f15</th>\n",
       "      <th>f16</th>\n",
       "      <th>f17</th>\n",
       "      <th>f18</th>\n",
       "      <th>f19</th>\n",
       "      <th>f20</th>\n",
       "      <th>f21</th>\n",
       "      <th>f22</th>\n",
       "      <th>f23</th>\n",
       "      <th>f24</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>16383.000000</td>\n",
       "      <td>1.638300e+04</td>\n",
       "      <td>16383.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>8192.000000</td>\n",
       "      <td>0.942135</td>\n",
       "      <td>43031.415720</td>\n",
       "      <td>1.044375</td>\n",
       "      <td>11.770938</td>\n",
       "      <td>118323.581456</td>\n",
       "      <td>1.044436</td>\n",
       "      <td>0.050052</td>\n",
       "      <td>117089.674113</td>\n",
       "      <td>169730.178600</td>\n",
       "      <td>...</td>\n",
       "      <td>25894.316914</td>\n",
       "      <td>119045.099005</td>\n",
       "      <td>184622.040835</td>\n",
       "      <td>1.047305</td>\n",
       "      <td>125959.667765</td>\n",
       "      <td>1.044558</td>\n",
       "      <td>1.045718</td>\n",
       "      <td>1.041934</td>\n",
       "      <td>3.271890e+04</td>\n",
       "      <td>1.043948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>4729.509065</td>\n",
       "      <td>0.233495</td>\n",
       "      <td>33596.053696</td>\n",
       "      <td>0.264806</td>\n",
       "      <td>353.187115</td>\n",
       "      <td>4518.059755</td>\n",
       "      <td>0.265601</td>\n",
       "      <td>0.293892</td>\n",
       "      <td>10261.292970</td>\n",
       "      <td>69396.677853</td>\n",
       "      <td>...</td>\n",
       "      <td>36086.993946</td>\n",
       "      <td>18321.987129</td>\n",
       "      <td>100590.811845</td>\n",
       "      <td>0.306239</td>\n",
       "      <td>31091.344158</td>\n",
       "      <td>0.262576</td>\n",
       "      <td>0.266874</td>\n",
       "      <td>0.246597</td>\n",
       "      <td>3.184929e+06</td>\n",
       "      <td>0.259640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>37.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.770000</td>\n",
       "      <td>23779.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4292.000000</td>\n",
       "      <td>4673.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>4674.000000</td>\n",
       "      <td>3130.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>117879.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>4096.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>20331.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.770000</td>\n",
       "      <td>118096.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>117961.000000</td>\n",
       "      <td>117906.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>4554.000000</td>\n",
       "      <td>118395.000000</td>\n",
       "      <td>118398.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>118274.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>8192.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>35530.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.770000</td>\n",
       "      <td>118300.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>117961.000000</td>\n",
       "      <td>128130.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>13234.000000</td>\n",
       "      <td>118929.000000</td>\n",
       "      <td>119095.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>118568.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>12287.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>74240.500000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.540000</td>\n",
       "      <td>118386.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>117961.000000</td>\n",
       "      <td>234498.500000</td>\n",
       "      <td>...</td>\n",
       "      <td>38902.000000</td>\n",
       "      <td>120539.000000</td>\n",
       "      <td>290919.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>120006.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>9.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>16383.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>312152.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>43910.160000</td>\n",
       "      <td>286791.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>311178.000000</td>\n",
       "      <td>311867.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>311696.000000</td>\n",
       "      <td>286792.000000</td>\n",
       "      <td>308574.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>311867.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>4.042886e+08</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Id             Y             f1            f2            f3  \\\n",
       "count  16383.000000  16383.000000   16383.000000  16383.000000  16383.000000   \n",
       "mean    8192.000000      0.942135   43031.415720      1.044375     11.770938   \n",
       "std     4729.509065      0.233495   33596.053696      0.264806    353.187115   \n",
       "min        1.000000      0.000000      37.000000      1.000000      1.770000   \n",
       "25%     4096.500000      1.000000   20331.000000      1.000000      1.770000   \n",
       "50%     8192.000000      1.000000   35530.000000      1.000000      1.770000   \n",
       "75%    12287.500000      1.000000   74240.500000      1.000000      3.540000   \n",
       "max    16383.000000      1.000000  312152.000000      7.000000  43910.160000   \n",
       "\n",
       "                  f4            f5            f6             f7  \\\n",
       "count   16383.000000  16383.000000  16383.000000   16383.000000   \n",
       "mean   118323.581456      1.044436      0.050052  117089.674113   \n",
       "std      4518.059755      0.265601      0.293892   10261.292970   \n",
       "min     23779.000000      1.000000      0.000000    4292.000000   \n",
       "25%    118096.000000      1.000000      0.000000  117961.000000   \n",
       "50%    118300.000000      1.000000      0.000000  117961.000000   \n",
       "75%    118386.000000      1.000000      0.000000  117961.000000   \n",
       "max    286791.000000      9.000000     10.000000  311178.000000   \n",
       "\n",
       "                  f8  ...            f15            f16            f17  \\\n",
       "count   16383.000000  ...   16383.000000   16383.000000   16383.000000   \n",
       "mean   169730.178600  ...   25894.316914  119045.099005  184622.040835   \n",
       "std     69396.677853  ...   36086.993946   18321.987129  100590.811845   \n",
       "min      4673.000000  ...      25.000000    4674.000000    3130.000000   \n",
       "25%    117906.000000  ...    4554.000000  118395.000000  118398.000000   \n",
       "50%    128130.000000  ...   13234.000000  118929.000000  119095.000000   \n",
       "75%    234498.500000  ...   38902.000000  120539.000000  290919.000000   \n",
       "max    311867.000000  ...  311696.000000  286792.000000  308574.000000   \n",
       "\n",
       "                f18            f19           f20           f21           f22  \\\n",
       "count  16383.000000   16383.000000  16383.000000  16383.000000  16383.000000   \n",
       "mean       1.047305  125959.667765      1.044558      1.045718      1.041934   \n",
       "std        0.306239   31091.344158      0.262576      0.266874      0.246597   \n",
       "min        1.000000  117879.000000      1.000000      1.000000      1.000000   \n",
       "25%        1.000000  118274.000000      1.000000      1.000000      1.000000   \n",
       "50%        1.000000  118568.000000      1.000000      1.000000      1.000000   \n",
       "75%        1.000000  120006.000000      1.000000      1.000000      1.000000   \n",
       "max       18.000000  311867.000000      8.000000      8.000000      7.000000   \n",
       "\n",
       "                f23           f24  \n",
       "count  1.638300e+04  16383.000000  \n",
       "mean   3.271890e+04      1.043948  \n",
       "std    3.184929e+06      0.259640  \n",
       "min    1.000000e+00      1.000000  \n",
       "25%    1.000000e+00      1.000000  \n",
       "50%    2.000000e+00      1.000000  \n",
       "75%    9.000000e+00      1.000000  \n",
       "max    4.042886e+08      8.000000  \n",
       "\n",
       "[8 rows x 26 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"train_final.csv\")\n",
    "df.describe()\n",
    "f1_avg = average(df)\n",
    "df['f1'].replace(-1,f1_avg, inplace = True)\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df.loc[:,'Y']\n",
    "X = df.loc[:,'f1':'f24']\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X,y,train_size = 0.75, test_size = 0.25,random_state = 42, shuffle = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8648734965318772\n",
      "0.9721657035891593\n",
      "0.961181640625\n"
     ]
    }
   ],
   "source": [
    "model_xgb = XGBClassifier(n_estimators=800, max_depth=4, learning_rate=0.05, n_jobs=-1, random_state=42) \n",
    "model_xgb.fit(X_train, y_train)\n",
    "soft_probs = model_xgb.predict_proba(X_valid)\n",
    "valid_auc = roc_auc_score(y_valid, soft_probs[:,1])\n",
    "print(valid_auc)\n",
    "train_preds = model_xgb.predict(X_train)\n",
    "acc = accuracy_score(y_train, train_preds)\n",
    "print(acc)\n",
    "preds = model_xgb.predict(X_valid)\n",
    "acc = accuracy_score(y_valid, preds)\n",
    "print(acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tuning without feature engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "        'min_child_weight': [1, 5, 10],\n",
    "        'subsample': [0.6, 0.8, 1.0],\n",
    "        'colsample_bytree': [0.8, 0.9, 1.0],\n",
    "        'max_depth': [4, 5, 6],\n",
    "        'n_estimators': [800, 1000, 1200]\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb = XGBClassifier(learning_rate=0.05,\n",
    "                    silent=True, nthread=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 324 candidates, totalling 1620 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  16 tasks      | elapsed:   39.1s\n",
      "[Parallel(n_jobs=-1)]: Done 112 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=-1)]: Done 272 tasks      | elapsed: 11.4min\n",
      "[Parallel(n_jobs=-1)]: Done 496 tasks      | elapsed: 23.7min\n",
      "[Parallel(n_jobs=-1)]: Done 784 tasks      | elapsed: 37.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1136 tasks      | elapsed: 59.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1552 tasks      | elapsed: 87.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1620 out of 1620 | elapsed: 92.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[19:31:38] WARNING: C:\\Users\\Administrator\\workspace\\xgboost-win64_release_1.2.0\\src\\learner.cc:516: \n",
      "Parameters: { silent } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      "\n",
      " Time taken: 1 hours 33 minutes and 3.72 seconds.\n"
     ]
    }
   ],
   "source": [
    "# folds = 5\n",
    "# param_comb = 20 #number of random parameter combos to pick\n",
    "# skf = StratifiedKFold(n_splits=folds, shuffle = True, random_state = 42)\n",
    "# grid_search = GridSearchCV(xgb, param_grid=params,scoring='roc_auc', \n",
    "#                             n_jobs=-1, cv=skf.split(X_train,y_train),verbose=3)\n",
    "# start_time = timer(None) # timing starts from this point for \"start_time\" variable\n",
    "# model4 = grid_search.fit(X_train, y_train)\n",
    "# timer(start_time) # timing ends here for \"start_time\" variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.875433515348288\n",
      "0.9798160657605599\n",
      "0.9619140625\n"
     ]
    }
   ],
   "source": [
    "soft_probs = model4.predict_proba(X_valid)\n",
    "valid_auc = roc_auc_score(y_valid, soft_probs[:,1])\n",
    "print(valid_auc)\n",
    "train_preds = model4.predict(X_train)\n",
    "acc = accuracy_score(y_train, train_preds)\n",
    "print(acc)\n",
    "preds = model4.predict(X_valid)\n",
    "acc = accuracy_score(y_valid, preds)\n",
    "print(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'colsample_bytree': 0.6, 'max_depth': 4, 'min_child_weight': 1, 'n_estimators': 1000, 'subsample': 0.8}\n"
     ]
    }
   ],
   "source": [
    "print(model4.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "{'colsample_bytree': 0.6, 'max_depth': 4, 'min_child_weight': 1, 'n_estimators': 1000, 'subsample': 0.8} These are the results from gridsearch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Slight improvement but the grid search was not worth it. Lets just submit this for the sake of a submission. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "testdf = pd.read_csv(\"test_final.csv\")\n",
    "testdf['f1'].replace(-1,f1_avg, inplace = True)\n",
    "X_test = testdf.loc[:,'f1':'f24']\n",
    "#X_test = scaler.transform(X_test)\n",
    "test_preds = model_grid.predict_proba(X_test)\n",
    "sub3 = {\"Id\":testdf['Id'],\"Y\":test_preds[:,1]}\n",
    "sub3 = pd.DataFrame(data=sub3)\n",
    "sub3.to_csv(\"submissions/sub3.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Submission improved score from previous XGBoost. from 0.87347 to 0.88364"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[20:44:08] WARNING: C:\\Users\\Administrator\\workspace\\xgboost-win64_release_1.2.0\\src\\learner.cc:516: \n",
      "Parameters: { njobs } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=0.6, gamma=0, gpu_id=-1,\n",
       "              importance_type='gain', interaction_constraints='',\n",
       "              learning_rate=0.05, max_delta_step=0, max_depth=4,\n",
       "              min_child_weight=1, missing=nan, monotone_constraints='()',\n",
       "              n_estimators=1000, n_jobs=0, njobs=-1, num_parallel_tree=1,\n",
       "              random_state=42, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n",
       "              subsample=1, tree_method='exact', validate_parameters=1,\n",
       "              verbosity=None)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_grid = XGBClassifier(n_estimators = 1000,colsample_bytree=0.6,learning_rate=0.05, njobs=-1,\n",
    "                    max_depth = 4, random_state=42)\n",
    "model_grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8714903704250295\n",
      "0.9750142426955318\n",
      "0.961669921875\n"
     ]
    }
   ],
   "source": [
    "soft_probs = model_grid.predict_proba(X_valid)\n",
    "valid_auc = roc_auc_score(y_valid, soft_probs[:,1])\n",
    "print(valid_auc)\n",
    "train_preds = model_grid.predict(X_train)\n",
    "acc = accuracy_score(y_train, train_preds)\n",
    "print(acc)\n",
    "preds = model_grid.predict(X_valid)\n",
    "acc = accuracy_score(y_valid, preds)\n",
    "print(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[20:44:24] WARNING: C:\\Users\\Administrator\\workspace\\xgboost-win64_release_1.2.0\\src\\learner.cc:516: \n",
      "Parameters: { njobs } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=0.6, gamma=0, gpu_id=-1,\n",
       "              importance_type='gain', interaction_constraints='',\n",
       "              learning_rate=0.05, max_delta_step=0, max_depth=4,\n",
       "              min_child_weight=1, missing=nan, monotone_constraints='()',\n",
       "              n_estimators=1000, n_jobs=0, njobs=-1, num_parallel_tree=1,\n",
       "              random_state=42, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n",
       "              subsample=1, tree_method='exact', validate_parameters=1,\n",
       "              verbosity=None)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#What if I normalize all data and use it\n",
    "scaler = StandardScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_valid = scaler.transform(X_valid)\n",
    "model_grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8714938293240849\n",
      "0.9750142426955318\n",
      "0.961669921875\n"
     ]
    }
   ],
   "source": [
    "soft_probs = model_grid.predict_proba(X_valid)\n",
    "valid_auc = roc_auc_score(y_valid, soft_probs[:,1])\n",
    "print(valid_auc)\n",
    "train_preds = model_grid.predict(X_train)\n",
    "acc = accuracy_score(y_train, train_preds)\n",
    "print(acc)\n",
    "preds = model_grid.predict(X_valid)\n",
    "acc = accuracy_score(y_valid, preds)\n",
    "print(acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Normalizing did not make ANY significant changes to the scores. Continue tuning without normalization. Continue Random search CV instead of grid. Ill now build off of the grid search results. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df.loc[:,'Y']\n",
    "X = df.loc[:,'f1':'f24']\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X,y,train_size = 0.75, test_size = 0.25,random_state = 42, shuffle = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "        'n_estimators' : [1530,1550,1570],\n",
    "        'learning_rate' : [0.01,0.011,0.012],\n",
    "        'colsample_bytree':[0.3,0.4],\n",
    "        'colsample_bylevel':[0.5,0,6]\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_tune = XGBClassifier(subsample = 0.7,max_depth=8,\n",
    "                            n_jobs=-1,base_score = 0.55,\n",
    "                            random_state=42,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 6 folds for each of 25 candidates, totalling 150 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  16 tasks      | elapsed:   25.4s\n",
      "[Parallel(n_jobs=-1)]: Done 112 tasks      | elapsed:  4.6min\n",
      "[Parallel(n_jobs=-1)]: Done 150 out of 150 | elapsed:  6.4min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Time taken: 0 hours 6 minutes and 38.11 seconds.\n"
     ]
    }
   ],
   "source": [
    "folds = 6\n",
    "param_comb = 25 #number of random parameter combos to pick\n",
    "skf = StratifiedKFold(n_splits=folds, shuffle = True, random_state = 42)\n",
    "random_search = RandomizedSearchCV(model_tune, param_distributions=params,\n",
    "                                   n_iter=param_comb, scoring='roc_auc', \n",
    "                                   n_jobs=-1, cv=skf.split(X_train,y_train), \n",
    "                                   verbose=3, random_state=42 )\n",
    "start_time = timer(None) # timing starts from this point for \"start_time\" variable\n",
    "tuned = random_search.fit(X_train, y_train)\n",
    "timer(start_time) # timing ends here for \"start_time\" variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8808005737160567\n",
      "0.9777813949702938\n",
      "0.960205078125\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.30      0.45       224\n",
      "           1       0.96      1.00      0.98      3872\n",
      "\n",
      "    accuracy                           0.96      4096\n",
      "   macro avg       0.94      0.65      0.72      4096\n",
      "weighted avg       0.96      0.96      0.95      4096\n",
      "\n"
     ]
    }
   ],
   "source": [
    "results(tuned,X_train, X_valid, y_train, y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 1530,\n",
       " 'learning_rate': 0.012,\n",
       " 'colsample_bytree': 0.3,\n",
       " 'colsample_bylevel': 0.5}"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tuned.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "{'n_estimators': 1000, 'max_depth': 5, 'learning_rate': 0.03}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8809746716351832\n",
      "0.975665337348417\n",
      "0.960205078125\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.30      0.45       224\n",
      "           1       0.96      1.00      0.98      3872\n",
      "\n",
      "    accuracy                           0.96      4096\n",
      "   macro avg       0.94      0.65      0.72      4096\n",
      "weighted avg       0.96      0.96      0.95      4096\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model_trial = XGBClassifier(colsample_bytree=0.3,subsample = 0.7,max_depth=8,\n",
    "                            n_estimators=1550, learning_rate =0.011,\n",
    "                            colsample_bylevel=0.5,n_jobs=-1,base_score = 0.55,\n",
    "                            random_state=42,)\n",
    "model_trial.fit(X_train, y_train)\n",
    "results(model_trial,X_train, X_valid, y_train, y_valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Until now we had NOT been training on the entire dataset LOOL. We can do much better. Lets try the above on the entire dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.55, booster='gbtree', colsample_bylevel=0.5,\n",
       "              colsample_bynode=1, colsample_bytree=0.3, gamma=0, gpu_id=-1,\n",
       "              importance_type='gain', interaction_constraints='',\n",
       "              learning_rate=0.011, max_delta_step=0, max_depth=8,\n",
       "              min_child_weight=1, missing=nan, monotone_constraints='()',\n",
       "              n_estimators=1550, n_jobs=-1, num_parallel_tree=1,\n",
       "              random_state=42, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n",
       "              subsample=0.7, tree_method='exact', validate_parameters=1,\n",
       "              verbosity=None)"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_trial.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9972919648068188\n",
      "0.9722273087957028\n"
     ]
    }
   ],
   "source": [
    "soft_probs = model_trial.predict_proba(X)\n",
    "valid_auc = roc_auc_score(y, soft_probs[:,1])\n",
    "print(valid_auc)\n",
    "train_preds = model_trial.predict(X)\n",
    "acc = accuracy_score(y, train_preds)\n",
    "print(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Counter testing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## good parameter log\n",
    "After some tuning, lets submit and check what our score is for these params\n",
    "(colsample_bytree=0.6,subsample = 0.95,max_depth=5,\n",
    "                            n_estimators=1310, learning_rate =0.02,\n",
    "                            random_state=42, gamma=1, min_child_weight = 2)\n",
    "th validation accuracy is pmuch the same however there is some regularization I want to test the performance of. \n",
    "(colsample_bytree=0.6,subsample = 0.8,max_depth=4,\n",
    "                            n_estimators=1300, learning_rate =0.03,\n",
    "                            random_state=42)\n",
    "(colsample_bytree=0.4,subsample = 0.7,max_depth=8,\n",
    "                            n_estimators=1850, learning_rate =0.01,\n",
    "                            colsample_bylevel=0.7,base_score = 0.55,\n",
    "                            random_state=42,)\n",
    "Hit 0.88 auc on validation with this one (colsample_bytree=0.3,subsample = 0.7,max_depth=8,\n",
    "                            n_estimators=1550, learning_rate =0.011,\n",
    "                            colsample_bylevel=0.5,n_jobs=-1,base_score = 0.55,\n",
    "                            random_state=42,)\n",
    "Big realization: Had not been fitting on entire dataset after validation. Now I fit it. Time to make a submission with the about best model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "testdf = pd.read_csv(\"test_final.csv\")\n",
    "testdf['f1'].replace(-1,f1_avg, inplace = True)\n",
    "X_test = testdf.loc[:,'f1':'f24']\n",
    "test_preds = model_trial.predict_proba(X_test)\n",
    "sub5 = {\"Id\":testdf['Id'],\"Y\":test_preds[:,1]}\n",
    "sub5 = pd.DataFrame(data=sub5)\n",
    "sub5.to_csv(\"submissions/sub5.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(colsample_bytree=0.6,subsample = 0.95,max_depth=5,\n",
    "                            n_estimators=1310, learning_rate =0.02,\n",
    "                            random_state=42, gamma=1, min_child_weight = 2)Did not work. Still need to tune the parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "GREAT\n",
    "(colsample_bytree=0.3,subsample = 0.7,max_depth=8,\n",
    "                            n_estimators=1550, learning_rate =0.011,\n",
    "                            colsample_bylevel=0.5,n_jobs=-1,base_score = 0.55,\n",
    "                            random_state=42,)\n",
    "XGBoosting on this one improved score on leader board to AUC 0.90550.\n",
    "Time to continue tuning. \n",
    "Notice how the recall is small. Need to improve recall for minority class. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "Now I will explore techniques to address the imbalanced dataset like class weights with the scale_pos_weight parameter. https://stats.stackexchange.com/questions/243207/what-is-the-proper-usage-of-scale-pos-weight-in-xgboost-for-imbalanced-datasets\n",
    "I will also test out smote - Synthetic Minority Oversampling technique\n",
    "\n",
    "Lets also start analyzing the confusion matrix for better insights on what parameters to change"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8809746716351832\n",
      "0.975665337348417\n",
      "0.960205078125\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.30      0.45       224\n",
      "           1       0.96      1.00      0.98      3872\n",
      "\n",
      "    accuracy                           0.96      4096\n",
      "   macro avg       0.94      0.65      0.72      4096\n",
      "weighted avg       0.96      0.96      0.95      4096\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model_trial = XGBClassifier(colsample_bytree=0.3,subsample = 0.7,max_depth=8,\n",
    "                            n_estimators=1550, learning_rate =0.011,\n",
    "                            colsample_bylevel=0.5,n_jobs=-1,base_score = 0.55,\n",
    "                            random_state=42,)\n",
    "#948/16383 -> neg class to pos class ratio #scale_pos_weight=0.05786\n",
    "model_trial.fit(X_train, y_train)\n",
    "results(model_trial,X_train, X_valid, y_train, y_valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tried multiple runs with class weights which did not help much. Improved the recall of minority class but ruined precision by a huge margin. The huge margin is also because maybe the class itself is small in the first place."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(23126, 24)"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#SMOTE\n",
    "from imblearn.over_sampling import SMOTE\n",
    "sm = SMOTE(random_state=42,k_neighbors=100)\n",
    "X_train_res, y_train_res = sm.fit_resample(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8265143060064936\n",
      "0.9759145550462682\n",
      "0.940673828125\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.44      0.33      0.38       224\n",
      "           1       0.96      0.98      0.97      3872\n",
      "\n",
      "    accuracy                           0.94      4096\n",
      "   macro avg       0.70      0.66      0.68      4096\n",
      "weighted avg       0.93      0.94      0.94      4096\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model_trial = XGBClassifier(colsample_bytree=0.3,subsample = 0.7,max_depth=8,\n",
    "                            n_estimators=1550, learning_rate =0.011,\n",
    "                            colsample_bylevel=0.3,n_jobs=-1,base_score = 0.55,\n",
    "                            random_state=42,)\n",
    "model_trial.fit(X_train_res,y_train_res)\n",
    "results(model_trial,X_train_res, X_valid, y_train_res, y_valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Oversampling is not making a dent in improving recall. In fact its ruing accuracy as well. Try undersampling and bagging."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.ensemble import BalancedBaggingClassifier\n",
    "model_trial = XGBClassifier(colsample_bytree=0.8,subsample = 0.8,max_depth=9,\n",
    "                            n_estimators=1500, learning_rate =0.01,\n",
    "                            colsample_bylevel=0.4,n_jobs=-1,base_score = 0.55,\n",
    "                            random_state=42)\n",
    "model_trial_best = XGBClassifier(colsample_bytree=1,subsample = 1,max_depth=4,\n",
    "                            n_estimators=300, learning_rate =0.05,\n",
    "                            colsample_bylevel=1,n_jobs=-1,base_score = 0.55,\n",
    "                            random_state=42,) #Low precision but higher recal (Highest at this time is 0.67)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.843363756272137\n",
      "0.847155530235208\n",
      "0.821044921875\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.18      0.64      0.28       224\n",
      "           1       0.98      0.83      0.90      3872\n",
      "\n",
      "    accuracy                           0.82      4096\n",
      "   macro avg       0.58      0.74      0.59      4096\n",
      "weighted avg       0.93      0.82      0.86      4096\n",
      "\n"
     ]
    }
   ],
   "source": [
    "balanced_bagging = BalancedBaggingClassifier(base_estimator=model_trial_best,\n",
    "...                                 sampling_strategy='auto',n_estimators=90,\n",
    "...                                 replacement=True,\n",
    "...                                 random_state=42)\n",
    "model_bagged = balanced_bagging.fit(X_train, y_train)\n",
    "results(model_bagged,X_train, X_valid, y_train, y_valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Smote and class weights give balanced but low precision and accuracy. Original XGBoost has high accuracy and low recall. Now my balanced bagging gives me the highest recall. Maybe combine the preds?\n",
    "Lets try that."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_trial = XGBClassifier(colsample_bytree=0.3,subsample = 0.7,max_depth=8,\n",
    "                            n_estimators=1550, learning_rate =0.012,\n",
    "                            colsample_bylevel=0.5,n_jobs=-1,base_score = 0.55,\n",
    "                            random_state=42,)\n",
    "for_bagging = XGBClassifier(colsample_bytree=0.3,subsample = 0.7,max_depth=8,\n",
    "                            n_estimators=1050, learning_rate =0.012,\n",
    "                            colsample_bylevel=1,n_jobs=-1,base_score = 0.55,\n",
    "                            random_state=42,)\n",
    "balanced_bagging = BalancedBaggingClassifier(base_estimator=for_bagging,\n",
    "                                 sampling_strategy='auto',n_estimators=25,\n",
    "                                 replacement=True,\n",
    "                                   random_state=42)\n",
    "#SMOTE\n",
    "from imblearn.over_sampling import SMOTE\n",
    "sm = SMOTE(random_state=42,sampling_strategy = 0.4)\n",
    "X_train_res, y_train_res = sm.fit_resample(X_train, y_train)\n",
    "\n",
    "model_smote = XGBClassifier(colsample_bytree=0.3,subsample = 0.7,max_depth=8,\n",
    "                            n_estimators=1550, learning_rate =0.012,\n",
    "                            colsample_bylevel=0.5,n_jobs=-1,base_score = 0.55,\n",
    "                            random_state=42,)\n",
    "\n",
    "\n",
    "model_smote.fit(X_train_res,y_train_res)\n",
    "model_trial.fit(X_train, y_train)\n",
    "model_bagged = balanced_bagging.fit(X_train, y_train)\n",
    "\n",
    "soft_preds_xgb = model_trial.predict_proba(X_valid)\n",
    "soft_preds_bagged = model_bagged.predict_proba(X_valid)\n",
    "soft_preds_smote = model_smote.predict_proba(X_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8666640532762693\n"
     ]
    }
   ],
   "source": [
    "a = 0.5\n",
    "b=0.5\n",
    "soft_probs=a*soft_preds_smote + b* soft_preds_bagged\n",
    "valid_auc = roc_auc_score(y_valid, soft_probs[:,1])\n",
    "print(valid_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8695533869539551\n",
      "0.9859200781313584\n",
      "0.960205078125\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.33      0.48       224\n",
      "           1       0.96      1.00      0.98      3872\n",
      "\n",
      "    accuracy                           0.96      4096\n",
      "   macro avg       0.91      0.66      0.73      4096\n",
      "weighted avg       0.96      0.96      0.95      4096\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#stacking instead\n",
    "from sklearn.ensemble import StackingClassifier, RandomForestClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "estimators = [\n",
    "     ('balanced_xgb', balanced_bagging),\n",
    "     ('simple_xgb',model_trial ),\n",
    "     ('smote',model_smote ),\n",
    " ]\n",
    "stacked = StackingClassifier(estimators = estimators,\n",
    "                              cv=5, n_jobs=-1)\n",
    "stacked.fit(X_train, y_train)\n",
    "results(stacked,X_train, X_valid, y_train, y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "stacked.fit(X,y)\n",
    "testdf = pd.read_csv(\"test_final.csv\")\n",
    "testdf['f1'].replace(-1,f1_avg, inplace = True)\n",
    "X_test = testdf.loc[:,'f1':'f24']\n",
    "test_preds = model_trial.predict_proba(X_test)\n",
    "sub7stacked = {\"Id\":testdf['Id'],\"Y\":test_preds[:,1]}\n",
    "sub7stacked = pd.DataFrame(data=sub7stacked)\n",
    "sub7stacked.to_csv(\"submissions/sub7stacked.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Balanced bagging log\n",
    "Using Balanced bagging Improved the recall of minority class although precision took a hit. Used same xgboost params as best trial yet. \n",
    "Just making a stacked submissions because why not...no evidence of it being better than simple xgboost. \n",
    "\n",
    "DID NOT WORK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Understanding nature of data with resampling\n",
    "Goal is to improve precision and recall of minority class (0 class)\n",
    "Not sure if smote is right for this dataset. Oversampling ruins precision and recall and we are not able to improve on this for now. Need to research further. Try stacking features now."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8809746716351832\n",
      "0.975665337348417\n",
      "0.960205078125\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.30      0.45       224\n",
      "           1       0.96      1.00      0.98      3872\n",
      "\n",
      "    accuracy                           0.96      4096\n",
      "   macro avg       0.94      0.65      0.72      4096\n",
      "weighted avg       0.96      0.96      0.95      4096\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model_trial = XGBClassifier(colsample_bytree=0.3,subsample = 0.7,max_depth=8,\n",
    "                            n_estimators=1550, learning_rate =0.011,\n",
    "                            colsample_bylevel=0.5,n_jobs=-1,base_score = 0.55,\n",
    "                            random_state=42,)\n",
    "model_trial.fit(X_train, y_train)\n",
    "results(model_trial,X_train, X_valid, y_train, y_valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stacking prediction outputs as new feature for xgboost.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.55, booster='gbtree', colsample_bylevel=0.5,\n",
       "              colsample_bynode=1, colsample_bytree=0.3, gamma=0, gpu_id=-1,\n",
       "              importance_type='gain', interaction_constraints='',\n",
       "              learning_rate=0.011, max_delta_step=0, max_depth=8,\n",
       "              min_child_weight=1, missing=nan, monotone_constraints='()',\n",
       "              n_estimators=1550, n_jobs=-1, num_parallel_tree=1,\n",
       "              random_state=42, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n",
       "              subsample=0.7, tree_method='exact', validate_parameters=1,\n",
       "              verbosity=None)"
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_train = model_trial.predict(X_train)\n",
    "preds_train=np.reshape(preds_train,(-1,1))\n",
    "X_train_add = np.concatenate((X_train,preds_train),1)\n",
    "model_trial.fit(X_train_add, y_train)\n",
    "preds_valid = model_trial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8431700579250295\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "feature_names mismatch: ['f0', 'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f20', 'f21', 'f22', 'f23', 'f24'] ['f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f20', 'f21', 'f22', 'f23', 'f24']\nexpected f0 in input data",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-217-b61327bd1ac6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mresults\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel_trial\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_valid\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_valid\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-14-b9d04ec192f1>\u001b[0m in \u001b[0;36mresults\u001b[1;34m(model, X_train, X_valid, y_train, y_valid)\u001b[0m\n\u001b[0;32m      3\u001b[0m     \u001b[0mvalid_auc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mroc_auc_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_valid\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msoft_probs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalid_auc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m     \u001b[0mtrain_preds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m     \u001b[0macc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0maccuracy_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_preds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0macc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda3\\envs\\dslab\\lib\\site-packages\\xgboost\\sklearn.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, data, output_margin, ntree_limit, validate_features, base_margin)\u001b[0m\n\u001b[0;32m    892\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mntree_limit\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    893\u001b[0m             \u001b[0mntree_limit\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"best_ntree_limit\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 894\u001b[1;33m         class_probs = self.get_booster().predict(\n\u001b[0m\u001b[0;32m    895\u001b[0m             \u001b[0mtest_dmatrix\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    896\u001b[0m             \u001b[0moutput_margin\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moutput_margin\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda3\\envs\\dslab\\lib\\site-packages\\xgboost\\core.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, data, output_margin, ntree_limit, pred_leaf, pred_contribs, approx_contribs, pred_interactions, validate_features, training)\u001b[0m\n\u001b[0;32m   1362\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1363\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mvalidate_features\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1364\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_validate_features\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1365\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1366\u001b[0m         \u001b[0mlength\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mc_bst_ulong\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\anaconda3\\envs\\dslab\\lib\\site-packages\\xgboost\\core.py\u001b[0m in \u001b[0;36m_validate_features\u001b[1;34m(self, data)\u001b[0m\n\u001b[0;32m   1933\u001b[0m                             ', '.join(str(s) for s in my_missing))\n\u001b[0;32m   1934\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1935\u001b[1;33m                 raise ValueError(msg.format(self.feature_names,\n\u001b[0m\u001b[0;32m   1936\u001b[0m                                             data.feature_names))\n\u001b[0;32m   1937\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: feature_names mismatch: ['f0', 'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f20', 'f21', 'f22', 'f23', 'f24'] ['f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f20', 'f21', 'f22', 'f23', 'f24']\nexpected f0 in input data"
     ]
    }
   ],
   "source": [
    "results(model_trial,X_train, X_valid, y_train, y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "        'n_estimators' : [1550,1600,1650,1800,2000],\n",
    "        'learning_rate' : [0.011,0.013,0.02,0.015,0.01],\n",
    "        'colsample_bylevel':[0.4,0.5,0.6,0.7],\n",
    "        'colsample_bytree':[0.4,0.3,0.5,0.6,0.7,0.8],\n",
    "        'min_child_weight':[1,2,3],\n",
    "        'gamma':[0,1,2],\n",
    "        'base_score':[0.5,0.6,0.7]\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_tune = XGBClassifier(subsample = 0.7,\n",
    "                           max_depth=8,scale_pos_weight=0.0578,\n",
    "                           random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 6 folds for each of 9 candidates, totalling 54 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  16 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done  54 out of  54 | elapsed:  5.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Time taken: 0 hours 6 minutes and 19.71 seconds.\n"
     ]
    }
   ],
   "source": [
    "folds = 6\n",
    "param_comb = 9 #number of random parameter combos to pick\n",
    "skf = StratifiedKFold(n_splits=folds, shuffle = True, random_state = 42)\n",
    "random_search = RandomizedSearchCV(model_tune, param_distributions=params,\n",
    "                                   n_iter=param_comb, scoring='roc_auc', \n",
    "                                   n_jobs=-1, cv=skf.split(X_train,y_train), \n",
    "                                   verbose=3, random_state=42 )\n",
    "start_time = timer(None) # timing starts from this point for \"start_time\" variable\n",
    "tuned = random_search.fit(X_train, y_train)\n",
    "timer(start_time) # timing ends here for \"start_time\" variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8695441632231404\n",
      "0.9850248229836412\n",
      "0.944091796875\n"
     ]
    }
   ],
   "source": [
    "results(tuned,X_train, X_valid, y_train, y_valid)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 2000,\n",
       " 'min_child_weight': 1,\n",
       " 'learning_rate': 0.02,\n",
       " 'gamma': 0,\n",
       " 'colsample_bytree': 0.5,\n",
       " 'colsample_bylevel': 0.4,\n",
       " 'base_score': 0.7}"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tuned.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Moving to tuning two file. Conclusion of this session: Tried balanced bagging, SMOTE, stacking, class weighing. Time to take our best XGBoost model and ensemble with LGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}