random_forest_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read the CSV and Perform Basic Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>koi_disposition</th>\n",
       "      <th>koi_fpflag_nt</th>\n",
       "      <th>koi_fpflag_ss</th>\n",
       "      <th>koi_fpflag_co</th>\n",
       "      <th>koi_fpflag_ec</th>\n",
       "      <th>koi_period</th>\n",
       "      <th>koi_period_err1</th>\n",
       "      <th>koi_period_err2</th>\n",
       "      <th>koi_time0bk</th>\n",
       "      <th>koi_time0bk_err1</th>\n",
       "      <th>...</th>\n",
       "      <th>koi_steff_err2</th>\n",
       "      <th>koi_slogg</th>\n",
       "      <th>koi_slogg_err1</th>\n",
       "      <th>koi_slogg_err2</th>\n",
       "      <th>koi_srad</th>\n",
       "      <th>koi_srad_err1</th>\n",
       "      <th>koi_srad_err2</th>\n",
       "      <th>ra</th>\n",
       "      <th>dec</th>\n",
       "      <th>koi_kepmag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CONFIRMED</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>54.418383</td>\n",
       "      <td>2.479000e-04</td>\n",
       "      <td>-2.479000e-04</td>\n",
       "      <td>162.513840</td>\n",
       "      <td>0.003520</td>\n",
       "      <td>...</td>\n",
       "      <td>-81</td>\n",
       "      <td>4.467</td>\n",
       "      <td>0.064</td>\n",
       "      <td>-0.096</td>\n",
       "      <td>0.927</td>\n",
       "      <td>0.105</td>\n",
       "      <td>-0.061</td>\n",
       "      <td>291.93423</td>\n",
       "      <td>48.141651</td>\n",
       "      <td>15.347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FALSE POSITIVE</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19.899140</td>\n",
       "      <td>1.490000e-05</td>\n",
       "      <td>-1.490000e-05</td>\n",
       "      <td>175.850252</td>\n",
       "      <td>0.000581</td>\n",
       "      <td>...</td>\n",
       "      <td>-176</td>\n",
       "      <td>4.544</td>\n",
       "      <td>0.044</td>\n",
       "      <td>-0.176</td>\n",
       "      <td>0.868</td>\n",
       "      <td>0.233</td>\n",
       "      <td>-0.078</td>\n",
       "      <td>297.00482</td>\n",
       "      <td>48.134129</td>\n",
       "      <td>15.436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FALSE POSITIVE</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.736952</td>\n",
       "      <td>2.630000e-07</td>\n",
       "      <td>-2.630000e-07</td>\n",
       "      <td>170.307565</td>\n",
       "      <td>0.000115</td>\n",
       "      <td>...</td>\n",
       "      <td>-174</td>\n",
       "      <td>4.564</td>\n",
       "      <td>0.053</td>\n",
       "      <td>-0.168</td>\n",
       "      <td>0.791</td>\n",
       "      <td>0.201</td>\n",
       "      <td>-0.067</td>\n",
       "      <td>285.53461</td>\n",
       "      <td>48.285210</td>\n",
       "      <td>15.597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CONFIRMED</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.525592</td>\n",
       "      <td>3.760000e-06</td>\n",
       "      <td>-3.760000e-06</td>\n",
       "      <td>171.595550</td>\n",
       "      <td>0.001130</td>\n",
       "      <td>...</td>\n",
       "      <td>-211</td>\n",
       "      <td>4.438</td>\n",
       "      <td>0.070</td>\n",
       "      <td>-0.210</td>\n",
       "      <td>1.046</td>\n",
       "      <td>0.334</td>\n",
       "      <td>-0.133</td>\n",
       "      <td>288.75488</td>\n",
       "      <td>48.226200</td>\n",
       "      <td>15.509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CONFIRMED</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.134435</td>\n",
       "      <td>1.050000e-05</td>\n",
       "      <td>-1.050000e-05</td>\n",
       "      <td>172.979370</td>\n",
       "      <td>0.001900</td>\n",
       "      <td>...</td>\n",
       "      <td>-232</td>\n",
       "      <td>4.486</td>\n",
       "      <td>0.054</td>\n",
       "      <td>-0.229</td>\n",
       "      <td>0.972</td>\n",
       "      <td>0.315</td>\n",
       "      <td>-0.105</td>\n",
       "      <td>296.28613</td>\n",
       "      <td>48.224670</td>\n",
       "      <td>15.714</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  koi_disposition  koi_fpflag_nt  koi_fpflag_ss  koi_fpflag_co  koi_fpflag_ec  \\\n",
       "0       CONFIRMED              0              0              0              0   \n",
       "1  FALSE POSITIVE              0              1              0              0   \n",
       "2  FALSE POSITIVE              0              1              0              0   \n",
       "3       CONFIRMED              0              0              0              0   \n",
       "4       CONFIRMED              0              0              0              0   \n",
       "\n",
       "   koi_period  koi_period_err1  koi_period_err2  koi_time0bk  \\\n",
       "0   54.418383     2.479000e-04    -2.479000e-04   162.513840   \n",
       "1   19.899140     1.490000e-05    -1.490000e-05   175.850252   \n",
       "2    1.736952     2.630000e-07    -2.630000e-07   170.307565   \n",
       "3    2.525592     3.760000e-06    -3.760000e-06   171.595550   \n",
       "4    4.134435     1.050000e-05    -1.050000e-05   172.979370   \n",
       "\n",
       "   koi_time0bk_err1  ...  koi_steff_err2  koi_slogg  koi_slogg_err1  \\\n",
       "0          0.003520  ...             -81      4.467           0.064   \n",
       "1          0.000581  ...            -176      4.544           0.044   \n",
       "2          0.000115  ...            -174      4.564           0.053   \n",
       "3          0.001130  ...            -211      4.438           0.070   \n",
       "4          0.001900  ...            -232      4.486           0.054   \n",
       "\n",
       "   koi_slogg_err2  koi_srad  koi_srad_err1  koi_srad_err2         ra  \\\n",
       "0          -0.096     0.927          0.105         -0.061  291.93423   \n",
       "1          -0.176     0.868          0.233         -0.078  297.00482   \n",
       "2          -0.168     0.791          0.201         -0.067  285.53461   \n",
       "3          -0.210     1.046          0.334         -0.133  288.75488   \n",
       "4          -0.229     0.972          0.315         -0.105  296.28613   \n",
       "\n",
       "         dec  koi_kepmag  \n",
       "0  48.141651      15.347  \n",
       "1  48.134129      15.436  \n",
       "2  48.285210      15.597  \n",
       "3  48.226200      15.509  \n",
       "4  48.224670      15.714  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# decision tree is very fast, even using the full data it will still finish in seconds.\n",
    "df = pd.read_csv(\"exoplanet_data.csv\")\n",
    "# Drop the null columns where all values are null\n",
    "df = df.dropna(axis='columns', how='all')\n",
    "# Drop the null rows\n",
    "df = df.dropna()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Select your features (columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6991, 16)\n"
     ]
    }
   ],
   "source": [
    "# Set features. This will also be used as your x values.\n",
    "# remove disposition flags, and errors columns\n",
    "selected_features = df[['koi_period', 'koi_time0bk', 'koi_impact', 'koi_duration', 'koi_depth', \n",
    "                        'koi_prad', 'koi_teq', 'koi_insol', 'koi_model_snr', 'koi_tce_plnt_num', \n",
    "                        'koi_steff', 'koi_slogg', 'koi_srad', 'ra', 'dec', 'koi_kepmag']]\n",
    "X = selected_features.values\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create a Train Test Split\n",
    "\n",
    "Use `koi_disposition` for the y values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 2, ..., 0, 2, 2])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# prepare y values\n",
    "y = df['koi_disposition'].values\n",
    "\n",
    "# Step 1: Label-encode data set\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "label_encoder = LabelEncoder()\n",
    "label_encoder.fit(y)\n",
    "encoded_y = label_encoder.transform(y)\n",
    "\n",
    "encoded_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, encoded_y, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.05484133e+01, 1.39064020e+02, 1.01700000e+00, ...,\n",
       "        2.98095430e+02, 4.47370610e+01, 1.32040000e+01],\n",
       "       [2.47543849e+01, 1.40207320e+02, 7.09000000e-01, ...,\n",
       "        2.95735350e+02, 4.25762480e+01, 1.55140000e+01],\n",
       "       [1.05733568e+00, 1.31792007e+02, 2.62000000e-01, ...,\n",
       "        2.92184170e+02, 4.93100400e+01, 1.54140000e+01],\n",
       "       ...,\n",
       "       [1.07027283e+00, 1.31835891e+02, 1.26900000e+00, ...,\n",
       "        2.99170620e+02, 4.08710590e+01, 1.59260000e+01],\n",
       "       [1.87464901e+01, 1.47130740e+02, 5.08000000e-01, ...,\n",
       "        2.88164700e+02, 4.58165090e+01, 1.58430000e+01],\n",
       "       [2.10907909e+01, 1.33963160e+02, 8.32000000e-01, ...,\n",
       "        2.85475040e+02, 4.80349310e+01, 1.42230000e+01]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf_model = RandomForestClassifier(random_state = 1)\n",
    "rf_model = rf_model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_score: 1.0\n",
      "test_score: 0.7654462242562929\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(0.06983886619639082, 'koi_period'),\n",
       " (0.056339065884505504, 'koi_time0bk'),\n",
       " (0.08204143111698879, 'koi_impact'),\n",
       " (0.06885226911480116, 'koi_duration'),\n",
       " (0.08170508570901058, 'koi_depth'),\n",
       " (0.12124406537329678, 'koi_prad'),\n",
       " (0.0611652648297357, 'koi_teq'),\n",
       " (0.061253698645244145, 'koi_insol'),\n",
       " (0.1412132144321545, 'koi_model_snr'),\n",
       " (0.0156662982924218, 'koi_tce_plnt_num'),\n",
       " (0.03984142338687951, 'koi_steff'),\n",
       " (0.03638505632707633, 'koi_slogg'),\n",
       " (0.03641348293926122, 'koi_srad'),\n",
       " (0.04756430572012431, 'ra'),\n",
       " (0.04070664542470149, 'dec'),\n",
       " (0.039769826607407334, 'koi_kepmag')]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_score = rf_model.score(X_train, y_train)\n",
    "test_score = rf_model.score(X_test, y_test)\n",
    "print(f\"train_score: {train_score}\")\n",
    "print(f\"test_score: {test_score}\")\n",
    "\n",
    "# print feature importance\n",
    "feature_importance = zip(rf_model.feature_importances_, selected_features.columns)\n",
    "list(feature_importance)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.1412132144321545, 'koi_model_snr'),\n",
       " (0.12124406537329678, 'koi_prad'),\n",
       " (0.08204143111698879, 'koi_impact'),\n",
       " (0.08170508570901058, 'koi_depth'),\n",
       " (0.06983886619639082, 'koi_period'),\n",
       " (0.06885226911480116, 'koi_duration'),\n",
       " (0.061253698645244145, 'koi_insol'),\n",
       " (0.0611652648297357, 'koi_teq'),\n",
       " (0.056339065884505504, 'koi_time0bk'),\n",
       " (0.04756430572012431, 'ra'),\n",
       " (0.04070664542470149, 'dec'),\n",
       " (0.03984142338687951, 'koi_steff'),\n",
       " (0.039769826607407334, 'koi_kepmag'),\n",
       " (0.03641348293926122, 'koi_srad'),\n",
       " (0.03638505632707633, 'koi_slogg'),\n",
       " (0.0156662982924218, 'koi_tce_plnt_num')]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# print sorted feature importance\n",
    "sorted_feature_importance = sorted(zip(rf_model.feature_importances_, selected_features.columns), reverse=True)\n",
    "list(sorted_feature_importance)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameter Tunning with GridSearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'bootstrap': True, 'ccp_alpha': 0.0, 'class_weight': None, 'criterion': 'gini', 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'max_samples': None, 'min_impurity_decrease': 0.0, 'min_impurity_split': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'n_estimators': 100, 'n_jobs': None, 'oob_score': False, 'random_state': 1, 'verbose': 0, 'warm_start': False}\n"
     ]
    }
   ],
   "source": [
    "# show current parameter\n",
    "print(rf_model.get_params())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "# Create the parameter grid based on the results of random search \n",
    "param_grid = {\n",
    "    'bootstrap': [True],\n",
    "    'max_depth': [85, 90, 95],\n",
    "    'max_features': [2, 3, 4],\n",
    "    'min_samples_leaf': [2, 3, 4],\n",
    "    'min_samples_split': [6, 7, 8, 9],\n",
    "    'n_estimators': [250, 300, 400, 500]\n",
    "}\n",
    "\n",
    "# Instantiate the grid search model\n",
    "grid_search = GridSearchCV(estimator = rf_model, param_grid = param_grid, \n",
    "                          cv = 3, n_jobs = 4, verbose = 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 432 candidates, totalling 1296 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  24 tasks      | elapsed:   15.1s\n",
      "[Parallel(n_jobs=4)]: Done 120 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=4)]: Done 280 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=4)]: Done 504 tasks      | elapsed:  6.0min\n",
      "[Parallel(n_jobs=4)]: Done 792 tasks      | elapsed:  9.5min\n",
      "[Parallel(n_jobs=4)]: Done 1144 tasks      | elapsed: 13.6min\n",
      "[Parallel(n_jobs=4)]: Done 1296 out of 1296 | elapsed: 15.8min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, estimator=RandomForestClassifier(random_state=1), n_jobs=4,\n",
       "             param_grid={'bootstrap': [True], 'max_depth': [85, 90, 95],\n",
       "                         'max_features': [2, 3, 4],\n",
       "                         'min_samples_leaf': [2, 3, 4],\n",
       "                         'min_samples_split': [6, 7, 8, 9],\n",
       "                         'n_estimators': [250, 300, 400, 500]},\n",
       "             verbose=3)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit grid search to the data\n",
    "grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bootstrap': True,\n",
       " 'max_depth': 85,\n",
       " 'max_features': 3,\n",
       " 'min_samples_leaf': 3,\n",
       " 'min_samples_split': 6,\n",
       " 'n_estimators': 250}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7596808651378827\n"
     ]
    }
   ],
   "source": [
    "print(grid_search.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_score: 0.955750524508869\n",
      "test_score: 0.7734553775743707\n"
     ]
    }
   ],
   "source": [
    "best_grid = grid_search.best_estimator_\n",
    "train_score = best_grid.score(X_train, y_train)\n",
    "test_score = best_grid.score(X_test, y_test)\n",
    "print(f\"train_score: {train_score}\")\n",
    "print(f\"test_score: {test_score}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Improvement = 0.7734 - 0.7654 = 0.008 = 0.8% accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove any feature which has importance less than 0.05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6991, 9)\n"
     ]
    }
   ],
   "source": [
    "# reduce feature from 16 down to 9\n",
    "selected_features2 = df[['koi_period', 'koi_time0bk', 'koi_impact', 'koi_duration', 'koi_depth', \n",
    "                        'koi_prad', 'koi_teq', 'koi_insol', 'koi_model_snr']]\n",
    "X2 = selected_features2.values\n",
    "print(X2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \n",
    "X_train2, X_test2, y_train2, y_test2 = train_test_split(X2, encoded_y, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_model2 = RandomForestClassifier(n_estimators=200)\n",
    "rf_model2 = rf_model2.fit(X_train2, y_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_score2: 1.0\n",
      "test_score2: 0.7654462242562929\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(0.10707327075244918, 'koi_period'),\n",
       " (0.07782068761782276, 'koi_time0bk'),\n",
       " (0.10628674740361546, 'koi_impact'),\n",
       " (0.10114738339056215, 'koi_duration'),\n",
       " (0.10062295314106147, 'koi_depth'),\n",
       " (0.165897023590182, 'koi_prad'),\n",
       " (0.08626121239053582, 'koi_teq'),\n",
       " (0.08631650271123194, 'koi_insol'),\n",
       " (0.16857421900253927, 'koi_model_snr')]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_score2 = rf_model2.score(X_train2, y_train2)\n",
    "test_score2 = rf_model2.score(X_test2, y_test2)\n",
    "print(f\"train_score2: {train_score2}\")\n",
    "print(f\"test_score2: {test_score2}\")\n",
    "\n",
    "# print feature importance\n",
    "feature_importance2 = zip(rf_model2.feature_importances_, selected_features2.columns)\n",
    "list(feature_importance2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.16857421900253927, 'koi_model_snr'),\n",
       " (0.165897023590182, 'koi_prad'),\n",
       " (0.10707327075244918, 'koi_period'),\n",
       " (0.10628674740361546, 'koi_impact'),\n",
       " (0.10114738339056215, 'koi_duration'),\n",
       " (0.10062295314106147, 'koi_depth'),\n",
       " (0.08631650271123194, 'koi_insol'),\n",
       " (0.08626121239053582, 'koi_teq'),\n",
       " (0.07782068761782276, 'koi_time0bk')]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# print sorted feature importance\n",
    "sorted_feature_importance2 = sorted(zip(rf_model2.feature_importances_, selected_features2.columns), reverse=True)\n",
    "list(sorted_feature_importance2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conclusion: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove less importance feature doesn't show improvments on accuracy\n",
    "# but it could help speed up model calculation in deep learning model!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['jamesye_random_forest_model.sav']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save your model by updating \"your_name\" with your name\n",
    "# and \"your_model\" with your model variable\n",
    "# be sure to turn this in to BCS\n",
    "# if joblib fails to import, try running the command to install in terminal/git-bash\n",
    "import joblib\n",
    "filename = 'jamesye_random_forest_model.sav'\n",
    "joblib.dump(best_grid, filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7734553775743707"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loaded_model = joblib.load(filename)\n",
    "result = loaded_model.score(X_test, y_test)\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernel_info": {
   "name": "dev"
  },
  "kernelspec": {
   "display_name": "PythonAdv",
   "language": "python",
   "name": "pythonadv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "nteract": {
   "version": "0.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}