(for reference only) 1 USD Pledge_model (sub category) .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import library\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: read the cleaned up dataframe by Stephy and Sanjay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>blurb_length</th>\n",
       "      <th>name_length</th>\n",
       "      <th>usd_pledged</th>\n",
       "      <th>success</th>\n",
       "      <th>US based</th>\n",
       "      <th>main_category_comics</th>\n",
       "      <th>main_category_crafts</th>\n",
       "      <th>main_category_dance</th>\n",
       "      <th>main_category_design</th>\n",
       "      <th>...</th>\n",
       "      <th>main_category_games</th>\n",
       "      <th>main_category_journalism</th>\n",
       "      <th>main_category_music</th>\n",
       "      <th>main_category_photography</th>\n",
       "      <th>main_category_publishing</th>\n",
       "      <th>main_category_technology</th>\n",
       "      <th>main_category_theater</th>\n",
       "      <th>start_Q_Q2</th>\n",
       "      <th>start_Q_Q3</th>\n",
       "      <th>start_Q_Q4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.163043</td>\n",
       "      <td>0.382353</td>\n",
       "      <td>0.230769</td>\n",
       "      <td>0.000705</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.315217</td>\n",
       "      <td>0.676471</td>\n",
       "      <td>0.269231</td>\n",
       "      <td>0.000455</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.315217</td>\n",
       "      <td>0.588235</td>\n",
       "      <td>0.230769</td>\n",
       "      <td>0.000129</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.478261</td>\n",
       "      <td>0.411765</td>\n",
       "      <td>0.192308</td>\n",
       "      <td>0.000559</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.641304</td>\n",
       "      <td>0.411765</td>\n",
       "      <td>0.115385</td>\n",
       "      <td>0.004696</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   duration  blurb_length  name_length  usd_pledged  success  US based  \\\n",
       "0  0.163043      0.382353     0.230769     0.000705      1.0       1.0   \n",
       "1  0.315217      0.676471     0.269231     0.000455      1.0       0.0   \n",
       "2  0.315217      0.588235     0.230769     0.000129      1.0       1.0   \n",
       "3  0.478261      0.411765     0.192308     0.000559      1.0       1.0   \n",
       "4  0.641304      0.411765     0.115385     0.004696      1.0       1.0   \n",
       "\n",
       "   main_category_comics  main_category_crafts  main_category_dance  \\\n",
       "0                   0.0                   0.0                  0.0   \n",
       "1                   1.0                   0.0                  0.0   \n",
       "2                   0.0                   0.0                  0.0   \n",
       "3                   0.0                   0.0                  0.0   \n",
       "4                   0.0                   0.0                  0.0   \n",
       "\n",
       "   main_category_design  ...  main_category_games  main_category_journalism  \\\n",
       "0                   0.0  ...                  1.0                       0.0   \n",
       "1                   0.0  ...                  0.0                       0.0   \n",
       "2                   0.0  ...                  0.0                       0.0   \n",
       "3                   0.0  ...                  0.0                       0.0   \n",
       "4                   0.0  ...                  0.0                       0.0   \n",
       "\n",
       "   main_category_music  main_category_photography  main_category_publishing  \\\n",
       "0                  0.0                        0.0                       0.0   \n",
       "1                  0.0                        0.0                       0.0   \n",
       "2                  0.0                        0.0                       0.0   \n",
       "3                  1.0                        0.0                       0.0   \n",
       "4                  0.0                        0.0                       0.0   \n",
       "\n",
       "   main_category_technology  main_category_theater  start_Q_Q2  start_Q_Q3  \\\n",
       "0                       0.0                    0.0         0.0         0.0   \n",
       "1                       0.0                    0.0         0.0         1.0   \n",
       "2                       0.0                    0.0         1.0         0.0   \n",
       "3                       0.0                    0.0         0.0         1.0   \n",
       "4                       1.0                    0.0         0.0         0.0   \n",
       "\n",
       "   start_Q_Q4  \n",
       "0         1.0  \n",
       "1         0.0  \n",
       "2         0.0  \n",
       "3         0.0  \n",
       "4         1.0  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data\\processed_data(by_quarter).csv')\n",
    "df.drop(['Unnamed: 0', 'goal_usd'], axis = 1, inplace = True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(170730, 23)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: read the raw dataframe so that there's columns for Main and Sub category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_cliff = pd.read_csv('data\\Kickstarter_projects_Feb19.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Below cell: Dropping duplicates from raw dataframe so the rows from cleanup dataframe and raw dataframe match up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_cliff.drop_duplicates(keep='first', inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell:\n",
    "### 1. create a new column called main_sub_category that is a merge of main_category and sub_category\n",
    "### 2. OneHotEncode the \"main_sub_category\" column\n",
    "### 3. Name the newly encoded columns (169 of them) by OHE.get_feature_names()\n",
    "### 4. Merge the encoded columns with the cleanup dateframe from Stephy and Sanjay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Making a new column from main_category and sub_category,  this way, we preserve the uniqueness of each sub_category, since 'Comedy' sub_category appears in 2 main_category(s)\n",
    "df_cliff['main_sub_category'] = df_cliff['main_category'] + ' ' + df_cliff['sub_category']\n",
    "\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "OHE = OneHotEncoder(sparse = False)   # add drop = 'first'  if you wanna drop first encoded column\n",
    "x = OHE.fit_transform(df1[['main_sub_category']])\n",
    "\n",
    "# Below, I am naming the new encoded columns using the OHE.get_feature_names() method since in the documentation example, the appearance of the encoded columns are sorted, and get_feature_names is also sorted, and the dimension matches up.\n",
    "\n",
    "df_one_hot_cat = pd.DataFrame(x, columns = OHE.get_feature_names())\n",
    "df_final = pd.concat([df,df_one_hot_cat], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>blurb_length</th>\n",
       "      <th>name_length</th>\n",
       "      <th>usd_pledged</th>\n",
       "      <th>success</th>\n",
       "      <th>US based</th>\n",
       "      <th>main_category_comics</th>\n",
       "      <th>main_category_crafts</th>\n",
       "      <th>main_category_dance</th>\n",
       "      <th>main_category_design</th>\n",
       "      <th>...</th>\n",
       "      <th>x0_technology Wearables</th>\n",
       "      <th>x0_technology Web</th>\n",
       "      <th>x0_theater Comedy</th>\n",
       "      <th>x0_theater Experimental</th>\n",
       "      <th>x0_theater Festivals</th>\n",
       "      <th>x0_theater Immersive</th>\n",
       "      <th>x0_theater Musical</th>\n",
       "      <th>x0_theater Plays</th>\n",
       "      <th>x0_theater Spaces</th>\n",
       "      <th>x0_theater Theater</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.163043</td>\n",
       "      <td>0.382353</td>\n",
       "      <td>0.230769</td>\n",
       "      <td>7.050565e-04</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.315217</td>\n",
       "      <td>0.676471</td>\n",
       "      <td>0.269231</td>\n",
       "      <td>4.553617e-04</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.315217</td>\n",
       "      <td>0.588235</td>\n",
       "      <td>0.230769</td>\n",
       "      <td>1.291227e-04</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.478261</td>\n",
       "      <td>0.411765</td>\n",
       "      <td>0.192308</td>\n",
       "      <td>5.591827e-04</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.641304</td>\n",
       "      <td>0.411765</td>\n",
       "      <td>0.115385</td>\n",
       "      <td>4.695878e-03</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170725</th>\n",
       "      <td>0.641304</td>\n",
       "      <td>0.588235</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>1.424143e-07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170726</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.038462</td>\n",
       "      <td>1.829517e-05</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170727</th>\n",
       "      <td>0.315217</td>\n",
       "      <td>0.352941</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170728</th>\n",
       "      <td>0.641304</td>\n",
       "      <td>0.058824</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>1.663473e-05</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170729</th>\n",
       "      <td>0.641304</td>\n",
       "      <td>0.558824</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>3.617762e-05</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>170730 rows × 192 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        duration  blurb_length  name_length   usd_pledged  success  US based  \\\n",
       "0       0.163043      0.382353     0.230769  7.050565e-04      1.0       1.0   \n",
       "1       0.315217      0.676471     0.269231  4.553617e-04      1.0       0.0   \n",
       "2       0.315217      0.588235     0.230769  1.291227e-04      1.0       1.0   \n",
       "3       0.478261      0.411765     0.192308  5.591827e-04      1.0       1.0   \n",
       "4       0.641304      0.411765     0.115385  4.695878e-03      1.0       1.0   \n",
       "...          ...           ...          ...           ...      ...       ...   \n",
       "170725  0.641304      0.588235     0.076923  1.424143e-07      0.0       0.0   \n",
       "170726  0.500000      0.500000     0.038462  1.829517e-05      0.0       0.0   \n",
       "170727  0.315217      0.352941     0.076923  0.000000e+00      0.0       1.0   \n",
       "170728  0.641304      0.058824     0.076923  1.663473e-05      0.0       1.0   \n",
       "170729  0.641304      0.558824     0.076923  3.617762e-05      0.0       1.0   \n",
       "\n",
       "        main_category_comics  main_category_crafts  main_category_dance  \\\n",
       "0                        0.0                   0.0                  0.0   \n",
       "1                        1.0                   0.0                  0.0   \n",
       "2                        0.0                   0.0                  0.0   \n",
       "3                        0.0                   0.0                  0.0   \n",
       "4                        0.0                   0.0                  0.0   \n",
       "...                      ...                   ...                  ...   \n",
       "170725                   0.0                   0.0                  0.0   \n",
       "170726                   0.0                   0.0                  0.0   \n",
       "170727                   0.0                   0.0                  0.0   \n",
       "170728                   0.0                   0.0                  0.0   \n",
       "170729                   0.0                   1.0                  0.0   \n",
       "\n",
       "        main_category_design  ...  x0_technology Wearables  x0_technology Web  \\\n",
       "0                        0.0  ...                      0.0                0.0   \n",
       "1                        0.0  ...                      0.0                0.0   \n",
       "2                        0.0  ...                      0.0                0.0   \n",
       "3                        0.0  ...                      0.0                0.0   \n",
       "4                        0.0  ...                      0.0                0.0   \n",
       "...                      ...  ...                      ...                ...   \n",
       "170725                   0.0  ...                      0.0                0.0   \n",
       "170726                   0.0  ...                      0.0                0.0   \n",
       "170727                   0.0  ...                      0.0                0.0   \n",
       "170728                   0.0  ...                      0.0                0.0   \n",
       "170729                   0.0  ...                      0.0                0.0   \n",
       "\n",
       "        x0_theater Comedy  x0_theater Experimental  x0_theater Festivals  \\\n",
       "0                     0.0                      0.0                   0.0   \n",
       "1                     0.0                      0.0                   0.0   \n",
       "2                     0.0                      0.0                   0.0   \n",
       "3                     0.0                      0.0                   0.0   \n",
       "4                     0.0                      0.0                   0.0   \n",
       "...                   ...                      ...                   ...   \n",
       "170725                0.0                      0.0                   0.0   \n",
       "170726                0.0                      0.0                   0.0   \n",
       "170727                0.0                      0.0                   0.0   \n",
       "170728                0.0                      0.0                   0.0   \n",
       "170729                0.0                      0.0                   0.0   \n",
       "\n",
       "        x0_theater Immersive  x0_theater Musical  x0_theater Plays  \\\n",
       "0                        0.0                 0.0               0.0   \n",
       "1                        0.0                 0.0               0.0   \n",
       "2                        0.0                 0.0               0.0   \n",
       "3                        0.0                 0.0               0.0   \n",
       "4                        0.0                 0.0               0.0   \n",
       "...                      ...                 ...               ...   \n",
       "170725                   0.0                 0.0               0.0   \n",
       "170726                   0.0                 0.0               0.0   \n",
       "170727                   0.0                 0.0               0.0   \n",
       "170728                   0.0                 0.0               0.0   \n",
       "170729                   0.0                 0.0               0.0   \n",
       "\n",
       "        x0_theater Spaces  x0_theater Theater  \n",
       "0                     0.0                 0.0  \n",
       "1                     0.0                 0.0  \n",
       "2                     0.0                 0.0  \n",
       "3                     0.0                 0.0  \n",
       "4                     0.0                 0.0  \n",
       "...                   ...                 ...  \n",
       "170725                0.0                 0.0  \n",
       "170726                0.0                 0.0  \n",
       "170727                0.0                 0.0  \n",
       "170728                0.0                 0.0  \n",
       "170729                0.0                 0.0  \n",
       "\n",
       "[170730 rows x 192 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['duration', 'blurb_length', 'name_length', 'usd_pledged', 'success',\n",
       "       'US based', 'main_category_comics', 'main_category_crafts',\n",
       "       'main_category_dance', 'main_category_design', 'main_category_fashion',\n",
       "       'main_category_film & video', 'main_category_food',\n",
       "       'main_category_games', 'main_category_journalism',\n",
       "       'main_category_music', 'main_category_photography',\n",
       "       'main_category_publishing', 'main_category_technology',\n",
       "       'main_category_theater', 'start_Q_Q2', 'start_Q_Q3', 'start_Q_Q4'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: Dropping the columns prefixed by \"main_category_\" so that there's only main_sub_category columns left for analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final = df_final.drop(['main_category_comics', 'main_category_crafts',\n",
    "       'main_category_dance', 'main_category_design', 'main_category_fashion',\n",
    "       'main_category_film & video', 'main_category_food',\n",
    "       'main_category_games', 'main_category_journalism',\n",
    "       'main_category_music', 'main_category_photography',\n",
    "       'main_category_publishing', 'main_category_technology',\n",
    "       'main_category_theater'], axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: No changes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Define function for performance result\n",
    "\n",
    "# Function to print KFold Cross validation performance on train set \n",
    "def KFoldresult_5fold(model, x_train, y_train, is_logreg):\n",
    "    accuracy = cross_val_score (model, x_train,y_train, cv=5)\n",
    "    print (model)\n",
    "    print (f'KFolds cross validation: \\n {accuracy} \\n')\n",
    "    print (f'Mean accuracy: \\n {accuracy.mean()}\\n')\n",
    "    print ('Coefficient of feature: \\n' )\n",
    "    if is_logreg:\n",
    "        for index, co in enumerate(model.coef_[0]):\n",
    "            print (f'Feature {index}: {co:.5f}')\n",
    "    else:\n",
    "        for index, co in enumerate(model.feature_importances_):\n",
    "            print (f'Feature {index}: {co:.5f}')\n",
    "    return accuracy\n",
    "\n",
    "# Function to return prediction and print prediction result on test set \n",
    "def predictionresult(model, x_test, y_test):\n",
    "    y_pred = model.predict(x_test)\n",
    "    print (f'Confusion_matrix: \\n {confusion_matrix(y_test, y_pred)} \\n')\n",
    "    print (f'Classification report: \\n {classification_report(y_test,y_pred)} \\n')\n",
    "    return y_pred\n",
    "\n",
    "# Function to print out Grid Search parameters: \n",
    "def gridsearch(model, parameters, X_train, y_train):\n",
    "    search = GridSearchCV(model, parameters, n_jobs=-1)\n",
    "    search.fit(x_train,y_train)\n",
    "    print(f'Parameter tested: {parameters}')\n",
    "    print(f'Best Score : {search.best_score_}')\n",
    "    print(f'Best parameters: {search.best_params_}')\n",
    "    return search\n",
    "\n",
    "def performace(y_ture, y_pred):\n",
    "    return [accuracy_score(y_ture, y_pred), recall_score(y_ture, y_pred), precision_score(y_ture, y_pred), f1_score(y_ture, y_pred)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: splitting the \"new\" dataframe into train and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split train set and test set \n",
    "y_sub = df_final['success']\n",
    "x_sub = df_final.drop('success',axis=1)\n",
    "x_train_sub, x_test_sub, y_train_sub, y_test_sub = train_test_split(x_sub,y_sub,train_size = 0.8, random_state = 42) #shuffle = False that means no random "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split train set and test set \n",
    "y = df['success']\n",
    "x = df.drop('success',axis=1)\n",
    "x_train, x_test, y_train, y_test = train_test_split(x,y,train_size = 0.8, random_state = 42) #shuffle = False that means no random "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression (log)\n",
    "#### -Develop model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: Log Regression with main_sub_category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\theba\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:762: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "C:\\Users\\theba\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:762: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "C:\\Users\\theba\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:762: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "C:\\Users\\theba\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:762: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "C:\\Users\\theba\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:762: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression()\n",
      "KFolds cross validation: \n",
      " [0.75535381 0.76146722 0.7553172  0.75820917 0.75973788] \n",
      "\n",
      "Mean accuracy: \n",
      " 0.7580170570551858\n",
      "\n",
      "Coefficient of feature: \n",
      "\n",
      "Feature 0: -2.15312\n",
      "Feature 1: -0.39134\n",
      "Feature 2: 2.48987\n",
      "Feature 3: 27.63620\n",
      "Feature 4: 0.17335\n",
      "Feature 5: -0.03469\n",
      "Feature 6: -0.13815\n",
      "Feature 7: -0.07063\n",
      "Feature 8: 4.87572\n",
      "Feature 9: -0.55883\n",
      "Feature 10: -0.92711\n",
      "Feature 11: -1.05971\n",
      "Feature 12: 2.53458\n",
      "Feature 13: -0.44769\n",
      "Feature 14: -0.71395\n",
      "Feature 15: -0.49055\n",
      "Feature 16: -0.61347\n",
      "Feature 17: 0.21606\n",
      "Feature 18: -0.83355\n",
      "Feature 19: -1.03108\n",
      "Feature 20: -1.29527\n",
      "Feature 21: 0.70158\n",
      "Feature 22: 4.95603\n",
      "Feature 23: 5.01829\n",
      "Feature 24: -0.85168\n",
      "Feature 25: 0.02257\n",
      "Feature 26: -0.08616\n",
      "Feature 27: -2.16046\n",
      "Feature 28: 4.84460\n",
      "Feature 29: -1.46840\n",
      "Feature 30: -1.77637\n",
      "Feature 31: -1.54220\n",
      "Feature 32: -1.09518\n",
      "Feature 33: -0.43270\n",
      "Feature 34: -0.24174\n",
      "Feature 35: -1.67983\n",
      "Feature 36: -2.47096\n",
      "Feature 37: -0.97163\n",
      "Feature 38: -0.61173\n",
      "Feature 39: -1.03584\n",
      "Feature 40: -1.43158\n",
      "Feature 41: 4.84350\n",
      "Feature 42: 0.30483\n",
      "Feature 43: 0.73704\n",
      "Feature 44: -1.08443\n",
      "Feature 45: -0.73523\n",
      "Feature 46: -1.42016\n",
      "Feature 47: -0.86007\n",
      "Feature 48: 4.92907\n",
      "Feature 49: -0.80073\n",
      "Feature 50: -1.66169\n",
      "Feature 51: 4.79445\n",
      "Feature 52: 0.14348\n",
      "Feature 53: 0.67831\n",
      "Feature 54: 0.60333\n",
      "Feature 55: -1.74387\n",
      "Feature 56: -2.17570\n",
      "Feature 57: 4.98745\n",
      "Feature 58: -1.12313\n",
      "Feature 59: -1.31870\n",
      "Feature 60: -1.05935\n",
      "Feature 61: -1.86799\n",
      "Feature 62: -1.83224\n",
      "Feature 63: -1.26696\n",
      "Feature 64: -0.69107\n",
      "Feature 65: 4.95213\n",
      "Feature 66: -0.61414\n",
      "Feature 67: -1.27225\n",
      "Feature 68: -1.60533\n",
      "Feature 69: -0.76591\n",
      "Feature 70: -0.46119\n",
      "Feature 71: 5.14479\n",
      "Feature 72: -0.89345\n",
      "Feature 73: -1.02188\n",
      "Feature 74: -0.96800\n",
      "Feature 75: 1.59398\n",
      "Feature 76: -1.04046\n",
      "Feature 77: -0.79696\n",
      "Feature 78: 5.00871\n",
      "Feature 79: -1.93783\n",
      "Feature 80: -1.03680\n",
      "Feature 81: 0.55205\n",
      "Feature 82: -1.56669\n",
      "Feature 83: -1.57932\n",
      "Feature 84: -1.29465\n",
      "Feature 85: -1.18231\n",
      "Feature 86: -1.91760\n",
      "Feature 87: -1.93981\n",
      "Feature 88: -1.67344\n",
      "Feature 89: 4.92687\n",
      "Feature 90: -2.26850\n",
      "Feature 91: -1.68819\n",
      "Feature 92: -1.13970\n",
      "Feature 93: -1.14707\n",
      "Feature 94: -0.94991\n",
      "Feature 95: 3.95920\n",
      "Feature 96: -1.14025\n",
      "Feature 97: -1.82372\n",
      "Feature 98: -2.38159\n",
      "Feature 99: -0.23637\n",
      "Feature 100: -0.80596\n",
      "Feature 101: 4.81568\n",
      "Feature 102: 4.90182\n",
      "Feature 103: -1.13900\n",
      "Feature 104: 0.31415\n",
      "Feature 105: -1.75290\n",
      "Feature 106: -1.56373\n",
      "Feature 107: -2.17370\n",
      "Feature 108: -1.99983\n",
      "Feature 109: -0.52369\n",
      "Feature 110: 0.65395\n",
      "Feature 111: 0.47688\n",
      "Feature 112: -1.38445\n",
      "Feature 113: 4.90906\n",
      "Feature 114: -1.02520\n",
      "Feature 115: -0.64494\n",
      "Feature 116: -1.45756\n",
      "Feature 117: 4.91513\n",
      "Feature 118: 0.05791\n",
      "Feature 119: -0.45777\n",
      "Feature 120: -1.33598\n",
      "Feature 121: -0.48619\n",
      "Feature 122: 4.88347\n",
      "Feature 123: 0.32723\n",
      "Feature 124: -0.39753\n",
      "Feature 125: -1.40597\n",
      "Feature 126: 4.91678\n",
      "Feature 127: -0.59867\n",
      "Feature 128: -1.56810\n",
      "Feature 129: -0.59299\n",
      "Feature 130: -1.85912\n",
      "Feature 131: -1.26110\n",
      "Feature 132: -0.65193\n",
      "Feature 133: 4.90197\n",
      "Feature 134: -1.95206\n",
      "Feature 135: -1.64808\n",
      "Feature 136: 0.09838\n",
      "Feature 137: 0.06905\n",
      "Feature 138: -0.69798\n",
      "Feature 139: 4.90502\n",
      "Feature 140: -0.86836\n",
      "Feature 141: 4.92918\n",
      "Feature 142: -0.29819\n",
      "Feature 143: -0.43008\n",
      "Feature 144: 0.30975\n",
      "Feature 145: 4.90154\n",
      "Feature 146: -0.73995\n",
      "Feature 147: -0.98989\n",
      "Feature 148: 5.04247\n",
      "Feature 149: -0.62729\n",
      "Feature 150: -1.55767\n",
      "Feature 151: -1.60905\n",
      "Feature 152: -0.29121\n",
      "Feature 153: -0.99030\n",
      "Feature 154: -1.87324\n",
      "Feature 155: -0.68067\n",
      "Feature 156: -0.58369\n",
      "Feature 157: -1.21173\n",
      "Feature 158: -1.82670\n",
      "Feature 159: -0.79416\n",
      "Feature 160: -0.53210\n",
      "Feature 161: -1.15465\n",
      "Feature 162: -0.82029\n",
      "Feature 163: -2.02395\n",
      "Feature 164: -0.54115\n",
      "Feature 165: -0.82794\n",
      "Feature 166: 4.96323\n",
      "Feature 167: -1.05205\n",
      "Feature 168: -2.38371\n",
      "Feature 169: -0.08927\n",
      "Feature 170: 0.00249\n",
      "Feature 171: -0.05557\n",
      "Feature 172: -0.15839\n",
      "Feature 173: -0.34480\n",
      "Feature 174: -0.07260\n",
      "Feature 175: -0.59557\n",
      "Feature 176: 4.95235\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\theba\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:762: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.75535381, 0.76146722, 0.7553172 , 0.75820917, 0.75973788])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create model -log\n",
    "log = LogisticRegression()\n",
    "log.fit (x_train_sub,y_train_sub)\n",
    "\n",
    "KFoldresult_5fold(log, x_train_sub, y_train_sub, True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: Log Regression with main_categories only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression()\n",
      "KFolds cross validation: \n",
      " [0.66830179 0.67141341 0.67459824 0.6784786  0.6715112 ] \n",
      "\n",
      "Mean accuracy: \n",
      " 0.6728606472980359\n",
      "\n",
      "Coefficient of feature: \n",
      "\n",
      "Feature 0: -2.39444\n",
      "Feature 1: -0.67378\n",
      "Feature 2: 2.93481\n",
      "Feature 3: 43.04130\n",
      "Feature 4: 0.10109\n",
      "Feature 5: 1.12956\n",
      "Feature 6: -0.56457\n",
      "Feature 7: 1.15761\n",
      "Feature 8: 0.11070\n",
      "Feature 9: -0.14227\n",
      "Feature 10: 0.17703\n",
      "Feature 11: -1.10396\n",
      "Feature 12: 0.07818\n",
      "Feature 13: -1.04814\n",
      "Feature 14: 0.44949\n",
      "Feature 15: -0.53562\n",
      "Feature 16: 0.54856\n",
      "Feature 17: -0.98039\n",
      "Feature 18: 0.30545\n",
      "Feature 19: -0.02547\n",
      "Feature 20: -0.12572\n",
      "Feature 21: -0.00833\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.66830179, 0.67141341, 0.67459824, 0.6784786 , 0.6715112 ])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create model -log\n",
    "log = LogisticRegression()\n",
    "log.fit (x_train,y_train)\n",
    "\n",
    "KFoldresult_5fold(log, x_train, y_train, True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: Results with sub_category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion_matrix: \n",
      " [[11671  3231]\n",
      " [ 4934 14310]] \n",
      "\n",
      "Classification report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.70      0.78      0.74     14902\n",
      "         1.0       0.82      0.74      0.78     19244\n",
      "\n",
      "    accuracy                           0.76     34146\n",
      "   macro avg       0.76      0.76      0.76     34146\n",
      "weighted avg       0.77      0.76      0.76     34146\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "log_y_predict_sub = predictionresult(log, x_test_sub, y_test_sub)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: Results with main_category only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion_matrix: \n",
      " [[ 7652  7250]\n",
      " [ 3792 15452]] \n",
      "\n",
      "Classification report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.67      0.51      0.58     14902\n",
      "         1.0       0.68      0.80      0.74     19244\n",
      "\n",
      "    accuracy                           0.68     34146\n",
      "   macro avg       0.67      0.66      0.66     34146\n",
      "weighted avg       0.68      0.68      0.67     34146\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "log_y_predict = predictionresult(log, x_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### -Optimizing hyperparameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: optimizing the regressor trained by the sub_category dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter tested: {'C': array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1. ]), 'penalty': ['l1', 'l2']}\n",
      "Best Score : 0.6728606472980359\n",
      "Best parameters: {'C': 1.0, 'penalty': 'l2'}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(estimator=LogisticRegression(), n_jobs=-1,\n",
       "             param_grid={'C': array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1. ]),\n",
       "                         'penalty': ['l1', 'l2']})"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param = {'C':np.linspace(0.1,1,10), 'penalty': ['l1', 'l2']} \n",
    "gridsearch (log, param, x_train_sub, y_train_sub)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: no changes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter tested: {'C': array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1. ]), 'penalty': ['l1', 'l2']}\n",
      "Best Score : 0.6728606472980359\n",
      "Best parameters: {'C': 1.0, 'penalty': 'l2'}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(estimator=LogisticRegression(), n_jobs=-1,\n",
       "             param_grid={'C': array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1. ]),\n",
       "                         'penalty': ['l1', 'l2']})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param = {'C':np.linspace(0.1,1,10), 'penalty': ['l1', 'l2']} \n",
    "gridsearch (log, param, x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Apply the best parameters {'C': 1.0, 'penalty': 'l2'} \n",
    " - best parameters is same as default parameters for model 'log'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest Classifier (ranforest)\n",
    "#### -Develop model "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: Random Forest with the sub_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier(n_jobs=-1, random_state=42)\n",
      "KFolds cross validation: \n",
      " [0.90079438 0.90225867 0.90233188 0.89852473 0.90133987] \n",
      "\n",
      "Mean accuracy: \n",
      " 0.9010499054791821\n",
      "\n",
      "Coefficient of feature: \n",
      "\n",
      "Feature 0: 0.05250\n",
      "Feature 1: 0.05095\n",
      "Feature 2: 0.04344\n",
      "Feature 3: 0.57038\n",
      "Feature 4: 0.00881\n",
      "Feature 5: 0.00778\n",
      "Feature 6: 0.00764\n",
      "Feature 7: 0.00751\n",
      "Feature 8: 0.00304\n",
      "Feature 9: 0.00043\n",
      "Feature 10: 0.00107\n",
      "Feature 11: 0.00126\n",
      "Feature 12: 0.00844\n",
      "Feature 13: 0.00065\n",
      "Feature 14: 0.00178\n",
      "Feature 15: 0.00171\n",
      "Feature 16: 0.00152\n",
      "Feature 17: 0.00159\n",
      "Feature 18: 0.00148\n",
      "Feature 19: 0.00044\n",
      "Feature 20: 0.00022\n",
      "Feature 21: 0.00059\n",
      "Feature 22: 0.00745\n",
      "Feature 23: 0.00094\n",
      "Feature 24: 0.00026\n",
      "Feature 25: 0.00152\n",
      "Feature 26: 0.00083\n",
      "Feature 27: 0.00036\n",
      "Feature 28: 0.00518\n",
      "Feature 29: 0.00026\n",
      "Feature 30: 0.00123\n",
      "Feature 31: 0.00019\n",
      "Feature 32: 0.00017\n",
      "Feature 33: 0.00024\n",
      "Feature 34: 0.00012\n",
      "Feature 35: 0.00028\n",
      "Feature 36: 0.00012\n",
      "Feature 37: 0.00036\n",
      "Feature 38: 0.00001\n",
      "Feature 39: 0.00012\n",
      "Feature 40: 0.00104\n",
      "Feature 41: 0.00318\n",
      "Feature 42: 0.00096\n",
      "Feature 43: 0.00006\n",
      "Feature 44: 0.00027\n",
      "Feature 45: 0.00023\n",
      "Feature 46: 0.00099\n",
      "Feature 47: 0.00062\n",
      "Feature 48: 0.00022\n",
      "Feature 49: 0.00167\n",
      "Feature 50: 0.00068\n",
      "Feature 51: 0.00520\n",
      "Feature 52: 0.00020\n",
      "Feature 53: 0.00243\n",
      "Feature 54: 0.00173\n",
      "Feature 55: 0.00079\n",
      "Feature 56: 0.00037\n",
      "Feature 57: 0.00036\n",
      "Feature 58: 0.00157\n",
      "Feature 59: 0.00131\n",
      "Feature 60: 0.00022\n",
      "Feature 61: 0.00114\n",
      "Feature 62: 0.00088\n",
      "Feature 63: 0.00236\n",
      "Feature 64: 0.00174\n",
      "Feature 65: 0.00658\n",
      "Feature 66: 0.00171\n",
      "Feature 67: 0.00059\n",
      "Feature 68: 0.00055\n",
      "Feature 69: 0.00054\n",
      "Feature 70: 0.00036\n",
      "Feature 71: 0.00050\n",
      "Feature 72: 0.00127\n",
      "Feature 73: 0.00052\n",
      "Feature 74: 0.00067\n",
      "Feature 75: 0.00320\n",
      "Feature 76: 0.00036\n",
      "Feature 77: 0.00104\n",
      "Feature 78: 0.00754\n",
      "Feature 79: 0.00145\n",
      "Feature 80: 0.00095\n",
      "Feature 81: 0.00168\n",
      "Feature 82: 0.00029\n",
      "Feature 83: 0.00040\n",
      "Feature 84: 0.00081\n",
      "Feature 85: 0.00265\n",
      "Feature 86: 0.00075\n",
      "Feature 87: 0.00053\n",
      "Feature 88: 0.00130\n",
      "Feature 89: 0.00094\n",
      "Feature 90: 0.00344\n",
      "Feature 91: 0.00286\n",
      "Feature 92: 0.00157\n",
      "Feature 93: 0.00078\n",
      "Feature 94: 0.00103\n",
      "Feature 95: 0.00012\n",
      "Feature 96: 0.00069\n",
      "Feature 97: 0.00134\n",
      "Feature 98: 0.00604\n",
      "Feature 99: 0.00198\n",
      "Feature 100: 0.00042\n",
      "Feature 101: 0.00459\n",
      "Feature 102: 0.00758\n",
      "Feature 103: 0.00047\n",
      "Feature 104: 0.00071\n",
      "Feature 105: 0.00017\n",
      "Feature 106: 0.00070\n",
      "Feature 107: 0.00053\n",
      "Feature 108: 0.00149\n",
      "Feature 109: 0.00037\n",
      "Feature 110: 0.00012\n",
      "Feature 111: 0.00168\n",
      "Feature 112: 0.00005\n",
      "Feature 113: 0.00629\n",
      "Feature 114: 0.00133\n",
      "Feature 115: 0.00091\n",
      "Feature 116: 0.00147\n",
      "Feature 117: 0.00672\n",
      "Feature 118: 0.00126\n",
      "Feature 119: 0.00032\n",
      "Feature 120: 0.00016\n",
      "Feature 121: 0.00077\n",
      "Feature 122: 0.00148\n",
      "Feature 123: 0.00140\n",
      "Feature 124: 0.00051\n",
      "Feature 125: 0.00033\n",
      "Feature 126: 0.00695\n",
      "Feature 127: 0.00134\n",
      "Feature 128: 0.00033\n",
      "Feature 129: 0.00087\n",
      "Feature 130: 0.00044\n",
      "Feature 131: 0.00097\n",
      "Feature 132: 0.00164\n",
      "Feature 133: 0.00184\n",
      "Feature 134: 0.00059\n",
      "Feature 135: 0.00096\n",
      "Feature 136: 0.00053\n",
      "Feature 137: 0.00166\n",
      "Feature 138: 0.00051\n",
      "Feature 139: 0.00553\n",
      "Feature 140: 0.00017\n",
      "Feature 141: 0.00873\n",
      "Feature 142: 0.00005\n",
      "Feature 143: 0.00042\n",
      "Feature 144: 0.00010\n",
      "Feature 145: 0.00652\n",
      "Feature 146: 0.00126\n",
      "Feature 147: 0.00119\n",
      "Feature 148: 0.00049\n",
      "Feature 149: 0.00088\n",
      "Feature 150: 0.00022\n",
      "Feature 151: 0.00068\n",
      "Feature 152: 0.00069\n",
      "Feature 153: 0.00126\n",
      "Feature 154: 0.00311\n",
      "Feature 155: 0.00095\n",
      "Feature 156: 0.00117\n",
      "Feature 157: 0.00042\n",
      "Feature 158: 0.00091\n",
      "Feature 159: 0.00302\n",
      "Feature 160: 0.00228\n",
      "Feature 161: 0.00064\n",
      "Feature 162: 0.00081\n",
      "Feature 163: 0.00367\n",
      "Feature 164: 0.00123\n",
      "Feature 165: 0.00055\n",
      "Feature 166: 0.00047\n",
      "Feature 167: 0.00218\n",
      "Feature 168: 0.00458\n",
      "Feature 169: 0.00019\n",
      "Feature 170: 0.00045\n",
      "Feature 171: 0.00054\n",
      "Feature 172: 0.00030\n",
      "Feature 173: 0.00100\n",
      "Feature 174: 0.00113\n",
      "Feature 175: 0.00033\n",
      "Feature 176: 0.00123\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.90079438, 0.90225867, 0.90233188, 0.89852473, 0.90133987])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ranforest = RandomForestClassifier(random_state = 42, n_jobs=-1)\n",
    "ranforest.fit (x_train_sub,y_train_sub)\n",
    "\n",
    "KFoldresult_5fold(ranforest, x_train_sub, y_train_sub, False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: Random Forest with the main_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier(n_jobs=-1, random_state=42)\n",
      "KFolds cross validation: \n",
      " [0.8791961  0.87886664 0.87974521 0.87674342 0.87897203] \n",
      "\n",
      "Mean accuracy: \n",
      " 0.8787046818613394\n",
      "\n",
      "Coefficient of feature: \n",
      "\n",
      "Feature 0: 0.07421\n",
      "Feature 1: 0.07898\n",
      "Feature 2: 0.05877\n",
      "Feature 3: 0.68954\n",
      "Feature 4: 0.00834\n",
      "Feature 5: 0.00399\n",
      "Feature 6: 0.00226\n",
      "Feature 7: 0.00191\n",
      "Feature 8: 0.00225\n",
      "Feature 9: 0.00281\n",
      "Feature 10: 0.00382\n",
      "Feature 11: 0.01615\n",
      "Feature 12: 0.00319\n",
      "Feature 13: 0.00291\n",
      "Feature 14: 0.00461\n",
      "Feature 15: 0.00245\n",
      "Feature 16: 0.00452\n",
      "Feature 17: 0.01819\n",
      "Feature 18: 0.00193\n",
      "Feature 19: 0.00654\n",
      "Feature 20: 0.00630\n",
      "Feature 21: 0.00634\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.8791961 , 0.87886664, 0.87974521, 0.87674342, 0.87897203])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ranforest = RandomForestClassifier(random_state = 42, n_jobs=-1)\n",
    "ranforest.fit (x_train,y_train)\n",
    "\n",
    "KFoldresult_5fold(ranforest, x_train, y_train, False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: Random Forest result with sub_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion_matrix: \n",
      " [[12896  2006]\n",
      " [ 1355 17889]] \n",
      "\n",
      "Classification report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.90      0.87      0.88     14902\n",
      "         1.0       0.90      0.93      0.91     19244\n",
      "\n",
      "    accuracy                           0.90     34146\n",
      "   macro avg       0.90      0.90      0.90     34146\n",
      "weighted avg       0.90      0.90      0.90     34146\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "ranforest_y_predict = predictionresult(ranforest, x_test_sub, y_test_sub)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: Random Forest result with main_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion_matrix: \n",
      " [[12331  2571]\n",
      " [ 1600 17644]] \n",
      "\n",
      "Classification report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.89      0.83      0.86     14902\n",
      "         1.0       0.87      0.92      0.89     19244\n",
      "\n",
      "    accuracy                           0.88     34146\n",
      "   macro avg       0.88      0.87      0.87     34146\n",
      "weighted avg       0.88      0.88      0.88     34146\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "ranforest_y_predict = predictionresult(ranforest, x_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### -Optimizing Hyperparameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: Random Forest optimizing with sub_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter tested: {'max_depth': [1, 2, 3, 4]}\n",
      "Best Score : 0.8648231155969773\n",
      "Best parameters: {'max_depth': 4}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(estimator=RandomForestClassifier(n_jobs=-1, random_state=42),\n",
       "             n_jobs=-1, param_grid={'max_depth': [1, 2, 3, 4]})"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param = {'max_depth': list(range(1,5))}\n",
    "gridsearch (ranforest, param, x_train_sub, y_train_sub)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: Random Forest optimizing with main_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter tested: {'max_depth': [1, 2, 3, 4]}\n",
      "Best Score : 0.8648231155969773\n",
      "Best parameters: {'max_depth': 4}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(estimator=RandomForestClassifier(n_jobs=-1, random_state=42),\n",
       "             n_jobs=-1, param_grid={'max_depth': [1, 2, 3, 4]})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param = {'max_depth': list(range(1,5))}\n",
    "gridsearch (ranforest, param, x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Apply the best parameters {'max_depth': 4}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: Applying to sub_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier(max_depth=4, n_jobs=-1, random_state=42)\n",
      "KFolds cross validation: \n",
      " [0.8680675  0.87019072 0.86740857 0.86455321 0.86971006] \n",
      "\n",
      "Mean accuracy: \n",
      " 0.8679860139111911\n",
      "\n",
      "Coefficient of feature: \n",
      "\n",
      "Feature 0: 0.07940\n",
      "Feature 1: 0.00140\n",
      "Feature 2: 0.05670\n",
      "Feature 3: 0.31137\n",
      "Feature 4: 0.00017\n",
      "Feature 5: 0.00006\n",
      "Feature 6: 0.00013\n",
      "Feature 7: 0.00007\n",
      "Feature 8: 0.00272\n",
      "Feature 9: 0.00000\n",
      "Feature 10: 0.00000\n",
      "Feature 11: 0.00003\n",
      "Feature 12: 0.01810\n",
      "Feature 13: 0.00000\n",
      "Feature 14: 0.00000\n",
      "Feature 15: 0.00000\n",
      "Feature 16: 0.00000\n",
      "Feature 17: 0.00024\n",
      "Feature 18: 0.00000\n",
      "Feature 19: 0.00000\n",
      "Feature 20: 0.00000\n",
      "Feature 21: 0.00000\n",
      "Feature 22: 0.03006\n",
      "Feature 23: 0.00020\n",
      "Feature 24: 0.00000\n",
      "Feature 25: 0.00000\n",
      "Feature 26: 0.00000\n",
      "Feature 27: 0.00000\n",
      "Feature 28: 0.00461\n",
      "Feature 29: 0.00000\n",
      "Feature 30: 0.00117\n",
      "Feature 31: 0.00000\n",
      "Feature 32: 0.00000\n",
      "Feature 33: 0.00000\n",
      "Feature 34: 0.00000\n",
      "Feature 35: 0.00000\n",
      "Feature 36: 0.00000\n",
      "Feature 37: 0.00000\n",
      "Feature 38: 0.00000\n",
      "Feature 39: 0.00000\n",
      "Feature 40: 0.00228\n",
      "Feature 41: 0.00663\n",
      "Feature 42: 0.00000\n",
      "Feature 43: 0.00000\n",
      "Feature 44: 0.00000\n",
      "Feature 45: 0.00000\n",
      "Feature 46: 0.00001\n",
      "Feature 47: 0.00000\n",
      "Feature 48: 0.00000\n",
      "Feature 49: 0.00000\n",
      "Feature 50: 0.00001\n",
      "Feature 51: 0.03327\n",
      "Feature 52: 0.00000\n",
      "Feature 53: 0.00098\n",
      "Feature 54: 0.00001\n",
      "Feature 55: 0.00000\n",
      "Feature 56: 0.00000\n",
      "Feature 57: 0.00001\n",
      "Feature 58: 0.00000\n",
      "Feature 59: 0.00000\n",
      "Feature 60: 0.00000\n",
      "Feature 61: 0.00141\n",
      "Feature 62: 0.00000\n",
      "Feature 63: 0.00184\n",
      "Feature 64: 0.00000\n",
      "Feature 65: 0.03555\n",
      "Feature 66: 0.00000\n",
      "Feature 67: 0.00000\n",
      "Feature 68: 0.00000\n",
      "Feature 69: 0.00000\n",
      "Feature 70: 0.00000\n",
      "Feature 71: 0.00001\n",
      "Feature 72: 0.00000\n",
      "Feature 73: 0.00000\n",
      "Feature 74: 0.00000\n",
      "Feature 75: 0.00914\n",
      "Feature 76: 0.00000\n",
      "Feature 77: 0.00000\n",
      "Feature 78: 0.03341\n",
      "Feature 79: 0.00179\n",
      "Feature 80: 0.00000\n",
      "Feature 81: 0.00041\n",
      "Feature 82: 0.00000\n",
      "Feature 83: 0.00000\n",
      "Feature 84: 0.00000\n",
      "Feature 85: 0.00216\n",
      "Feature 86: 0.00027\n",
      "Feature 87: 0.00000\n",
      "Feature 88: 0.00000\n",
      "Feature 89: 0.00002\n",
      "Feature 90: 0.01907\n",
      "Feature 91: 0.00675\n",
      "Feature 92: 0.00054\n",
      "Feature 93: 0.00000\n",
      "Feature 94: 0.00000\n",
      "Feature 95: 0.00000\n",
      "Feature 96: 0.00000\n",
      "Feature 97: 0.00087\n",
      "Feature 98: 0.02741\n",
      "Feature 99: 0.00000\n",
      "Feature 100: 0.00000\n",
      "Feature 101: 0.01935\n",
      "Feature 102: 0.01351\n",
      "Feature 103: 0.00000\n",
      "Feature 104: 0.00000\n",
      "Feature 105: 0.00000\n",
      "Feature 106: 0.00000\n",
      "Feature 107: 0.00090\n",
      "Feature 108: 0.00182\n",
      "Feature 109: 0.00000\n",
      "Feature 110: 0.00000\n",
      "Feature 111: 0.00103\n",
      "Feature 112: 0.00000\n",
      "Feature 113: 0.05240\n",
      "Feature 114: 0.00004\n",
      "Feature 115: 0.00000\n",
      "Feature 116: 0.00343\n",
      "Feature 117: 0.02854\n",
      "Feature 118: 0.00000\n",
      "Feature 119: 0.00000\n",
      "Feature 120: 0.00000\n",
      "Feature 121: 0.00000\n",
      "Feature 122: 0.00103\n",
      "Feature 123: 0.00000\n",
      "Feature 124: 0.00000\n",
      "Feature 125: 0.00000\n",
      "Feature 126: 0.04740\n",
      "Feature 127: 0.00000\n",
      "Feature 128: 0.00000\n",
      "Feature 129: 0.00000\n",
      "Feature 130: 0.00000\n",
      "Feature 131: 0.00000\n",
      "Feature 132: 0.00000\n",
      "Feature 133: 0.00157\n",
      "Feature 134: 0.00004\n",
      "Feature 135: 0.00069\n",
      "Feature 136: 0.00000\n",
      "Feature 137: 0.00000\n",
      "Feature 138: 0.00000\n",
      "Feature 139: 0.03119\n",
      "Feature 140: 0.00000\n",
      "Feature 141: 0.03741\n",
      "Feature 142: 0.00000\n",
      "Feature 143: 0.00000\n",
      "Feature 144: 0.00000\n",
      "Feature 145: 0.02376\n",
      "Feature 146: 0.00000\n",
      "Feature 147: 0.00000\n",
      "Feature 148: 0.00000\n",
      "Feature 149: 0.00000\n",
      "Feature 150: 0.00000\n",
      "Feature 151: 0.00000\n",
      "Feature 152: 0.00000\n",
      "Feature 153: 0.00000\n",
      "Feature 154: 0.01341\n",
      "Feature 155: 0.00000\n",
      "Feature 156: 0.00000\n",
      "Feature 157: 0.00000\n",
      "Feature 158: 0.00000\n",
      "Feature 159: 0.00006\n",
      "Feature 160: 0.00000\n",
      "Feature 161: 0.00000\n",
      "Feature 162: 0.00000\n",
      "Feature 163: 0.01403\n",
      "Feature 164: 0.00000\n",
      "Feature 165: 0.00000\n",
      "Feature 166: 0.00073\n",
      "Feature 167: 0.00005\n",
      "Feature 168: 0.01723\n",
      "Feature 169: 0.00000\n",
      "Feature 170: 0.00000\n",
      "Feature 171: 0.00000\n",
      "Feature 172: 0.00000\n",
      "Feature 173: 0.00000\n",
      "Feature 174: 0.00000\n",
      "Feature 175: 0.00000\n",
      "Feature 176: 0.00007\n",
      "Confusion_matrix: \n",
      " [[11235  3667]\n",
      " [  922 18322]] \n",
      "\n",
      "Classification report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.92      0.75      0.83     14902\n",
      "         1.0       0.83      0.95      0.89     19244\n",
      "\n",
      "    accuracy                           0.87     34146\n",
      "   macro avg       0.88      0.85      0.86     34146\n",
      "weighted avg       0.87      0.87      0.86     34146\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "ranforest = RandomForestClassifier(max_depth=4, random_state = 42, n_jobs=-1)\n",
    "ranforest.fit (x_train_sub,y_train_sub)\n",
    "\n",
    "KFoldresult_5fold(ranforest, x_train_sub, y_train_sub, False)\n",
    "ranforest_y_predict = predictionresult(ranforest, x_test_sub, y_test_sub)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: Applying to main_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier(max_depth=4, n_jobs=-1, random_state=42)\n",
      "KFolds cross validation: \n",
      " [0.86660321 0.86685946 0.86462642 0.86078266 0.86524381] \n",
      "\n",
      "Mean accuracy: \n",
      " 0.8648231155969773\n",
      "\n",
      "Coefficient of feature: \n",
      "\n",
      "Feature 0: 0.06871\n",
      "Feature 1: 0.00287\n",
      "Feature 2: 0.04939\n",
      "Feature 3: 0.68332\n",
      "Feature 4: 0.00169\n",
      "Feature 5: 0.01605\n",
      "Feature 6: 0.00208\n",
      "Feature 7: 0.00658\n",
      "Feature 8: 0.00008\n",
      "Feature 9: 0.00001\n",
      "Feature 10: 0.00065\n",
      "Feature 11: 0.07265\n",
      "Feature 12: 0.00034\n",
      "Feature 13: 0.00667\n",
      "Feature 14: 0.01246\n",
      "Feature 15: 0.00147\n",
      "Feature 16: 0.01278\n",
      "Feature 17: 0.06159\n",
      "Feature 18: 0.00023\n",
      "Feature 19: 0.00007\n",
      "Feature 20: 0.00024\n",
      "Feature 21: 0.00009\n",
      "Confusion_matrix: \n",
      " [[11446  3456]\n",
      " [ 1148 18096]] \n",
      "\n",
      "Classification report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.91      0.77      0.83     14902\n",
      "         1.0       0.84      0.94      0.89     19244\n",
      "\n",
      "    accuracy                           0.87     34146\n",
      "   macro avg       0.87      0.85      0.86     34146\n",
      "weighted avg       0.87      0.87      0.86     34146\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "ranforest = RandomForestClassifier(max_depth=4, random_state = 42, n_jobs=-1)\n",
    "ranforest.fit (x_train,y_train)\n",
    "\n",
    "KFoldresult_5fold(ranforest, x_train, y_train, False)\n",
    "ranforest_y_predict = predictionresult(ranforest, x_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XG Boost\n",
    "#### -Develop model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XG Boost with sub_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\theba\\anaconda3\\lib\\site-packages\\xgboost\\data.py:112: UserWarning: Use subset (sliced data) of np.ndarray is not recommended because it will generate extra copies and increase memory consumption\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "              colsample_bynode=1, colsample_bytree=1, eval_metric='mlogloss',\n",
      "              gamma=0, gpu_id=-1, importance_type='gain',\n",
      "              interaction_constraints='', learning_rate=0.300000012,\n",
      "              max_delta_step=0, max_depth=6, min_child_weight=1, missing=nan,\n",
      "              monotone_constraints='()', n_estimators=100, n_jobs=-1,\n",
      "              num_parallel_tree=1, random_state=0, reg_alpha=0, reg_lambda=1,\n",
      "              scale_pos_weight=1, subsample=1, tree_method='exact',\n",
      "              use_label_encoder=False, validate_parameters=1, verbosity=None)\n",
      "KFolds cross validation: \n",
      " [0.90419885 0.90507742 0.90357653 0.90112384 0.90415873] \n",
      "\n",
      "Mean accuracy: \n",
      " 0.9036270758775473\n",
      "\n",
      "Coefficient of feature: \n",
      "\n",
      "Feature 0: 0.00569\n",
      "Feature 1: 0.00062\n",
      "Feature 2: 0.00056\n",
      "Feature 3: 0.04737\n",
      "Feature 4: 0.00120\n",
      "Feature 5: 0.00092\n",
      "Feature 6: 0.00074\n",
      "Feature 7: 0.00077\n",
      "Feature 8: 0.01387\n",
      "Feature 9: 0.00000\n",
      "Feature 10: 0.00126\n",
      "Feature 11: 0.00442\n",
      "Feature 12: 0.01529\n",
      "Feature 13: 0.00000\n",
      "Feature 14: 0.00345\n",
      "Feature 15: 0.00976\n",
      "Feature 16: 0.00354\n",
      "Feature 17: 0.00956\n",
      "Feature 18: 0.00070\n",
      "Feature 19: 0.00000\n",
      "Feature 20: 0.00000\n",
      "Feature 21: 0.00000\n",
      "Feature 22: 0.02261\n",
      "Feature 23: 0.00890\n",
      "Feature 24: 0.00000\n",
      "Feature 25: 0.00000\n",
      "Feature 26: 0.00250\n",
      "Feature 27: 0.00000\n",
      "Feature 28: 0.01476\n",
      "Feature 29: 0.00241\n",
      "Feature 30: 0.00386\n",
      "Feature 31: 0.00000\n",
      "Feature 32: 0.00000\n",
      "Feature 33: 0.00000\n",
      "Feature 34: 0.00000\n",
      "Feature 35: 0.00000\n",
      "Feature 36: 0.00000\n",
      "Feature 37: 0.00224\n",
      "Feature 38: 0.00000\n",
      "Feature 39: 0.00000\n",
      "Feature 40: 0.00058\n",
      "Feature 41: 0.01601\n",
      "Feature 42: 0.00635\n",
      "Feature 43: 0.00000\n",
      "Feature 44: 0.00000\n",
      "Feature 45: 0.00000\n",
      "Feature 46: 0.00814\n",
      "Feature 47: 0.00386\n",
      "Feature 48: 0.00420\n",
      "Feature 49: 0.00108\n",
      "Feature 50: 0.00316\n",
      "Feature 51: 0.01375\n",
      "Feature 52: 0.00000\n",
      "Feature 53: 0.01279\n",
      "Feature 54: 0.01036\n",
      "Feature 55: 0.00650\n",
      "Feature 56: 0.00250\n",
      "Feature 57: 0.00499\n",
      "Feature 58: 0.01294\n",
      "Feature 59: 0.00269\n",
      "Feature 60: 0.00000\n",
      "Feature 61: 0.00769\n",
      "Feature 62: 0.00241\n",
      "Feature 63: 0.01382\n",
      "Feature 64: 0.00195\n",
      "Feature 65: 0.01767\n",
      "Feature 66: 0.00250\n",
      "Feature 67: 0.00030\n",
      "Feature 68: 0.00466\n",
      "Feature 69: 0.00206\n",
      "Feature 70: 0.00000\n",
      "Feature 71: 0.00656\n",
      "Feature 72: 0.00506\n",
      "Feature 73: 0.00427\n",
      "Feature 74: 0.00013\n",
      "Feature 75: 0.01125\n",
      "Feature 76: 0.00000\n",
      "Feature 77: 0.00466\n",
      "Feature 78: 0.02102\n",
      "Feature 79: 0.01225\n",
      "Feature 80: 0.00478\n",
      "Feature 81: 0.01064\n",
      "Feature 82: 0.00291\n",
      "Feature 83: 0.00000\n",
      "Feature 84: 0.00333\n",
      "Feature 85: 0.01305\n",
      "Feature 86: 0.00149\n",
      "Feature 87: 0.00369\n",
      "Feature 88: 0.01051\n",
      "Feature 89: 0.00617\n",
      "Feature 90: 0.01729\n",
      "Feature 91: 0.01582\n",
      "Feature 92: 0.00337\n",
      "Feature 93: 0.00930\n",
      "Feature 94: 0.00423\n",
      "Feature 95: 0.00250\n",
      "Feature 96: 0.00336\n",
      "Feature 97: 0.00881\n",
      "Feature 98: 0.01545\n",
      "Feature 99: 0.00596\n",
      "Feature 100: 0.00000\n",
      "Feature 101: 0.01588\n",
      "Feature 102: 0.01709\n",
      "Feature 103: 0.00000\n",
      "Feature 104: 0.00514\n",
      "Feature 105: 0.00000\n",
      "Feature 106: 0.00132\n",
      "Feature 107: 0.00234\n",
      "Feature 108: 0.00153\n",
      "Feature 109: 0.00000\n",
      "Feature 110: 0.00000\n",
      "Feature 111: 0.00733\n",
      "Feature 112: 0.00000\n",
      "Feature 113: 0.02013\n",
      "Feature 114: 0.00210\n",
      "Feature 115: 0.00000\n",
      "Feature 116: 0.00486\n",
      "Feature 117: 0.02076\n",
      "Feature 118: 0.00147\n",
      "Feature 119: 0.00000\n",
      "Feature 120: 0.00000\n",
      "Feature 121: 0.00497\n",
      "Feature 122: 0.01099\n",
      "Feature 123: 0.00756\n",
      "Feature 124: 0.00337\n",
      "Feature 125: 0.00000\n",
      "Feature 126: 0.02135\n",
      "Feature 127: 0.00206\n",
      "Feature 128: 0.00011\n",
      "Feature 129: 0.00090\n",
      "Feature 130: 0.00000\n",
      "Feature 131: 0.00144\n",
      "Feature 132: 0.00530\n",
      "Feature 133: 0.01195\n",
      "Feature 134: 0.00000\n",
      "Feature 135: 0.00145\n",
      "Feature 136: 0.00245\n",
      "Feature 137: 0.00000\n",
      "Feature 138: 0.00000\n",
      "Feature 139: 0.01863\n",
      "Feature 140: 0.00000\n",
      "Feature 141: 0.02186\n",
      "Feature 142: 0.00000\n",
      "Feature 143: 0.00254\n",
      "Feature 144: 0.00000\n",
      "Feature 145: 0.01986\n",
      "Feature 146: 0.00243\n",
      "Feature 147: 0.00650\n",
      "Feature 148: 0.00653\n",
      "Feature 149: 0.00311\n",
      "Feature 150: 0.00000\n",
      "Feature 151: 0.00000\n",
      "Feature 152: 0.00391\n",
      "Feature 153: 0.00842\n",
      "Feature 154: 0.01502\n",
      "Feature 155: 0.00973\n",
      "Feature 156: 0.00628\n",
      "Feature 157: 0.00429\n",
      "Feature 158: 0.00724\n",
      "Feature 159: 0.01884\n",
      "Feature 160: 0.01713\n",
      "Feature 161: 0.00656\n",
      "Feature 162: 0.00567\n",
      "Feature 163: 0.01633\n",
      "Feature 164: 0.01153\n",
      "Feature 165: 0.00314\n",
      "Feature 166: 0.00527\n",
      "Feature 167: 0.01298\n",
      "Feature 168: 0.01787\n",
      "Feature 169: 0.00182\n",
      "Feature 170: 0.00289\n",
      "Feature 171: 0.00456\n",
      "Feature 172: 0.00000\n",
      "Feature 173: 0.00000\n",
      "Feature 174: 0.00446\n",
      "Feature 175: 0.00211\n",
      "Feature 176: 0.01084\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.90419885, 0.90507742, 0.90357653, 0.90112384, 0.90415873])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgmodel = XGBClassifier(use_label_encoder = False, eval_metric='mlogloss',n_jobs = -1 )\n",
    "xgmodel.fit (x_train_sub,y_train_sub)\n",
    "\n",
    "KFoldresult_5fold(xgmodel, x_train_sub, y_train_sub, False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XG Boost with main_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\user\\anaconda3\\lib\\site-packages\\xgboost\\data.py:112: UserWarning: Use subset (sliced data) of np.ndarray is not recommended because it will generate extra copies and increase memory consumption\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "              colsample_bynode=1, colsample_bytree=1, eval_metric='mlogloss',\n",
      "              gamma=0, gpu_id=-1, importance_type='gain',\n",
      "              interaction_constraints='', learning_rate=0.300000012,\n",
      "              max_delta_step=0, max_depth=6, min_child_weight=1, missing=nan,\n",
      "              monotone_constraints='()', n_estimators=100, n_jobs=-1,\n",
      "              num_parallel_tree=1, random_state=0, reg_alpha=0, reg_lambda=1,\n",
      "              scale_pos_weight=1, subsample=1, tree_method='exact',\n",
      "              use_label_encoder=False, validate_parameters=1, verbosity=None)\n",
      "KFolds cross validation: \n",
      " [0.8852363  0.88417469 0.8852363  0.88358897 0.88431688] \n",
      "\n",
      "Mean accuracy: \n",
      " 0.8845106294020857\n",
      "\n",
      "Coefficient of feature: \n",
      "\n",
      "Feature 0: 0.03108\n",
      "Feature 1: 0.00756\n",
      "Feature 2: 0.00699\n",
      "Feature 3: 0.26823\n",
      "Feature 4: 0.01556\n",
      "Feature 5: 0.03867\n",
      "Feature 6: 0.01115\n",
      "Feature 7: 0.04227\n",
      "Feature 8: 0.01742\n",
      "Feature 9: 0.01967\n",
      "Feature 10: 0.01836\n",
      "Feature 11: 0.14094\n",
      "Feature 12: 0.03676\n",
      "Feature 13: 0.01656\n",
      "Feature 14: 0.04232\n",
      "Feature 15: 0.01976\n",
      "Feature 16: 0.03752\n",
      "Feature 17: 0.19400\n",
      "Feature 18: 0.01180\n",
      "Feature 19: 0.00791\n",
      "Feature 20: 0.00822\n",
      "Feature 21: 0.00728\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.8852363 , 0.88417469, 0.8852363 , 0.88358897, 0.88431688])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgmodel = XGBClassifier(use_label_encoder = False, eval_metric='mlogloss',n_jobs = -1 )\n",
    "xgmodel.fit (x_train,y_train)\n",
    "\n",
    "KFoldresult_5fold(xgmodel, x_train, y_train, False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: XG Boost results with sub_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion_matrix: \n",
      " [[12852  2050]\n",
      " [ 1288 17956]] \n",
      "\n",
      "Classification report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.91      0.86      0.89     14902\n",
      "         1.0       0.90      0.93      0.91     19244\n",
      "\n",
      "    accuracy                           0.90     34146\n",
      "   macro avg       0.90      0.90      0.90     34146\n",
      "weighted avg       0.90      0.90      0.90     34146\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "xgmodel_y_predict = predictionresult(xgmodel, x_test_sub, y_test_sub)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: XG Boost results with main_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion_matrix: \n",
      " [[12297  2605]\n",
      " [ 1334 17910]] \n",
      "\n",
      "Classification report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.90      0.83      0.86     14902\n",
      "         1.0       0.87      0.93      0.90     19244\n",
      "\n",
      "    accuracy                           0.88     34146\n",
      "   macro avg       0.89      0.88      0.88     34146\n",
      "weighted avg       0.89      0.88      0.88     34146\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "xgmodel_y_predict = predictionresult(xgmodel, x_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### -Optimizing Hyperparameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimizing with sub_categories (did not run, took took long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "param = {'max_depth': list(range(1,5))}\n",
    "gridsearch (xgmodel, param, x_train_sub, y_train_sub)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimizaing with main_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter tested: {'max_depth': [1, 2, 3, 4]}\n",
      "Best Score : 0.8846936661427026\n",
      "Best parameters: {'max_depth': 4}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(estimator=XGBClassifier(base_score=0.5, booster='gbtree',\n",
       "                                     colsample_bylevel=1, colsample_bynode=1,\n",
       "                                     colsample_bytree=1, eval_metric='mlogloss',\n",
       "                                     gamma=0, gpu_id=-1, importance_type='gain',\n",
       "                                     interaction_constraints='',\n",
       "                                     learning_rate=0.300000012,\n",
       "                                     max_delta_step=0, max_depth=6,\n",
       "                                     min_child_weight=1, missing=nan,\n",
       "                                     monotone_constraints='()',\n",
       "                                     n_estimators=100, n_jobs=-1,\n",
       "                                     num_parallel_tree=1, random_state=0,\n",
       "                                     reg_alpha=0, reg_lambda=1,\n",
       "                                     scale_pos_weight=1, subsample=1,\n",
       "                                     tree_method='exact',\n",
       "                                     use_label_encoder=False,\n",
       "                                     validate_parameters=1, verbosity=None),\n",
       "             n_jobs=-1, param_grid={'max_depth': [1, 2, 3, 4]})"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param = {'max_depth': list(range(1,5))}\n",
    "gridsearch (xgmodel, param, x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Apply the best parameters {'max_depth': 4}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\user\\anaconda3\\lib\\site-packages\\xgboost\\data.py:112: UserWarning: Use subset (sliced data) of np.ndarray is not recommended because it will generate extra copies and increase memory consumption\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "              colsample_bynode=1, colsample_bytree=1, eval_metric='mlogloss',\n",
      "              gamma=0, gpu_id=-1, importance_type='gain',\n",
      "              interaction_constraints='', learning_rate=0.300000012,\n",
      "              max_delta_step=0, max_depth=4, min_child_weight=1, missing=nan,\n",
      "              monotone_constraints='()', n_estimators=100, n_jobs=-1,\n",
      "              num_parallel_tree=1, random_state=0, reg_alpha=0, reg_lambda=1,\n",
      "              scale_pos_weight=1, subsample=1, tree_method='exact',\n",
      "              use_label_encoder=False, validate_parameters=1, verbosity=None)\n",
      "KFolds cross validation: \n",
      " [0.88611487 0.88530951 0.88501666 0.88263719 0.8843901 ] \n",
      "\n",
      "Mean accuracy: \n",
      " 0.8846936661427026\n",
      "\n",
      "Coefficient of feature: \n",
      "\n",
      "Feature 0: 0.04288\n",
      "Feature 1: 0.00569\n",
      "Feature 2: 0.00476\n",
      "Feature 3: 0.36035\n",
      "Feature 4: 0.01319\n",
      "Feature 5: 0.04188\n",
      "Feature 6: 0.01294\n",
      "Feature 7: 0.03399\n",
      "Feature 8: 0.00997\n",
      "Feature 9: 0.01678\n",
      "Feature 10: 0.01453\n",
      "Feature 11: 0.14604\n",
      "Feature 12: 0.02020\n",
      "Feature 13: 0.01210\n",
      "Feature 14: 0.03491\n",
      "Feature 15: 0.01562\n",
      "Feature 16: 0.03379\n",
      "Feature 17: 0.15181\n",
      "Feature 18: 0.00768\n",
      "Feature 19: 0.00625\n",
      "Feature 20: 0.00982\n",
      "Feature 21: 0.00481\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.88611487, 0.88530951, 0.88501666, 0.88263719, 0.8843901 ])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgmodel = XGBClassifier(use_label_encoder = False, eval_metric='mlogloss',n_jobs = -1, max_depth = 4)\n",
    "xgmodel.fit (x_train,y_train)\n",
    "\n",
    "KFoldresult_5fold(xgmodel, x_train, y_train, False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: Prediction results with sub_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgmodel_y_predict = predictionresult(xgmodel, x_test_sub, y_test_sub)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Below cell: Prediction results with main_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion_matrix: \n",
      " [[12231  2671]\n",
      " [ 1298 17946]] \n",
      "\n",
      "Classification report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.90      0.82      0.86     14902\n",
      "         1.0       0.87      0.93      0.90     19244\n",
      "\n",
      "    accuracy                           0.88     34146\n",
      "   macro avg       0.89      0.88      0.88     34146\n",
      "weighted avg       0.89      0.88      0.88     34146\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "xgmodel_y_predict = predictionresult(xgmodel, x_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## did not touch the following codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Logistic Regression</th>\n",
       "      <th>Random Forest Classifier</th>\n",
       "      <th>XGBoost</th>\n",
       "      <th>Best Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Accuracy</th>\n",
       "      <td>0.676624</td>\n",
       "      <td>0.865167</td>\n",
       "      <td>0.883764</td>\n",
       "      <td>XGBoost</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall</th>\n",
       "      <td>0.802952</td>\n",
       "      <td>0.940345</td>\n",
       "      <td>0.932550</td>\n",
       "      <td>Random Forest Classifier</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Precision</th>\n",
       "      <td>0.680645</td>\n",
       "      <td>0.839644</td>\n",
       "      <td>0.870447</td>\n",
       "      <td>XGBoost</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1 Score</th>\n",
       "      <td>0.736757</td>\n",
       "      <td>0.887146</td>\n",
       "      <td>0.900429</td>\n",
       "      <td>XGBoost</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Logistic Regression  Random Forest Classifier   XGBoost  \\\n",
       "Accuracy              0.676624                  0.865167  0.883764   \n",
       "Recall                0.802952                  0.940345  0.932550   \n",
       "Precision             0.680645                  0.839644  0.870447   \n",
       "F1 Score              0.736757                  0.887146  0.900429   \n",
       "\n",
       "                         Best Score  \n",
       "Accuracy                    XGBoost  \n",
       "Recall     Random Forest Classifier  \n",
       "Precision                   XGBoost  \n",
       "F1 Score                    XGBoost  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_score = performace(y_test, log_y_predict)\n",
    "rf_score = performace(y_test, ranforest_y_predict)\n",
    "xg_score = performace(y_test, xgmodel_y_predict)\n",
    "\n",
    "models_scores_table = pd.DataFrame({'Logistic Regression': log_score, 'Random Forest Classifier': rf_score, 'XGBoost':xg_score},\n",
    "                                    index=['Accuracy', 'Recall', 'Precision', 'F1 Score'])\n",
    "\n",
    "models_scores_table['Best Score'] = models_scores_table.idxmax(axis=1)\n",
    "\n",
    "models_scores_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}