Tree0.34Calibration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "from sklearn.model_selection import StratifiedKFold, KFold,StratifiedShuffleSplit, cross_val_score, train_test_split\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "\n",
    "\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from catboost import CatBoostClassifier, Pool\n",
    "from sklearn.model_selection import RandomizedSearchCV, GridSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import svm\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import warnings\n",
    "from scipy import stats\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dftrain = pd.read_csv('MPP_Dataset/Train.csv')\n",
    "dftest = pd.read_csv('MPP_Dataset/Test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Store_Ratio</th>\n",
       "      <td>18208.0</td>\n",
       "      <td>0.544283</td>\n",
       "      <td>0.202709</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.411000</td>\n",
       "      <td>0.573000</td>\n",
       "      <td>0.69900</td>\n",
       "      <td>0.998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Basket_Ratio</th>\n",
       "      <td>18208.0</td>\n",
       "      <td>0.483585</td>\n",
       "      <td>0.302010</td>\n",
       "      <td>0.000216</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.517000</td>\n",
       "      <td>0.74200</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Category_1</th>\n",
       "      <td>18208.0</td>\n",
       "      <td>5.155536</td>\n",
       "      <td>3.535068</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>8.00000</td>\n",
       "      <td>11.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Store_Score</th>\n",
       "      <td>18208.0</td>\n",
       "      <td>-12.198086</td>\n",
       "      <td>8.370566</td>\n",
       "      <td>-47.576000</td>\n",
       "      <td>-16.496250</td>\n",
       "      <td>-9.166500</td>\n",
       "      <td>-5.94375</td>\n",
       "      <td>-0.079</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Category_2</th>\n",
       "      <td>18208.0</td>\n",
       "      <td>0.648506</td>\n",
       "      <td>0.477450</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Store_Presence</th>\n",
       "      <td>18208.0</td>\n",
       "      <td>0.477702</td>\n",
       "      <td>0.380634</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.086175</td>\n",
       "      <td>0.430000</td>\n",
       "      <td>0.89500</td>\n",
       "      <td>0.996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Score_1</th>\n",
       "      <td>18208.0</td>\n",
       "      <td>0.322109</td>\n",
       "      <td>0.413493</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.002245</td>\n",
       "      <td>0.85900</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Score_2</th>\n",
       "      <td>18208.0</td>\n",
       "      <td>0.164888</td>\n",
       "      <td>0.136531</td>\n",
       "      <td>0.011900</td>\n",
       "      <td>0.095300</td>\n",
       "      <td>0.112000</td>\n",
       "      <td>0.17600</td>\n",
       "      <td>0.991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Score_3</th>\n",
       "      <td>18208.0</td>\n",
       "      <td>0.421440</td>\n",
       "      <td>0.271922</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.184750</td>\n",
       "      <td>0.393000</td>\n",
       "      <td>0.64000</td>\n",
       "      <td>0.999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Score_4</th>\n",
       "      <td>18208.0</td>\n",
       "      <td>115.305776</td>\n",
       "      <td>31.478303</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>90.974250</td>\n",
       "      <td>114.022500</td>\n",
       "      <td>134.99700</td>\n",
       "      <td>219.701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>time</th>\n",
       "      <td>18208.0</td>\n",
       "      <td>218010.142849</td>\n",
       "      <td>96482.888585</td>\n",
       "      <td>31844.000000</td>\n",
       "      <td>169620.250000</td>\n",
       "      <td>204646.000000</td>\n",
       "      <td>247156.25000</td>\n",
       "      <td>3908467.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>popularity</th>\n",
       "      <td>18208.0</td>\n",
       "      <td>3.850231</td>\n",
       "      <td>0.544226</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.00000</td>\n",
       "      <td>5.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  count           mean           std           min  \\\n",
       "Store_Ratio     18208.0       0.544283      0.202709      0.000000   \n",
       "Basket_Ratio    18208.0       0.483585      0.302010      0.000216   \n",
       "Category_1      18208.0       5.155536      3.535068      0.000000   \n",
       "Store_Score     18208.0     -12.198086      8.370566    -47.576000   \n",
       "Category_2      18208.0       0.648506      0.477450      0.000000   \n",
       "Store_Presence  18208.0       0.477702      0.380634      0.000000   \n",
       "Score_1         18208.0       0.322109      0.413493      0.000000   \n",
       "Score_2         18208.0       0.164888      0.136531      0.011900   \n",
       "Score_3         18208.0       0.421440      0.271922      0.000000   \n",
       "Score_4         18208.0     115.305776     31.478303      0.000000   \n",
       "time            18208.0  218010.142849  96482.888585  31844.000000   \n",
       "popularity      18208.0       3.850231      0.544226      0.000000   \n",
       "\n",
       "                          25%            50%           75%          max  \n",
       "Store_Ratio          0.411000       0.573000       0.69900        0.998  \n",
       "Basket_Ratio         0.200000       0.517000       0.74200        1.000  \n",
       "Category_1           2.000000       5.000000       8.00000       11.000  \n",
       "Store_Score        -16.496250      -9.166500      -5.94375       -0.079  \n",
       "Category_2           0.000000       1.000000       1.00000        1.000  \n",
       "Store_Presence       0.086175       0.430000       0.89500        0.996  \n",
       "Score_1              0.000001       0.002245       0.85900        1.000  \n",
       "Score_2              0.095300       0.112000       0.17600        0.991  \n",
       "Score_3              0.184750       0.393000       0.64000        0.999  \n",
       "Score_4             90.974250     114.022500     134.99700      219.701  \n",
       "time            169620.250000  204646.000000  247156.25000  3908467.000  \n",
       "popularity           4.000000       4.000000       4.00000        5.000  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dftrain.describe().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('MPP_Dataset/Train.csv')\n",
    "test = pd.read_csv('MPP_Dataset/Test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train.head()\n",
    "\n",
    "train.popularity = train.popularity.replace(5,2) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    15231\n",
       "3     2166\n",
       "2      472\n",
       "1      323\n",
       "0       16\n",
       "Name: popularity, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.popularity.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Store_Ratio</th>\n",
       "      <td>12140.0</td>\n",
       "      <td>0.543776</td>\n",
       "      <td>0.200109</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.414750</td>\n",
       "      <td>0.57000</td>\n",
       "      <td>0.69600</td>\n",
       "      <td>0.978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Basket_Ratio</th>\n",
       "      <td>12140.0</td>\n",
       "      <td>0.488879</td>\n",
       "      <td>0.301217</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.213000</td>\n",
       "      <td>0.52100</td>\n",
       "      <td>0.74500</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Category_1</th>\n",
       "      <td>12140.0</td>\n",
       "      <td>5.121417</td>\n",
       "      <td>3.528765</td>\n",
       "      <td>0.000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.00000</td>\n",
       "      <td>8.00000</td>\n",
       "      <td>11.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Store_Score</th>\n",
       "      <td>12140.0</td>\n",
       "      <td>-12.062847</td>\n",
       "      <td>8.300385</td>\n",
       "      <td>-46.847</td>\n",
       "      <td>-16.066000</td>\n",
       "      <td>-9.04650</td>\n",
       "      <td>-5.89175</td>\n",
       "      <td>0.662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Category_2</th>\n",
       "      <td>12140.0</td>\n",
       "      <td>0.642916</td>\n",
       "      <td>0.479160</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Store_Presence</th>\n",
       "      <td>12140.0</td>\n",
       "      <td>0.474675</td>\n",
       "      <td>0.377582</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.087775</td>\n",
       "      <td>0.43050</td>\n",
       "      <td>0.88200</td>\n",
       "      <td>0.996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Score_1</th>\n",
       "      <td>12140.0</td>\n",
       "      <td>0.321641</td>\n",
       "      <td>0.414438</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.00205</td>\n",
       "      <td>0.86100</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Score_2</th>\n",
       "      <td>12140.0</td>\n",
       "      <td>0.165942</td>\n",
       "      <td>0.139109</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.095900</td>\n",
       "      <td>0.11200</td>\n",
       "      <td>0.17325</td>\n",
       "      <td>0.994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Score_3</th>\n",
       "      <td>12140.0</td>\n",
       "      <td>0.420485</td>\n",
       "      <td>0.270508</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.190000</td>\n",
       "      <td>0.39200</td>\n",
       "      <td>0.63300</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Score_4</th>\n",
       "      <td>12140.0</td>\n",
       "      <td>115.748041</td>\n",
       "      <td>31.519999</td>\n",
       "      <td>0.000</td>\n",
       "      <td>90.908750</td>\n",
       "      <td>114.61850</td>\n",
       "      <td>135.82925</td>\n",
       "      <td>219.701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>time</th>\n",
       "      <td>12140.0</td>\n",
       "      <td>215403.104942</td>\n",
       "      <td>82311.080038</td>\n",
       "      <td>34067.000</td>\n",
       "      <td>170000.000000</td>\n",
       "      <td>204341.50000</td>\n",
       "      <td>244522.25000</td>\n",
       "      <td>1782178.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  count           mean           std        min  \\\n",
       "Store_Ratio     12140.0       0.543776      0.200109      0.000   \n",
       "Basket_Ratio    12140.0       0.488879      0.301217      0.000   \n",
       "Category_1      12140.0       5.121417      3.528765      0.000   \n",
       "Store_Score     12140.0     -12.062847      8.300385    -46.847   \n",
       "Category_2      12140.0       0.642916      0.479160      0.000   \n",
       "Store_Presence  12140.0       0.474675      0.377582      0.000   \n",
       "Score_1         12140.0       0.321641      0.414438      0.000   \n",
       "Score_2         12140.0       0.165942      0.139109      0.000   \n",
       "Score_3         12140.0       0.420485      0.270508      0.000   \n",
       "Score_4         12140.0     115.748041     31.519999      0.000   \n",
       "time            12140.0  215403.104942  82311.080038  34067.000   \n",
       "\n",
       "                          25%           50%           75%          max  \n",
       "Store_Ratio          0.414750       0.57000       0.69600        0.978  \n",
       "Basket_Ratio         0.213000       0.52100       0.74500        1.000  \n",
       "Category_1           2.000000       5.00000       8.00000       11.000  \n",
       "Store_Score        -16.066000      -9.04650      -5.89175        0.662  \n",
       "Category_2           0.000000       1.00000       1.00000        1.000  \n",
       "Store_Presence       0.087775       0.43050       0.88200        0.996  \n",
       "Score_1              0.000001       0.00205       0.86100        1.000  \n",
       "Score_2              0.095900       0.11200       0.17325        0.994  \n",
       "Score_3              0.190000       0.39200       0.63300        1.000  \n",
       "Score_4             90.908750     114.61850     135.82925      219.701  \n",
       "time            170000.000000  204341.50000  244522.25000  1782178.000  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.describe().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train['Store_Ratio'] = np.sqrt(train['Store_Ratio'])\n",
    "# test['Store_Ratio'] = np.sqrt(test['Store_Ratio'])\n",
    "\n",
    "# train['Basket_Ratio'] = np.sqrt(train['Basket_Ratio'])\n",
    "# test['Basket_Ratio'] = np.sqrt(test['Basket_Ratio'])\n",
    "\n",
    "# train['Store_Ratio'],fitted_lambda = stats.boxcox(train['Store_Ratio'] + 47.576)\n",
    "# test['Store_Ratio'] = stats.boxcox((test['Store_Ratio'] + 46.847), fitted_lambda)\n",
    "\n",
    "# train['Store_Presence'] = np.sqrt(train['Store_Presence'])\n",
    "# test['Store_Presence'] = np.sqrt(test['Store_Presence'])\n",
    "\n",
    "\n",
    "# #train['Store_Score'] = np.sqrt(train['Store_Score'])\n",
    "# #test['Store_Score'] = np.sqrt(test['Store_Score'])\n",
    "\n",
    "# train['Score_1'] = np.sqrt(train['Score_1'])\n",
    "# test['Score_1'] = np.sqrt(test['Score_1'])\n",
    "\n",
    "\n",
    "# train['Score_2'] = np.sqrt(train['Score_2'])\n",
    "# test['Score_2'] = np.sqrt(test['Score_2'])\n",
    "\n",
    "\n",
    "# train['Score_3'] = np.sqrt(train['Score_3'])\n",
    "# test['Score_3'] = np.sqrt(test['Score_3'])\n",
    "\n",
    "\n",
    "# train['Score_4'] = np.sqrt(train['Score_4'])\n",
    "# test['Score_4'] = np.sqrt(test['Score_4'])\n",
    "\n",
    "\n",
    "# train['time'] = np.log1p(train['time'])\n",
    "# test['time'] = np.log1p(test['time'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #Train\n",
    "# train['Std'] = train.std(axis=1).round(2)\n",
    "# train['quantile1'] = train.quantile(q=0.01,axis=1).round(2)\n",
    "# train['quantile3'] = train.quantile(q=0.25,axis=1).round(2)\n",
    "# train['quantile5'] = train.quantile(q=0.50,axis=1).round(2)                                              \n",
    "# train['Mean'] = train.mean(axis=1).round(2)\n",
    "# #Test\n",
    "# test['Std'] = test.std(axis=1).round(2)\n",
    "# test['quantile1'] = test.quantile(q=0.01,axis=1).round(2)\n",
    "# test['quantile3'] = test.quantile(q=0.25,axis=1).round(2)\n",
    "# test['quantile5'] = test.quantile(q=0.50,axis=1).round(2)                                              \n",
    "# test['Mean'] = test.mean(axis=1).round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def onehotencoder(train,test,cols=[]):\n",
    "    \n",
    "    from sklearn.preprocessing import OneHotEncoder\n",
    "    #object_cols = catVar1(train) #catVar1 gives desired categorical column and not all object columns\n",
    "    object_cols=cols\n",
    "    print(object_cols)\n",
    "    \n",
    "    OH_encoder = OneHotEncoder(handle_unknown='ignore', sparse=False)\n",
    "    OH_cols_train = pd.DataFrame(OH_encoder.fit_transform(train[object_cols]))\n",
    "    OH_cols_test = pd.DataFrame(OH_encoder.transform(test[object_cols]))\n",
    "\n",
    "    # One-hot encoding removed index; put it back\n",
    "    OH_cols_train.index = train.index\n",
    "    OH_cols_test.index = test.index\n",
    "\n",
    "    ##hack for restoring columns names just like get dummies\n",
    "    column_name = OH_encoder.get_feature_names(object_cols)\n",
    "    OH_cols_train.columns = column_name\n",
    "    OH_cols_test.columns = column_name\n",
    "    \n",
    "\n",
    "    # Remove desired categorical columns (will replace with one-hot encoding)\n",
    "    num_train = train.drop(object_cols, axis=1)\n",
    "    num_test = test.drop(object_cols, axis=1)\n",
    "\n",
    "    # Add one-hot encoded columns to numerical/remaining features\n",
    "    OH_train = pd.concat([num_train, OH_cols_train], axis=1)\n",
    "    OH_test = pd.concat([num_test, OH_cols_test], axis=1)\n",
    "    \n",
    "\n",
    "    \n",
    "    print(OH_train.shape,OH_test.shape)\n",
    "    \n",
    "    \n",
    "    return OH_train,OH_test\n",
    "\n",
    "def targetencoding(train,test,y_train,drop=False,cols=[]):\n",
    "    import category_encoders as ce\n",
    "    # Create the encoder itself\n",
    "    cat_features = cols\n",
    "    print(f'targest emcoding for features {cat_features}')\n",
    "    target_enc = ce.TargetEncoder(cols=cat_features)\n",
    "\n",
    "    \n",
    "\n",
    "    # Fit the encoder using the categorical features and target\n",
    "    target_enc.fit(train[cat_features], y_train)\n",
    "    \n",
    "\n",
    "    # Transform the features, rename the columns with _target suffix, and join to dataframe\n",
    "    traintrgtenc = train.join(target_enc.transform(train[cat_features]).add_suffix('_target'))\n",
    "    testtrgtenc = test.join(target_enc.transform(test[cat_features]).add_suffix('_target'))\n",
    "\n",
    "    if drop :\n",
    "        traintrgtenc = traintrgtenc.drop(cat_features, axis =1)\n",
    "        testtrgtenc = testtrgtenc.drop(cat_features, axis =1)\n",
    "\n",
    "    \n",
    "    print(traintrgtenc.shape,testtrgtenc.shape)\n",
    "    \n",
    "    return traintrgtenc,testtrgtenc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "targest emcoding for features ['Category_1']\n",
      "(18208, 12) (12140, 11)\n",
      "['Category_2']\n",
      "(18208, 13) (12140, 12)\n"
     ]
    }
   ],
   "source": [
    "trainenc,testenc = targetencoding(train,test,train.popularity,drop=True,cols =['Category_1'])\n",
    "trainencohe,testencohe = onehotencoder(trainenc,testenc,cols =['Category_2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = trainenc.drop('popularity',axis=1)\n",
    "Y = trainenc['popularity']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train, X_valid, y_train, y_valid = train_test_split(X, Y, train_size=0.70, test_size=0.30,random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model1 = ExtraTreesClassifier(n_estimators=500,criterion= 'gini',warm_start=True,\n",
    "#                                 oob_score=False,bootstrap=False,max_features='auto',\n",
    "#                                 random_state=2)\n",
    "\n",
    "\n",
    "# model5= XGBClassifier(subsample= 1.0, n_estimators= 425, min_child_weight= 1, max_depth= 4,\n",
    "#                          learning_rate=0.2, gamma= 0.5, colsample_bytree= 0.6,random_state=2)\n",
    "# model6= LGBMClassifier(random_state=2)\n",
    "# model7 = CatBoostClassifier(loss_function='MultiClass', eval_metric='TotalF1', \n",
    "#                          classes_count=6,\n",
    "#                          depth=6,\n",
    "#                          random_seed=101, \n",
    "#                          iterations=3500, \n",
    "#                          learning_rate=0.1,\n",
    "#                          leaf_estimation_iterations=1,\n",
    "#                          l2_leaf_reg=1,\n",
    "#                          bootstrap_type='Bayesian', \n",
    "#                          bagging_temperature=0.8, \n",
    "#                          random_strength=1,\n",
    "#                          od_type='Iter', \n",
    "#                          border_count=100,\n",
    "#                          od_wait=500)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def score_dataset(X_train, X_valid, y_train, y_valid,models = []):\n",
    "    \n",
    "    for model in models:\n",
    "        clf1 = model\n",
    "        clf1.fit(X_train,y_train)\n",
    "\n",
    "\n",
    "        preds = clf1.predict(X_valid)\n",
    "        predsforlog = clf1.predict_proba(X_valid)\n",
    "\n",
    "\n",
    "        target_names = ['class_0','class_1','class_2','class_3','class_4']\n",
    "        print(classification_report(y_valid, preds, target_names=target_names,labels= [0,1,2,3,4]))\n",
    "        print(confusion_matrix(y_valid,preds))\n",
    "        print(log_loss(y_valid,predsforlog))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# score_dataset(X_train, X_valid, y_train, y_valid,models=[model1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# score_dataset(X_train, X_valid, y_train, y_valid,models=[model4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# score_dataset(X_train, X_valid, y_train, y_valid,models=[model5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# score_dataset(X_train, X_valid, y_train, y_valid,models=[model6])\n",
    "from sklearn.metrics import make_scorer, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model7 = CatBoostClassifier(loss_function='MultiClass', \n",
    "#                          eval_metric='MultiClass', \n",
    "#                          classes_count=5,\n",
    "#                          depth=6,\n",
    "#                          random_seed=101, \n",
    "#                          iterations=500, \n",
    "#                          learning_rate=0.1,\n",
    "#                          leaf_estimation_iterations=1,\n",
    "#                          l2_leaf_reg=1,\n",
    "#                          bootstrap_type='Bayesian', \n",
    "#                          bagging_temperature=0.8, \n",
    "#                          random_strength=1,\n",
    "#                          od_type='Iter', \n",
    "#                          border_count=100,\n",
    "#                          od_wait=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# score_dataset(X_train, X_valid, y_train, y_valid,models=[model7])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def logloss(X,y,test,model,n_folds=5):\n",
    "#     kf = StratifiedKFold(n_folds, shuffle=True, random_state=2).get_n_splits(X_train.values)\n",
    "#     logloss= (-cross_val_score(model, X, y, scoring=\"neg_log_loss\", cv = kf))\n",
    "#     return(logloss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # Function for comparing different final approaches and do final prediction\n",
    "# def score_dataset_predictions(train,test,model):\n",
    "    \n",
    "#     folds = StratifiedKFold(n_splits=5,shuffle=True ,random_state=42)\n",
    "#     oof_preds = []\n",
    "#     oof_scores = []\n",
    "#     target_names = ['class_1','class_2','class_3','class_4','class_5']\n",
    "\n",
    "#     for train_index, test_index in skf.split(X, y):\n",
    "#         X_train, X_test = X[train_index], X[test_index]\n",
    "#         y_train, y_test = y[train_index], y[test_index]\n",
    "#         model = model\n",
    "#         model.fit(X_train,y_train)\n",
    "#         preds = model.predict_proba(X_test)\n",
    "#         preds_c = model.predict(X_test)\n",
    "#         score = log_loss(y_test,preds)\n",
    "#         oof_pred_fold = model.predict_proba(scaler.transform(test))\n",
    "#         oof_preds.append(oof_pred_fold)\n",
    "      \n",
    "#         print(score)\n",
    "#         #print(classification_report(y_valid, preds_c, target_names=target_names,labels= [1,2,3,4,5]))\n",
    "#         #print(confusion_matrix(y_valid,preds))\n",
    "#         oof_scores.append(score)\n",
    "\n",
    "#     print(\"Mean OOF Score :\",np.mean(oof_scores))\n",
    "#     final_preds = np.mean(np.array(oof_preds),axis=0)\n",
    "       \n",
    "        \n",
    "    \n",
    "# #return log_loss(y_valid,predsforlog)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def score_dataset(X_train, X_valid, y_train, y_valid):\n",
    "    \n",
    "#     from xgboost import XGBClassifier\n",
    "#     from ngboost.distns import k_categorical\n",
    "#     from ngboost.learners import default_tree_learner\n",
    "#     from sklearn.metrics import log_loss\n",
    "    \n",
    "#     from sklearn.metrics import classification_report\n",
    "#     from sklearn.tree import DecisionTreeClassifier\n",
    "#     from ngboost import NGBClassifier\n",
    "#     from sklearn.ensemble import BaggingClassifier\n",
    "#     from lightgbm import LGBMClassifier\n",
    "#     from sklearn.calibration import CalibratedClassifierCV\n",
    "#     from sklearn.model_selection import StratifiedKFold, KFold,StratifiedShuffleSplit\n",
    "#     from scipy.sparse import csr_matrix, hstack\n",
    "#     from sklearn.linear_model import Ridge\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#     clf1 = DecisionTreeClassifier(ccp_alpha=0.0, class_weight=None, criterion='entropy',\n",
    "#                        max_depth=None, max_features=None, max_leaf_nodes=None,\n",
    "#                        min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "#                        min_samples_leaf=8, min_samples_split=2,\n",
    "#                        min_weight_fraction_leaf=0.0, presort='deprecated',\n",
    "#                        random_state=None, splitter='best')\n",
    "\n",
    "#     clf2 = BaggingClassifier(base_estimator=clf1, n_estimators=10, \n",
    "#                              bootstrap=True,\n",
    "#                              bootstrap_features=False, oob_score=True, warm_start=False,\n",
    "#                              n_jobs=-1, random_state=786, verbose=0)\n",
    "    \n",
    "#     #base = Ridge(alpha=0.0, copy_X=True, fit_intercept=True, max_iter=None,\n",
    "#     #              normalize=False, random_state=None, solver='auto', tol=0.001) \n",
    "#     #clf2 = NGBClassifier(natural_gradient=True, verbose=False, Dist=k_categorical(16),\n",
    "#      #                   minibatch_frac= 1.0, n_estimators= 20)\n",
    "\n",
    "#     clf2 = XGBClassifier(subsample= 1.0, n_estimators= 125, min_child_weight= 1, max_depth= 3,\n",
    "#                          learning_rate=0.2, gamma= 0.5, colsample_bytree= 0.6,random_state=2)\n",
    "#     #clf2 = LGBMClassifier(random_state=2)\n",
    "#     #clf2 = CatBoostClassifier(iterations=300,\n",
    "# #                            depth=4,\n",
    "# #                            learning_rate=0.2,\n",
    "# #                            loss_function='MultiClass',\n",
    "# #                            verbose=True)\n",
    "\n",
    "#     #clf2 = NGBClassifier(random_state=2,Dist=k_categorical(16))\n",
    "    \n",
    "#     clf2.fit(X_train,y_train)\n",
    "    \n",
    "#     sss = StratifiedShuffleSplit(n_splits=10, test_size=0.4, random_state=3)\n",
    "#     sss.get_n_splits(X_train, y_train)\n",
    "    \n",
    "#     clf_sigmoid = CalibratedClassifierCV(clf2, cv=sss, method='isotonic')\n",
    "#     clf_sigmoid.fit(X_train, y_train)\n",
    "#     preds = clf_sigmoid.predict_proba(X_valid)\n",
    "#     preds1 = clf_sigmoid.predict(X_valid)\n",
    "\n",
    "# #     preds1 = clf2.predict(X_valid)\n",
    "# #     preds = clf2.predict_proba(X_valid)\n",
    "#     target_names = ['class 0', 'class 1', 'class 2','class 3', 'class 4', 'class 5', 'class 6',\n",
    "#                     'class 7', 'class 8', 'class 9','class 10', 'class 11', 'class 12', 'class 13',\n",
    "#                     'class 14', 'class 15']\n",
    "#     #print(preds)\n",
    "#     print(classification_report(y_valid, preds1, target_names=target_names,labels= [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]))\n",
    "#     return log_loss(y_valid,preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import lightgbm as lgb\n",
    "# train_data = lgb.Dataset(X_train, label=y_train)\n",
    "# test_data = lgb.Dataset(X_valid, label=y_valid)\n",
    "\n",
    "# param = {'objective': 'multiclass',\n",
    "#          'boosting': 'gbdt',  \n",
    "#          'metric': 'multiclass',\n",
    "#          'learning_rate': 0.1, \n",
    "#          'num_iterations': 500,\n",
    "#          'num_leaves': 500,\n",
    "#          'max_depth': -1,\n",
    "#          'min_data_in_leaf': 11,\n",
    "#          'bagging_fraction': 0.90,\n",
    "#          'bagging_freq': 1,\n",
    "#          'bagging_seed': 3,\n",
    "#          'feature_fraction': 0.90,\n",
    "#          'feature_fraction_seed': 2,\n",
    "#          'early_stopping_round': 200,\n",
    "#          'max_bin': 1000,\n",
    "#          'num_class': 5,\n",
    "#          }\n",
    "\n",
    "# lgbm = lgb.train(params=param, verbose_eval=100, train_set=train_data, valid_sets=[test_data])\n",
    "\n",
    "# y_pred_lgbm = lgbm.predict(X_valid)\n",
    "# print('loss:', (log_loss((y_valid),(y_pred_lgbm))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import seaborn as sns\n",
    "# feature_imp = pd.DataFrame(sorted(zip(lgbm.feature_importance(), X.columns), reverse=True)[:50], \n",
    "#                            columns=['Value','Feature'])\n",
    "# plt.figure(figsize=(12, 10))\n",
    "# sns.barplot(x=\"Value\", y=\"Feature\", data=feature_imp.sort_values(by=\"Value\", ascending=False))\n",
    "# plt.title('LightGBM Features')\n",
    "# plt.tight_layout()\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(val_set[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Store_Ratio          0\n",
       "Basket_Ratio         0\n",
       "Store_Score          0\n",
       "Category_2           0\n",
       "Store_Presence       0\n",
       "Score_1              0\n",
       "Score_2              0\n",
       "Score_3              0\n",
       "Score_4              0\n",
       "time                 0\n",
       "Category_1_target    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "##calibration \n",
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "model = ExtraTreesClassifier(n_estimators=500,criterion= 'gini',warm_start=True,\n",
    "                                oob_score=False,bootstrap=False,max_features='auto',\n",
    "                                random_state=2)\n",
    "calibrate = CalibratedClassifierCV(model, cv=5, method='isotonic')\n",
    "calibrate.fit(X, Y)\n",
    "preds = calibrate.predict_proba(testenc)\n",
    "\n",
    "\n",
    "#isotonic only performs good               \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.010066</td>\n",
       "      <td>0.017170</td>\n",
       "      <td>0.827305</td>\n",
       "      <td>0.145459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011057</td>\n",
       "      <td>0.010261</td>\n",
       "      <td>0.024272</td>\n",
       "      <td>0.954410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.012788</td>\n",
       "      <td>0.015441</td>\n",
       "      <td>0.040149</td>\n",
       "      <td>0.931623</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006853</td>\n",
       "      <td>0.012394</td>\n",
       "      <td>0.048174</td>\n",
       "      <td>0.932579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000522</td>\n",
       "      <td>0.002245</td>\n",
       "      <td>0.001444</td>\n",
       "      <td>0.995789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12135</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008168</td>\n",
       "      <td>0.010003</td>\n",
       "      <td>0.057345</td>\n",
       "      <td>0.924484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12136</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000520</td>\n",
       "      <td>0.002391</td>\n",
       "      <td>0.007286</td>\n",
       "      <td>0.989804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12137</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007277</td>\n",
       "      <td>0.002077</td>\n",
       "      <td>0.011931</td>\n",
       "      <td>0.978715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12138</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000521</td>\n",
       "      <td>0.000764</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.998715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12139</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.027198</td>\n",
       "      <td>0.014621</td>\n",
       "      <td>0.277617</td>\n",
       "      <td>0.680564</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12140 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0         1         2         3         4\n",
       "0      0.0  0.010066  0.017170  0.827305  0.145459\n",
       "1      0.0  0.011057  0.010261  0.024272  0.954410\n",
       "2      0.0  0.012788  0.015441  0.040149  0.931623\n",
       "3      0.0  0.006853  0.012394  0.048174  0.932579\n",
       "4      0.0  0.000522  0.002245  0.001444  0.995789\n",
       "...    ...       ...       ...       ...       ...\n",
       "12135  0.0  0.008168  0.010003  0.057345  0.924484\n",
       "12136  0.0  0.000520  0.002391  0.007286  0.989804\n",
       "12137  0.0  0.007277  0.002077  0.011931  0.978715\n",
       "12138  0.0  0.000521  0.000764  0.000000  0.998715\n",
       "12139  0.0  0.027198  0.014621  0.277617  0.680564\n",
       "\n",
       "[12140 rows x 5 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission = pd.DataFrame(preds)\n",
    "submission\n",
    "#submission.to_csv(\"etr_cal.csv\",index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.010066</td>\n",
       "      <td>0.017170</td>\n",
       "      <td>0.827305</td>\n",
       "      <td>0.145459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011057</td>\n",
       "      <td>0.010261</td>\n",
       "      <td>0.024272</td>\n",
       "      <td>0.954410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.012788</td>\n",
       "      <td>0.015441</td>\n",
       "      <td>0.040149</td>\n",
       "      <td>0.931623</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006853</td>\n",
       "      <td>0.012394</td>\n",
       "      <td>0.048174</td>\n",
       "      <td>0.932579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000522</td>\n",
       "      <td>0.002245</td>\n",
       "      <td>0.001444</td>\n",
       "      <td>0.995789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12135</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008168</td>\n",
       "      <td>0.010003</td>\n",
       "      <td>0.057345</td>\n",
       "      <td>0.924484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12136</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000520</td>\n",
       "      <td>0.002391</td>\n",
       "      <td>0.007286</td>\n",
       "      <td>0.989804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12137</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007277</td>\n",
       "      <td>0.002077</td>\n",
       "      <td>0.011931</td>\n",
       "      <td>0.978715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12138</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000521</td>\n",
       "      <td>0.000764</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.998715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12139</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.027198</td>\n",
       "      <td>0.014621</td>\n",
       "      <td>0.277617</td>\n",
       "      <td>0.680564</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12140 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0         1         2         3         4\n",
       "0      0.0  0.010066  0.017170  0.827305  0.145459\n",
       "1      0.0  0.011057  0.010261  0.024272  0.954410\n",
       "2      0.0  0.012788  0.015441  0.040149  0.931623\n",
       "3      0.0  0.006853  0.012394  0.048174  0.932579\n",
       "4      0.0  0.000522  0.002245  0.001444  0.995789\n",
       "...    ...       ...       ...       ...       ...\n",
       "12135  0.0  0.008168  0.010003  0.057345  0.924484\n",
       "12136  0.0  0.000520  0.002391  0.007286  0.989804\n",
       "12137  0.0  0.007277  0.002077  0.011931  0.978715\n",
       "12138  0.0  0.000521  0.000764  0.000000  0.998715\n",
       "12139  0.0  0.027198  0.014621  0.277617  0.680564\n",
       "\n",
       "[12140 rows x 5 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.372759581565962\n",
      "0.36950829479315167\n",
      "0.352366525675425\n",
      "0.37528318162945606\n",
      "0.35794674166532986\n",
      "Mean OOF Score : 0.3655728650658649\n"
     ]
    }
   ],
   "source": [
    "# scaler = StandardScaler()\n",
    "# X_ = scaler.fit_transform(X)\n",
    "# X = pd.DataFrame(X_,columns=X.columns)\n",
    "\n",
    "folds = StratifiedKFold(n_splits=5,shuffle=False ,random_state=42)\n",
    "\n",
    "oof_preds = []\n",
    "oof_scores = []\n",
    "\n",
    "for i , (train_idx,test_idx) in enumerate(folds.split(X,Y)):\n",
    "  train_set = (X.iloc[train_idx],Y.iloc[train_idx])\n",
    "  test_set = (X.iloc[test_idx],Y.iloc[test_idx])\n",
    "\n",
    "  model = ExtraTreesClassifier(n_estimators=500,criterion= 'gini',warm_start=True,\n",
    "                                oob_score=False,bootstrap=False,max_features='auto',\n",
    "                                random_state=2)\n",
    "\n",
    "  model.fit(*train_set)\n",
    "\n",
    "  preds = model.predict_proba(test_set[0])\n",
    "  score = log_loss(test_set[1],preds)\n",
    "  \n",
    "  oof_pred_fold = model.predict_proba((testenc)) #scaler.transform\n",
    "  oof_preds.append(oof_pred_fold)\n",
    "\n",
    "  print(score)\n",
    "  oof_scores.append(score)\n",
    "\n",
    "print(\"Mean OOF Score :\",np.mean(oof_scores))\n",
    "final_preds = np.mean(np.array(oof_preds),axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0136</td>\n",
       "      <td>0.0132</td>\n",
       "      <td>0.8252</td>\n",
       "      <td>0.1480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0076</td>\n",
       "      <td>0.0128</td>\n",
       "      <td>0.0312</td>\n",
       "      <td>0.9484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0104</td>\n",
       "      <td>0.0200</td>\n",
       "      <td>0.0364</td>\n",
       "      <td>0.9332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0028</td>\n",
       "      <td>0.0156</td>\n",
       "      <td>0.0496</td>\n",
       "      <td>0.9320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0008</td>\n",
       "      <td>0.0008</td>\n",
       "      <td>0.9984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12135</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0072</td>\n",
       "      <td>0.0136</td>\n",
       "      <td>0.0548</td>\n",
       "      <td>0.9244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12136</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0020</td>\n",
       "      <td>0.0056</td>\n",
       "      <td>0.9924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12137</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0036</td>\n",
       "      <td>0.0012</td>\n",
       "      <td>0.0132</td>\n",
       "      <td>0.9820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12138</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>1.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12139</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0248</td>\n",
       "      <td>0.0204</td>\n",
       "      <td>0.2788</td>\n",
       "      <td>0.6760</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12140 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0       1       2       3       4\n",
       "0      0.0  0.0136  0.0132  0.8252  0.1480\n",
       "1      0.0  0.0076  0.0128  0.0312  0.9484\n",
       "2      0.0  0.0104  0.0200  0.0364  0.9332\n",
       "3      0.0  0.0028  0.0156  0.0496  0.9320\n",
       "4      0.0  0.0000  0.0008  0.0008  0.9984\n",
       "...    ...     ...     ...     ...     ...\n",
       "12135  0.0  0.0072  0.0136  0.0548  0.9244\n",
       "12136  0.0  0.0000  0.0020  0.0056  0.9924\n",
       "12137  0.0  0.0036  0.0012  0.0132  0.9820\n",
       "12138  0.0  0.0000  0.0000  0.0000  1.0000\n",
       "12139  0.0  0.0248  0.0204  0.2788  0.6760\n",
       "\n",
       "[12140 rows x 5 columns]"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission = pd.DataFrame(final_preds)\n",
    "submission.to_csv(\"submission4.csv\",index=None)\n",
    "submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0136</td>\n",
       "      <td>0.0132</td>\n",
       "      <td>0.8252</td>\n",
       "      <td>0.1480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0076</td>\n",
       "      <td>0.0128</td>\n",
       "      <td>0.0312</td>\n",
       "      <td>0.9484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0104</td>\n",
       "      <td>0.0200</td>\n",
       "      <td>0.0364</td>\n",
       "      <td>0.9332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0028</td>\n",
       "      <td>0.0156</td>\n",
       "      <td>0.0496</td>\n",
       "      <td>0.9320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0008</td>\n",
       "      <td>0.0008</td>\n",
       "      <td>0.9984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12135</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0072</td>\n",
       "      <td>0.0136</td>\n",
       "      <td>0.0548</td>\n",
       "      <td>0.9244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12136</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0020</td>\n",
       "      <td>0.0056</td>\n",
       "      <td>0.9924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12137</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0036</td>\n",
       "      <td>0.0012</td>\n",
       "      <td>0.0132</td>\n",
       "      <td>0.9820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12138</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>1.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12139</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0248</td>\n",
       "      <td>0.0204</td>\n",
       "      <td>0.2788</td>\n",
       "      <td>0.6760</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12140 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0       1       2       3       4\n",
       "0      0.0  0.0136  0.0132  0.8252  0.1480\n",
       "1      0.0  0.0076  0.0128  0.0312  0.9484\n",
       "2      0.0  0.0104  0.0200  0.0364  0.9332\n",
       "3      0.0  0.0028  0.0156  0.0496  0.9320\n",
       "4      0.0  0.0000  0.0008  0.0008  0.9984\n",
       "...    ...     ...     ...     ...     ...\n",
       "12135  0.0  0.0072  0.0136  0.0548  0.9244\n",
       "12136  0.0  0.0000  0.0020  0.0056  0.9924\n",
       "12137  0.0  0.0036  0.0012  0.0132  0.9820\n",
       "12138  0.0  0.0000  0.0000  0.0000  1.0000\n",
       "12139  0.0  0.0248  0.0204  0.2788  0.6760\n",
       "\n",
       "[12140 rows x 5 columns]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission = pd.DataFrame(final_preds)\n",
    "submission.to_csv(\"submission3.csv\",index=None)\n",
    "submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.6844"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission[0].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}