cheat2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "oldsysstdout = sys.stdout\n",
    "class flushfile():\n",
    "    def __init__(self, f):\n",
    "        self.f = f\n",
    "    def __getattr__(self,name): \n",
    "        return object.__getattribute__(self.f, name)\n",
    "    def write(self, x):\n",
    "        self.f.write(x)\n",
    "        self.f.flush()\n",
    "    def flush(self):\n",
    "        self.f.flush()\n",
    "sys.stdout = flushfile(sys.stdout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "import numpy as np\n",
    "import csv\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn import ensemble\n",
    "from itertools import product\n",
    "\n",
    "def find_delimiter(df, col):\n",
    "    \"\"\"\n",
    "    Function that trying to find an approximate delimiter used for scaling.\n",
    "    So we can undo the feature scaling.\n",
    "    \"\"\"\n",
    "    vals = df[col].dropna().sort_values().round(8)\n",
    "    vals = pd.rolling_apply(vals, 2, lambda x: x[1] - x[0])\n",
    "    vals = vals[vals > 0.000001]\n",
    "    return vals.value_counts().idxmax() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "from hyperopt import hp, fmin, tpe\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.cross_validation import train_test_split, StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"data/train.csv\")\n",
    "target = train['target'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "newtrain, X_test, newtarget, y_test = train_test_split(train, target, test_size=.75, stratify=target,\n",
    "                                                    random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(newtrain, newtarget, test_size=.2, stratify=newtarget,\n",
    "                                                    random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def baseline(X_train, X_test, y_train, y_test):\n",
    "    X_train = X_train.drop(['ID','target','v8','v23','v25','v31','v36','v37','v46',\n",
    "                        'v51','v53','v54','v63','v73','v75','v79','v81','v82',\n",
    "                        'v89','v92','v95','v105','v107','v108','v109','v110',\n",
    "                        'v116','v117','v118','v119','v123','v124','v128'],axis=1)\n",
    "    X_test = X_test.drop(['ID','target','v8','v23','v25','v31','v36','v37','v46','v51',\n",
    "                          'v53','v54','v63','v73','v75','v79','v81','v82','v89',\n",
    "                          'v92','v95','v105','v107','v108','v109','v110','v116',\n",
    "                          'v117','v118','v119','v123','v124','v128'],axis=1)\n",
    "\n",
    "    for (train_name, train_series), (test_name, test_series) in zip(X_train.iteritems(),X_test.iteritems()):\n",
    "        if train_series.dtype == 'O':\n",
    "            #for objects: factorize\n",
    "            X_train[train_name], tmp_indexer = pd.factorize(X_train[train_name])\n",
    "            X_test[test_name] = tmp_indexer.get_indexer(X_test[test_name])\n",
    "            #but now we have -1 values (NaN)\n",
    "        else:\n",
    "            #for int or float: fill NaN\n",
    "            tmp_len = len(X_train[train_series.isnull()])\n",
    "            if tmp_len>0:\n",
    "                #print \"mean\", train_series.mean()\n",
    "                X_train.loc[train_series.isnull(), train_name] = -999\n",
    "            #and Test\n",
    "            tmp_len = len(X_test[test_series.isnull()])\n",
    "            if tmp_len>0:\n",
    "                X_test.loc[test_series.isnull(), test_name] = -999\n",
    "\n",
    "    extc = ExtraTreesClassifier(n_estimators=850,max_features= 60,criterion= 'entropy',min_samples_split= 4,\n",
    "                                max_depth= 40, min_samples_leaf= 2, n_jobs = -1)      \n",
    "\n",
    "    extc.fit(X_train, y_train)\n",
    "    y_pred = extc.predict_proba(X_test)\n",
    "    return log_loss(y_test, y_pred[:, 1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Baseline ET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.469913562479\n",
      "0.473848904516\n",
      "0.47572746921\n",
      "0.467934125021\n",
      "0.478691461967\n",
      "mean 0.473223104639\n"
     ]
    }
   ],
   "source": [
    "skf = StratifiedKFold(newtarget, n_folds=5, random_state=42)\n",
    "res = []\n",
    "for train_index, test_index in skf:\n",
    "    X_train, X_test = newtrain.iloc[train_index].copy(), newtrain.iloc[test_index].copy()\n",
    "    y_train, y_test = newtarget[train_index], newtarget[test_index]\n",
    "    r = baseline(X_train, X_test, y_train, y_test)\n",
    "    print(r)\n",
    "    res.append(r)\n",
    "print('mean', np.mean(res))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.        ,  0.15322361,  1.01088794, -6.2033558 ],\n",
       "       [        nan,         nan,         nan,         nan],\n",
       "       [        nan,         nan,         nan,         nan],\n",
       "       ..., \n",
       "       [        nan,         nan,         nan,         nan],\n",
       "       [ 0.        , -6.02684484, -3.09924293, -6.48420617],\n",
       "       [ 0.        , -4.79081156, -1.19054852, -5.29813207]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[['v1']].values - X_train[['v1', 'v2', 'v4', 'v5']].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def xgbtest(X_train, X_test, y_train, y_test):\n",
    "    X_train = X_train.drop(['ID','target','v8','v23','v25','v31','v36','v37',\n",
    "                        'v46','v51','v53','v54','v63','v73','v75','v79',\n",
    "                        'v81','v82','v89','v92','v95','v105','v107','v108',\n",
    "                        'v109','v110','v116','v117','v118','v119','v123','v124','v128'],axis=1)\n",
    "    X_test = X_test.drop(['ID','target', 'v8','v23','v25','v31','v36','v37','v46','v51',\n",
    "                      'v53','v54','v63','v73','v75','v79','v81','v82','v89',\n",
    "                      'v92','v95','v105','v107','v108','v109','v110','v116',\n",
    "                      'v117','v118','v119','v123','v124','v128'],axis=1)\n",
    "    num_vars = ['v1', 'v2', 'v4', 'v5', 'v6', 'v7', 'v9', 'v10', 'v11',\n",
    "                'v12', 'v13', 'v14', 'v15', 'v16', 'v17', 'v18', 'v19', 'v20',\n",
    "                'v21', 'v26', 'v27', 'v28', 'v29', 'v32', 'v33', 'v34', 'v35', 'v38',\n",
    "                'v39', 'v40', 'v41', 'v42', 'v43', 'v44', 'v45', 'v48', 'v49', 'v50',\n",
    "                'v55', 'v57', 'v58', 'v59', 'v60', 'v61', 'v62', 'v64', 'v65', 'v67',\n",
    "                'v68', 'v69', 'v70', 'v72', 'v76', 'v77', 'v78', 'v80', 'v83', 'v84', \n",
    "                'v85', 'v86', 'v87', 'v88', 'v90', 'v93', 'v94', 'v96', 'v97', 'v98', \n",
    "                'v99', 'v100', 'v101', 'v102', 'v103', 'v104', 'v106', 'v111', 'v114',\n",
    "                'v115', 'v120', 'v121', 'v122', 'v126', 'v127', 'v129', 'v130', 'v131']\n",
    "\n",
    "    dropcols = []\n",
    "    floatcols = []\n",
    "    traincat = np.zeros((X_train.shape[0], 1))\n",
    "    testcat = np.zeros((X_test.shape[0], 1))\n",
    "\n",
    "    skfi = StratifiedKFold(y_train, n_folds=5, random_state=42)\n",
    "\n",
    "    vs = pd.concat([X_train, X_test])\n",
    "    for c in num_vars:\n",
    "        if c not in X_train.columns:\n",
    "            continue\n",
    "\n",
    "        X_train.loc[X_train[c].round(5) == 0, c] = 0\n",
    "        X_test.loc[X_test[c].round(5) == 0, c] = 0\n",
    "\n",
    "        delimiter = find_delimiter(vs, c)\n",
    "        X_train[c] *= 1/delimiter\n",
    "        X_test[c] *= 1/delimiter\n",
    "\n",
    "    for (train_name, train_series), (test_name, test_series) in zip(X_train.iteritems(), X_test.iteritems()):\n",
    "        if train_series.dtype == 'O' or train_series.dtype == 'int64':\n",
    "            dropcols.append(train_name)\n",
    "            if train_name != 'v22':\n",
    "                X_train.loc[:, train_name+'c'] = np.zeros(X_train.shape[0])\n",
    "                for train_index, test_index in skfi:\n",
    "                    X_X_train = X_train.loc[train.index[train_index], [train_name, 'target']]\n",
    "                    d = X_X_train.groupby(by=train_name).mean()\n",
    "                    predvec = X_train.loc[X_train.index[test_index], train_name]\n",
    "                    predvec = predvec.map(dict([(i, d.loc[i, 'target']) for i in d.index]))\n",
    "                    X_train.loc[X_train.index[test_index], train_name+'c'] = predvec.values\n",
    "                predvec = X_test.loc[:, train_name]\n",
    "                predvec = predvec.map(dict([(i, d.loc[i, 'target']) for i in d.index]))\n",
    "                X_test.loc[:, train_name+'c'] = predvec.values\n",
    "\n",
    "                X_train[train_name], tmp_indexer = pd.factorize(X_train[train_name])\n",
    "                X_test[test_name] = tmp_indexer.get_indexer(X_test[test_name])\n",
    "                ohe = OneHotEncoder()\n",
    "                tr = ohe.fit_transform(X_train[train_name].values.reshape(-1, 1) + 1).toarray()\n",
    "                ts = ohe.transform(X_test[test_name].values.reshape(-1, 1) + 1).toarray()\n",
    "            else:\n",
    "                X_train[train_name], tmp_indexer = pd.factorize(X_train[train_name])\n",
    "                X_test[test_name] = tmp_indexer.get_indexer(X_test[test_name])\n",
    "                a, b = pd.factorize(X_train[train_name].value_counts()[X_train[train_name].value_counts() > 100].index)\n",
    "                trv = b.get_indexer(X_train[train_name])\n",
    "                tsv = b.get_indexer(X_test[test_name])\n",
    "                ohe = OneHotEncoder()\n",
    "                tr = ohe.fit_transform(trv.reshape(-1, 1) + 1).toarray()\n",
    "                ts = ohe.transform(tsv.reshape(-1, 1) + 1).toarray()\n",
    "                tr = np.hstack((X_train[train_name].values.reshape(-1, 1), tr))\n",
    "                ts = np.hstack((X_test[test_name].values.reshape(-1, 1), ts))\n",
    "\n",
    "            traincat = np.hstack((traincat, tr))\n",
    "            testcat = np.hstack((testcat, ts))\n",
    "\n",
    "        else:\n",
    "            floatcols.append(train_name)\n",
    "            #for int or float: fill NaN\n",
    "            tmp_len = len(X_train[train_series.isnull()])\n",
    "            if tmp_len>0:\n",
    "                #print \"mean\", train_series.mean()\n",
    "                X_train.loc[train_series.isnull(), train_name] = -1\n",
    "            #and Test\n",
    "            tmp_len = len(X_test[test_series.isnull()])\n",
    "            if tmp_len>0:\n",
    "                X_test.loc[test_series.isnull(), test_name] = -1\n",
    "\n",
    "    pca = PCA(n_components=10)\n",
    "    trainpca = pca.fit_transform(X_train[floatcols].values)\n",
    "    testpca = pca.transform(X_test[floatcols].values)\n",
    "\n",
    "    for i in range(len(floatcols)):\n",
    "        for j in range(i, len(floatcols)):\n",
    "            X_train.loc[:, floatcols[i]+'-'+floatcols[j]] = X_train.loc[:, floatcols[i]] - X_train.loc[:, floatcols[j]]\n",
    "            X_train.loc[:, floatcols[i]+'*'+floatcols[j]] = X_train.loc[:, floatcols[i]] * X_train.loc[:, floatcols[j]]\n",
    "            X_train.loc[:, floatcols[i]+'/'+floatcols[j]] = X_train.loc[:, floatcols[i]] / X_train.loc[:, floatcols[j]]\n",
    "            X_test.loc[:, floatcols[i]+'-'+floatcols[j]] = X_test.loc[:, floatcols[i]] - X_test.loc[:, floatcols[j]]\n",
    "            X_test.loc[:, floatcols[i]+'*'+floatcols[j]] = X_test.loc[:, floatcols[i]] * X_test.loc[:, floatcols[j]]\n",
    "            X_test.loc[:, floatcols[i]+'/'+floatcols[j]] = X_test.loc[:, floatcols[i]] / X_test.loc[:, floatcols[j]]\n",
    "        print(floatcols[i])\n",
    "        \n",
    "    X_train_f = np.hstack((X_train[X_train.columns.difference(dropcols)].values, traincat, trainpca))\n",
    "    X_test_f = np.hstack((X_test[X_test.columns.difference(dropcols)].values, testcat, testpca))\n",
    "\n",
    "    dtrain = xgb.DMatrix(X_train_f, label=y_train)\n",
    "    dval = xgb.DMatrix(X_test_f, label=y_test)\n",
    "\n",
    "    watchlist = ((dtrain, 'train'), (dval, 'val'))\n",
    "\n",
    "    params = {\n",
    "        'nthread': 12,\n",
    "        'eta': .01,\n",
    "        'max_depth': 12,\n",
    "        'colsample_bytree': .4,\n",
    "        'subsample': 1.,\n",
    "        'min_child_weight': 4.,\n",
    "        'lambda': 7.,\n",
    "        'alpha': .75,\n",
    "        'seed': 0,\n",
    "        'objective': 'binary:logistic',\n",
    "        'eval_metric': 'logloss',\n",
    "        'silent': 1\n",
    "    }\n",
    "\n",
    "    gbm = xgb.train(params, dtrain, num_boost_round=3000, early_stopping_rounds=300, evals=watchlist,\n",
    "              verbose_eval=True)\n",
    "    pred = gbm.predict(dval, ntree_limit=gbm.best_iteration)\n",
    "    return log_loss(y_test, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/libfun/vrt/lib/python3.4/site-packages/ipykernel/__main__.py:16: FutureWarning: pd.rolling_apply is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=2).apply(func=<function>,args=<tuple>,kwargs=<dict>)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "v1\n",
      "v2\n",
      "v4\n",
      "v5\n",
      "v6\n",
      "v7\n",
      "v9\n",
      "v10\n",
      "v11\n",
      "v12\n",
      "v13\n",
      "v14\n",
      "v15\n",
      "v16\n",
      "v17\n",
      "v18\n",
      "v19\n",
      "v20\n",
      "v21\n",
      "v26\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-11-4e32279e127e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mxgbtest\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-10-86a43f840350>\u001b[0m in \u001b[0;36mxgbtest\u001b[1;34m(X_train, X_test, y_train, y_test)\u001b[0m\n\u001b[0;32m     95\u001b[0m             \u001b[0mX_test\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfloatcols\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;34m'-'\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mfloatcols\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mj\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX_test\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfloatcols\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mX_test\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfloatcols\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mj\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     96\u001b[0m             \u001b[0mX_test\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfloatcols\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;34m'*'\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mfloatcols\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mj\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX_test\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfloatcols\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mX_test\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfloatcols\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mj\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 97\u001b[1;33m             \u001b[0mX_test\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfloatcols\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;34m'/'\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mfloatcols\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mj\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX_test\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfloatcols\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m/\u001b[0m \u001b[0mX_test\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfloatcols\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mj\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     98\u001b[0m         \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfloatcols\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     99\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt/lib/python3.4/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m__setitem__\u001b[1;34m(self, key, value)\u001b[0m\n\u001b[0;32m    126\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__setitem__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    127\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_setitem_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 128\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_setitem_with_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    129\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    130\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_has_valid_type\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt/lib/python3.4/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_setitem_with_indexer\u001b[1;34m(self, indexer, value)\u001b[0m\n\u001b[0;32m    286\u001b[0m                         new_indexer = convert_from_missing_indexer_tuple(\n\u001b[0;32m    287\u001b[0m                             indexer, self.obj.axes)\n\u001b[1;32m--> 288\u001b[1;33m                         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_setitem_with_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnew_indexer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    289\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    290\u001b[0m                         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt/lib/python3.4/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_setitem_with_indexer\u001b[1;34m(self, indexer, value)\u001b[0m\n\u001b[0;32m    227\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    228\u001b[0m         \u001b[1;31m# maybe partial set\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 229\u001b[1;33m         \u001b[0mtake_split_path\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_is_mixed_type\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    230\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    231\u001b[0m         \u001b[1;31m# if there is only one block/type, still have to take split path\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt/lib/python3.4/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m_is_mixed_type\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   2751\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_is_mixed_type\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2752\u001b[0m         \u001b[0mf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mlambda\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_mixed_type\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2753\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_protect_consolidate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2754\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2755\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt/lib/python3.4/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m_protect_consolidate\u001b[1;34m(self, f)\u001b[0m\n\u001b[0;32m   2713\u001b[0m         \"\"\"\n\u001b[0;32m   2714\u001b[0m         \u001b[0mblocks_before\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mblocks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2715\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2716\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mblocks\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mblocks_before\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2717\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_clear_item_cache\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt/lib/python3.4/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m()\u001b[0m\n\u001b[0;32m   2750\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2751\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_is_mixed_type\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2752\u001b[1;33m         \u001b[0mf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mlambda\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_mixed_type\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2753\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_protect_consolidate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2754\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt/lib/python3.4/site-packages/pandas/core/internals.py\u001b[0m in \u001b[0;36mis_mixed_type\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   2945\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mis_mixed_type\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2946\u001b[0m         \u001b[1;31m# Warning, consolidation needs to get checked upstairs\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2947\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_consolidate_inplace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2948\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mblocks\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2949\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt/lib/python3.4/site-packages/pandas/core/internals.py\u001b[0m in \u001b[0;36m_consolidate_inplace\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   3211\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_consolidate_inplace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3212\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_consolidated\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3213\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mblocks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtuple\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_consolidate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mblocks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3214\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_is_consolidated\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3215\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_known_consolidated\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt/lib/python3.4/site-packages/pandas/core/internals.py\u001b[0m in \u001b[0;36m_consolidate\u001b[1;34m(blocks)\u001b[0m\n\u001b[0;32m   4196\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0m_can_consolidate\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgroup_blocks\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mgrouper\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4197\u001b[0m         merged_blocks = _merge_blocks(list(group_blocks), dtype=dtype,\n\u001b[1;32m-> 4198\u001b[1;33m                                       _can_consolidate=_can_consolidate)\n\u001b[0m\u001b[0;32m   4199\u001b[0m         \u001b[0mnew_blocks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_extend_blocks\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmerged_blocks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnew_blocks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4200\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mnew_blocks\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/libfun/vrt/lib/python3.4/site-packages/pandas/core/internals.py\u001b[0m in \u001b[0;36m_merge_blocks\u001b[1;34m(blocks, dtype, _can_consolidate)\u001b[0m\n\u001b[0;32m   4219\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4220\u001b[0m         \u001b[0margsort\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margsort\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnew_mgr_locs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 4221\u001b[1;33m         \u001b[0mnew_values\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnew_values\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0margsort\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   4222\u001b[0m         \u001b[0mnew_mgr_locs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnew_mgr_locs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0margsort\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4223\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "xgbtest(X_train, X_test, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['v1', 'v2', 'v3', 'v4', 'v5', 'v6', 'v7', 'v9', 'v10', 'v11',\n",
       "       ...\n",
       "       'v106/v80', 'v9-v9', 'v9*v9', 'v9/v9', 'v9-v80', 'v9*v80', 'v9/v80',\n",
       "       'v80-v80', 'v80*v80', 'v80/v80'],\n",
       "      dtype='object', length=248)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train_f[np.isnan(X_train_f)] = -999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test_f[np.isnan(X_test_f)] = -999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import log_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.475724395496\n"
     ]
    }
   ],
   "source": [
    "extc = ExtraTreesClassifier(n_estimators=2850,max_features= 100,criterion= 'entropy',min_samples_split= 4,\n",
    "                            max_depth= 40, min_samples_leaf= 2, n_jobs = -1)      \n",
    " \n",
    "extc.fit(X_train_f, y_train)\n",
    "preds = extc.predict_proba(X_test_f)\n",
    "print(log_loss(y_test, preds[:, 1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "z = X_train.columns.difference(dropcols)\n",
    "zz = extc.feature_importances_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "lst = [(z[i], zz[i]) for i in range(98)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lst.sort(key=lambda x: x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('v112c', 0.0),\n",
       " ('v113c', 0.0),\n",
       " ('v125c', 0.0),\n",
       " ('v24c', 0.0),\n",
       " ('v30c', 0.0),\n",
       " ('v3c', 0.0),\n",
       " ('v47c', 0.0),\n",
       " ('v52c', 0.0),\n",
       " ('v56c', 0.0),\n",
       " ('v66c', 0.0),\n",
       " ('v71c', 0.0),\n",
       " ('v74c', 0.0),\n",
       " ('v91c', 0.0),\n",
       " ('v38', 0.001900790083705275),\n",
       " ('v65', 0.0033821642288781126),\n",
       " ('v20', 0.003418688414782641),\n",
       " ('v67', 0.0034806137619644018),\n",
       " ('v29', 0.0034892562170432753),\n",
       " ('v41', 0.0034961417074067156),\n",
       " ('v48', 0.0034972356023415213),\n",
       " ('v96', 0.0035643468489207041),\n",
       " ('v49', 0.003573162943010951),\n",
       " ('v61', 0.0036088682663659648),\n",
       " ('v42', 0.0036223029859698331),\n",
       " ('v106', 0.0036223612460628799),\n",
       " ('v11', 0.0036262430938946381),\n",
       " ('v77', 0.0036761157102065587),\n",
       " ('v93', 0.0036868323844363384),\n",
       " ('v68', 0.0037163850259029447),\n",
       " ('v64', 0.0037400838643919742),\n",
       " ('v17', 0.0037459769886174958),\n",
       " ('v76', 0.0037626594425400912),\n",
       " ('v19', 0.0038269384288362762),\n",
       " ('v13', 0.0038358818575447567),\n",
       " ('v104', 0.0038526945640091894),\n",
       " ('v94', 0.0038735666128595922),\n",
       " ('v59', 0.0039376260799359377),\n",
       " ('v44', 0.0039411043973116618),\n",
       " ('v33', 0.0039806761761985963),\n",
       " ('v126', 0.0040169859209530012),\n",
       " ('v4', 0.0040312016924595106),\n",
       " ('v35', 0.0040801354531853698),\n",
       " ('v90', 0.0040805285021656449),\n",
       " ('v103', 0.0041039398178090566),\n",
       " ('v130', 0.0041133577756419125),\n",
       " ('v101', 0.0041236182541913285),\n",
       " ('v32', 0.0041306198838194433),\n",
       " ('v121', 0.004134126030077347),\n",
       " ('v83', 0.0041462216284525013),\n",
       " ('v7', 0.0041724923582968166),\n",
       " ('v57', 0.0041841746082345518),\n",
       " ('v78', 0.0041867182677799509),\n",
       " ('v26', 0.0042184124186981191),\n",
       " ('v97', 0.0042224127283301098),\n",
       " ('v86', 0.0042349075763619204),\n",
       " ('v15', 0.0042413133233841199),\n",
       " ('v60', 0.0042543629411048531),\n",
       " ('v9', 0.0042671965513735501),\n",
       " ('v55', 0.0042759496415134771),\n",
       " ('v43', 0.0042768684115403422),\n",
       " ('v27', 0.0042830978661859576),\n",
       " ('v111', 0.0042995465313877285),\n",
       " ('v115', 0.0043469429938266428),\n",
       " ('v122', 0.0044191346628433903),\n",
       " ('v18', 0.004428929589947098),\n",
       " ('v39', 0.0044410132777916054),\n",
       " ('v69', 0.004453060613687069),\n",
       " ('v2', 0.0044545913966719444),\n",
       " ('v102', 0.0044585902278693641),\n",
       " ('v45', 0.0044716266508135049),\n",
       " ('v84', 0.0044842689807074705),\n",
       " ('v6', 0.0045291441979946057),\n",
       " ('v85', 0.0046756267801906955),\n",
       " ('v80', 0.004746097308938579),\n",
       " ('v99', 0.0047828202361511769),\n",
       " ('v127', 0.004784468355033965),\n",
       " ('v16', 0.0048044898260538177),\n",
       " ('v28', 0.0048800245411698209),\n",
       " ('v88', 0.0048894219162102341),\n",
       " ('v5', 0.0050324762794818909),\n",
       " ('v70', 0.0050616963985943035),\n",
       " ('v131', 0.0051277490235826402),\n",
       " ('v120', 0.005167875723030655),\n",
       " ('v1', 0.0052413081158700576),\n",
       " ('v58', 0.0053345149071179305),\n",
       " ('v87', 0.0053372029870451115),\n",
       " ('v98', 0.0053396991978170456),\n",
       " ('v100', 0.0053402360121418795),\n",
       " ('v129', 0.0082489482514040077),\n",
       " ('v72', 0.0086750680961755339),\n",
       " ('v62', 0.012183287900995918),\n",
       " ('v21', 0.012931302929067971),\n",
       " ('v34', 0.014601596220077399),\n",
       " ('v114', 0.015906816382493728),\n",
       " ('v14', 0.016185788958852261),\n",
       " ('v40', 0.016431982023853729),\n",
       " ('v10', 0.020266664081646284),\n",
       " ('v12', 0.02080711873840085)]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  5.24130812e-03,   2.02666641e-02,   5.34023601e-03,\n",
       "         4.12361825e-03,   4.45859023e-03,   4.10393982e-03,\n",
       "         3.85269456e-03,   3.62236125e-03,   3.62624309e-03,\n",
       "         4.29954653e-03,   0.00000000e+00,   0.00000000e+00,\n",
       "         1.59068164e-02,   4.34694299e-03,   2.08071187e-02,\n",
       "         5.16787572e-03,   4.13412603e-03,   4.41913466e-03,\n",
       "         0.00000000e+00,   4.01698592e-03,   4.78446836e-03,\n",
       "         8.24894825e-03,   3.83588186e-03,   4.11335778e-03,\n",
       "         5.12774902e-03,   1.61857890e-02,   4.24131332e-03,\n",
       "         4.80448983e-03,   3.74597699e-03,   4.42892959e-03,\n",
       "         3.82693843e-03,   4.45459140e-03,   3.41868841e-03,\n",
       "         1.29313029e-02,   0.00000000e+00,   4.21841242e-03,\n",
       "         4.28309787e-03,   4.88002454e-03,   3.48925622e-03,\n",
       "         0.00000000e+00,   4.13061988e-03,   3.98067618e-03,\n",
       "         1.46015962e-02,   4.08013545e-03,   1.90079008e-03,\n",
       "         4.44101328e-03,   0.00000000e+00,   4.03120169e-03,\n",
       "         1.64319820e-02,   3.49614171e-03,   3.62230299e-03,\n",
       "         4.27686841e-03,   3.94110440e-03,   4.47162665e-03,\n",
       "         0.00000000e+00,   3.49723560e-03,   3.57316294e-03,\n",
       "         5.03247628e-03,   0.00000000e+00,   4.27594964e-03,\n",
       "         0.00000000e+00,   4.18417461e-03,   5.33451491e-03,\n",
       "         3.93762608e-03,   4.52914420e-03,   4.25436294e-03,\n",
       "         3.60886827e-03,   1.21832879e-02,   3.74008386e-03,\n",
       "         3.38216423e-03,   0.00000000e+00,   3.48061376e-03,\n",
       "         3.71638503e-03,   4.45306061e-03,   4.17249236e-03,\n",
       "         5.06169640e-03,   0.00000000e+00,   8.67506810e-03,\n",
       "         0.00000000e+00,   3.76265944e-03,   3.67611571e-03,\n",
       "         4.18671827e-03,   4.74609731e-03,   4.14622163e-03,\n",
       "         4.48426898e-03,   4.67562678e-03,   4.23490758e-03,\n",
       "         5.33720299e-03,   4.88942192e-03,   4.26719655e-03,\n",
       "         4.08052850e-03,   0.00000000e+00,   3.68683238e-03,\n",
       "         3.87356661e-03,   3.56434685e-03,   4.22241273e-03,\n",
       "         5.33969920e-03,   4.78282024e-03,   0.00000000e+00,\n",
       "         8.30452774e-04,   8.68334621e-04,   7.47481894e-05,\n",
       "         4.89975875e-05,   1.26386465e-02,   6.49860859e-03,\n",
       "         1.17681727e-03,   1.28098591e-03,   5.72215521e-04,\n",
       "         4.13607711e-04,   3.38958295e-04,   2.68012399e-04,\n",
       "         3.70791249e-04,   2.31031798e-04,   2.88963392e-04,\n",
       "         1.68226212e-04,   2.48813243e-04,   2.46364241e-04,\n",
       "         1.74051178e-04,   1.39805215e-04,   1.78854478e-04,\n",
       "         1.64353585e-04,   2.84292312e-04,   7.53348105e-05,\n",
       "         1.83543049e-04,   7.74080444e-05,   1.93312735e-04,\n",
       "         2.41722380e-04,   1.29813174e-04,   1.11168768e-04,\n",
       "         1.18335917e-04,   1.07809965e-04,   2.57047026e-04,\n",
       "         2.16569909e-04,   1.22392303e-04,   8.29581742e-05,\n",
       "         7.39161546e-05,   1.75061292e-04,   8.14329438e-05,\n",
       "         1.55887938e-04,   1.15364745e-04,   6.77338459e-05,\n",
       "         1.08887310e-04,   1.04256307e-04,   1.26225512e-04,\n",
       "         1.10736029e-04,   7.26966158e-05,   1.40473077e-04,\n",
       "         1.19769610e-04,   9.90942571e-05,   8.28238615e-05,\n",
       "         9.81659823e-05,   1.79244154e-04,   1.77822058e-04,\n",
       "         1.63620500e-04,   1.01060471e-04,   8.91633970e-05,\n",
       "         1.36050898e-04,   9.06307879e-05,   6.84965321e-05,\n",
       "         7.19107128e-05,   1.51943350e-04,   1.45199925e-04,\n",
       "         1.64235579e-04,   2.36383264e-04,   1.30847105e-04,\n",
       "         7.95915010e-03,   6.16356446e-03,   7.04564032e-03,\n",
       "         3.78111239e-03,   2.19331008e-03,   8.63754933e-03,\n",
       "         7.66326975e-03,   3.83525745e-03,   1.33556563e-03,\n",
       "         1.57992001e-03,   2.97652713e-03,   1.66189563e-03,\n",
       "         9.61195798e-05,   1.80801095e-02,   1.48014846e-03,\n",
       "         4.90926321e-03,   2.18444067e-03,   1.31266697e-03,\n",
       "         6.87475556e-04,   1.21163179e-03,   2.29471905e-05,\n",
       "         2.15313825e-05,   0.00000000e+00,   0.00000000e+00,\n",
       "         5.59571079e-03,   5.21900587e-03,   5.69364555e-03,\n",
       "         5.36164999e-03,   5.49528470e-03,   5.50826922e-03,\n",
       "         5.91846944e-03,   5.49819279e-03,   5.30141502e-03,\n",
       "         5.36789560e-03,   5.01429837e-03,   5.67851384e-03,\n",
       "         2.39061598e-03,   3.86599990e-03,   7.86963930e-04,\n",
       "         5.25512417e-03,   3.40647151e-03,   1.44373470e-03,\n",
       "         4.79869738e-03,   1.91036071e-03,   1.16993671e-03,\n",
       "         1.71981616e-03,   2.30135258e-03,   2.72615205e-04,\n",
       "         9.39492826e-05,   2.52811187e-03,   9.87207205e-05,\n",
       "         4.21225864e-05,   2.35432739e-03,   9.78981964e-04,\n",
       "         4.49666853e-04,   6.81572310e-04,   9.29849442e-04,\n",
       "         7.62901763e-04,   3.77206219e-04,   7.93220578e-04,\n",
       "         8.02505407e-04,   1.41231708e-03,   4.03522448e-04,\n",
       "         1.65783631e-04,   3.19744929e-03,   3.80025425e-03,\n",
       "         2.62531230e-03,   1.62053158e-04,   2.19051138e-04,\n",
       "         1.11518984e-03,   2.50027595e-03,   6.84411801e-03,\n",
       "         6.93838144e-04,   9.96907730e-04,   3.55171558e-04,\n",
       "         1.32221469e-03,   9.74329037e-04,   2.81281341e-04,\n",
       "         9.76594699e-06,   7.16570760e-04,   1.40350239e-04,\n",
       "         9.09123379e-04,   5.83849068e-05,   3.67315107e-04,\n",
       "         8.46513236e-05,   4.46170728e-05,   1.83282897e-04,\n",
       "         2.42076168e-04,   0.00000000e+00,   1.01532002e-06,\n",
       "         8.36908920e-05,   2.60854192e-05,   2.62340776e-05,\n",
       "         7.81622807e-05,   1.91123752e-05,   5.01635277e-05,\n",
       "         8.31509705e-05,   8.82401587e-05,   3.57169278e-05,\n",
       "         1.10003268e-04,   3.35810459e-06,   0.00000000e+00,\n",
       "         0.00000000e+00,   8.87294456e-06,   9.28174194e-05,\n",
       "         4.57390159e-05,   5.68863439e-07,   3.92428119e-05,\n",
       "         1.68522889e-06,   0.00000000e+00,   4.38594411e-06,\n",
       "         4.49755383e-05,   9.17187441e-06,   9.75712408e-08,\n",
       "         3.70345560e-05,   6.87374336e-05,   1.19175974e-05,\n",
       "         2.34376902e-05,   7.74954369e-07,   6.07399599e-07,\n",
       "         0.00000000e+00,   3.69506576e-06,   3.25124570e-06,\n",
       "         0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         3.50270356e-07,   0.00000000e+00,   4.71685376e-07,\n",
       "         5.77493597e-09,   0.00000000e+00,   1.65439654e-06,\n",
       "         0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         8.73048184e-09,   0.00000000e+00,   0.00000000e+00,\n",
       "         0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         0.00000000e+00,   4.61072902e-08,   0.00000000e+00,\n",
       "         0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         0.00000000e+00,   0.00000000e+00,   7.47400772e-03,\n",
       "         8.49066787e-03,   1.70421654e-02,   7.99381091e-03,\n",
       "         8.83685246e-03,   3.90089945e-03,   2.93470941e-06,\n",
       "         0.00000000e+00,   7.27328602e-07,   0.00000000e+00,\n",
       "         0.00000000e+00,   5.66358188e-04,   5.14907364e-04,\n",
       "         2.59712557e-05,   0.00000000e+00,   1.26503566e-04,\n",
       "         9.00180115e-03,   9.91908105e-03,   6.32627536e-03,\n",
       "         9.12059932e-03,   9.48227842e-03,   1.88616205e-03,\n",
       "         3.15135822e-04,   2.33034870e-03,   4.54486073e-03,\n",
       "         2.89812967e-03,   3.95048516e-03,   2.51051534e-03,\n",
       "         2.69718668e-03,   2.37191033e-03,   1.05265670e-03,\n",
       "         3.68945070e-03,   2.89741692e-03,   7.13389232e-03,\n",
       "         3.18288156e-03,   4.41315966e-03,   1.27986362e-03,\n",
       "         2.45160950e-03,   1.75886566e-03,   1.45915391e-03,\n",
       "         2.20678343e-03,   2.56848888e-03,   1.23146885e-03,\n",
       "         7.45464672e-04,   4.40949496e-04,   1.08726421e-02,\n",
       "         5.17326472e-03,   3.02733741e-03,   1.87454017e-03,\n",
       "         5.97919538e-04,   2.07106662e-03,   1.39242209e-03,\n",
       "         1.98602763e-04,   6.56461875e-04,   1.77070558e-03,\n",
       "         1.15282917e-03,   5.59520139e-04,   7.90040086e-04,\n",
       "         1.01599416e-03,   5.30356797e-04,   2.78923491e-04,\n",
       "         1.35940942e-03,   9.71912632e-04,   8.12721362e-04,\n",
       "         2.78440768e-04,   3.53583954e-04,   4.74911614e-04,\n",
       "         7.14623199e-04,   3.65290042e-04,   1.02494258e-03,\n",
       "         1.58422157e-03,   2.35946484e-04,   5.47400201e-04,\n",
       "         3.66028098e-04,   1.55329623e-04,   6.13143897e-05,\n",
       "         6.26615894e-04,   1.99948473e-04,   6.86468837e-05,\n",
       "         1.67687806e-05,   0.00000000e+00,   4.03195627e-05,\n",
       "         1.54375601e-03,   3.31482357e-03,   7.19428009e-04,\n",
       "         2.66316212e-03,   2.77297637e-03,   1.16731831e-03,\n",
       "         4.44405605e-04,   9.08066977e-04,   7.61120579e-04,\n",
       "         8.16327038e-04,   1.90067833e-03,   1.19969429e-03,\n",
       "         8.09440159e-04,   1.49282065e-03,   1.94555238e-03,\n",
       "         1.31858516e-03,   7.04614561e-04,   2.36235062e-03,\n",
       "         6.23749007e-04,   3.76047473e-04,   9.75414739e-04,\n",
       "         2.15321988e-03,   7.19836969e-04,   2.02814346e-03,\n",
       "         7.87276413e-04,   2.80274081e-03,   2.80491134e-03,\n",
       "         1.82780369e-03,   8.00207309e-04,   7.97436147e-04,\n",
       "         5.72066762e-04,   4.79542060e-04,   1.87734432e-03,\n",
       "         3.03192742e-04,   2.17104653e-03,   1.27242155e-03,\n",
       "         2.30190800e-03,   9.32832045e-04,   8.19286469e-04,\n",
       "         1.79938080e-03,   6.30405867e-04,   5.77361877e-04,\n",
       "         1.16913447e-03,   1.11623934e-03,   5.77923068e-04,\n",
       "         5.64154402e-04,   5.89123080e-04,   4.04826489e-04,\n",
       "         1.59832055e-03,   7.11234377e-04,   3.04883772e-04,\n",
       "         5.34654115e-04,   9.67701265e-04,   4.04079286e-04,\n",
       "         7.72998045e-04,   7.22354040e-04,   2.80883948e-04,\n",
       "         9.52176062e-04,   6.26941475e-05,   5.93108623e-04,\n",
       "         1.00024001e-03,   3.48153694e-04,   9.81663070e-04,\n",
       "         8.42253898e-04,   2.75657431e-04,   1.18533175e-03,\n",
       "         6.99451697e-04,   7.36223801e-04,   3.16733507e-04,\n",
       "         7.13601725e-04,   3.71446445e-04,   4.26388232e-04,\n",
       "         5.72905603e-04,   1.84896122e-04,   1.30160978e-03,\n",
       "         4.44699538e-04,   4.20909404e-04,   3.17308090e-04,\n",
       "         3.22925386e-04,   2.46767863e-04,   2.36119727e-04,\n",
       "         3.57685442e-04,   6.35246972e-04,   2.55815749e-04,\n",
       "         1.80043247e-04,   1.10044934e-04,   2.39818838e-05,\n",
       "         1.51158344e-05,   9.73318052e-07,   0.00000000e+00])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extc.feature_importances_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/libfun/vrt/lib/python3.4/site-packages/ipykernel/__main__.py:16: FutureWarning: pd.rolling_apply is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=2).apply(args=<tuple>,kwargs=<dict>,func=<function>)\n"
     ]
    }
   ],
   "source": [
    "X_train = X_train.drop(['ID','target','v8','v23','v25','v31','v36','v37',\n",
    "                    'v46','v51','v53','v54','v63','v73','v75','v79',\n",
    "                    'v81','v82','v89','v92','v95','v105','v107','v108',\n",
    "                    'v109','v110','v116','v117','v118','v119','v123','v124','v128'],axis=1)\n",
    "X_test = X_test.drop(['ID','target', 'v8','v23','v25','v31','v36','v37','v46','v51',\n",
    "                  'v53','v54','v63','v73','v75','v79','v81','v82','v89',\n",
    "                  'v92','v95','v105','v107','v108','v109','v110','v116',\n",
    "                  'v117','v118','v119','v123','v124','v128'],axis=1)\n",
    "num_vars = ['v1', 'v2', 'v4', 'v5', 'v6', 'v7', 'v9', 'v10', 'v11',\n",
    "            'v12', 'v13', 'v14', 'v15', 'v16', 'v17', 'v18', 'v19', 'v20',\n",
    "            'v21', 'v26', 'v27', 'v28', 'v29', 'v32', 'v33', 'v34', 'v35', 'v38',\n",
    "            'v39', 'v40', 'v41', 'v42', 'v43', 'v44', 'v45', 'v48', 'v49', 'v50',\n",
    "            'v55', 'v57', 'v58', 'v59', 'v60', 'v61', 'v62', 'v64', 'v65', 'v67',\n",
    "            'v68', 'v69', 'v70', 'v72', 'v76', 'v77', 'v78', 'v80', 'v83', 'v84', \n",
    "            'v85', 'v86', 'v87', 'v88', 'v90', 'v93', 'v94', 'v96', 'v97', 'v98', \n",
    "            'v99', 'v100', 'v101', 'v102', 'v103', 'v104', 'v106', 'v111', 'v114',\n",
    "            'v115', 'v120', 'v121', 'v122', 'v126', 'v127', 'v129', 'v130', 'v131']\n",
    "\n",
    "dropcols = []\n",
    "traincat = np.zeros((X_train.shape[0], 1))\n",
    "testcat = np.zeros((X_test.shape[0], 1))\n",
    "\n",
    "vs = pd.concat([X_train, X_test])\n",
    "for c in num_vars:\n",
    "    if c not in X_train.columns:\n",
    "        continue\n",
    "\n",
    "    X_train.loc[X_train[c].round(5) == 0, c] = 0\n",
    "    X_test.loc[X_test[c].round(5) == 0, c] = 0\n",
    "\n",
    "    delimiter = find_delimiter(vs, c)\n",
    "    X_train[c] *= 1/delimiter\n",
    "    X_test[c] *= 1/delimiter\n",
    "\n",
    "for (train_name, train_series), (test_name, test_series) in zip(X_train.iteritems(), X_test.iteritems()):\n",
    "    if train_series.dtype == 'O' or train_series.dtype == 'int64':\n",
    "        dropcols.append(train_name)\n",
    "        if train_name != 'v22':\n",
    "            X_train[train_name], tmp_indexer = pd.factorize(X_train[train_name])\n",
    "            X_test[test_name] = tmp_indexer.get_indexer(X_test[test_name])\n",
    "            ohe = OneHotEncoder()\n",
    "            tr = ohe.fit_transform(X_train[train_name].values.reshape(-1, 1) + 1).toarray()\n",
    "            ts = ohe.transform(X_test[test_name].values.reshape(-1, 1) + 1).toarray()\n",
    "        else:\n",
    "            X_train[train_name], tmp_indexer = pd.factorize(X_train[train_name])\n",
    "            X_test[test_name] = tmp_indexer.get_indexer(X_test[test_name])\n",
    "            a, b = pd.factorize(X_train[train_name].value_counts()[X_train[train_name].value_counts() > 100].index)\n",
    "            trv = b.get_indexer(X_train[train_name])\n",
    "            tsv = b.get_indexer(X_test[test_name])\n",
    "            ohe = OneHotEncoder()\n",
    "            tr = ohe.fit_transform(trv.reshape(-1, 1) + 1).toarray()\n",
    "            ts = ohe.transform(tsv.reshape(-1, 1) + 1).toarray()\n",
    "            tr = np.hstack((X_train[train_name].values.reshape(-1, 1), tr))\n",
    "            ts = np.hstack((X_test[test_name].values.reshape(-1, 1), ts))\n",
    "\n",
    "        traincat = np.hstack((traincat, tr))\n",
    "        testcat = np.hstack((testcat, ts))\n",
    "\n",
    "X_train_f = np.hstack((X_train.values, traincat))\n",
    "X_test_f = np.hstack((X_test.values, testcat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dtrain = xgb.DMatrix(X_train_f, label=y_train)\n",
    "dval = xgb.DMatrix(X_test_f, label=y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "watchlist = ((dtrain, 'train'), (dval, 'val'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import log_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "space = {'colsample_bytree': hp.uniform('colsample_bytree', .3, 1.),\n",
    "         'subsample': hp.uniform('subsample', .3, 1.),\n",
    "         'max_depth': hp.quniform('max_depth', 3, 15, 1),\n",
    "         'min_child_weight': hp.quniform('min_samples_leaf', 1, 5, 1),\n",
    "         'lambda': hp.quniform('lambda', 1., 10., 1.),\n",
    "         'alpha': hp.quniform('alpha', 0., 1., .05)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def objective(space):\n",
    "    params = {\n",
    "        'nthread': 12,\n",
    "        'eta': .1,\n",
    "        'max_depth': int(space['max_depth']),\n",
    "        'colsample_bytree': space['colsample_bytree'],\n",
    "        'subsample': space['subsample'],\n",
    "        'min_child_weight': space['min_child_weight'],\n",
    "        'lambda': space['lambda'],\n",
    "        'alpha': space['alpha'],\n",
    "        'seed': 0,\n",
    "        'objective': 'binary:logistic',\n",
    "        'eval_metric': 'logloss',\n",
    "        'silent': 1\n",
    "    }\n",
    "    \n",
    "    gbm = xgb.train(params, dtrain, num_boost_round=3000, early_stopping_rounds=300, evals=watchlist,\n",
    "              verbose_eval=False)\n",
    "    \n",
    "    pred = gbm.predict(dval, ntree_limit=gbm.best_iteration)\n",
    "    \n",
    "    print(space)\n",
    "    print(log_loss(y_test, pred))\n",
    "    return(log_loss(y_test, pred))\n",
    "#print y_pred\n",
    "\n",
    "#pd.DataFrame({\"ID\": id_test, \"PredictedProb\": y_pred[:,1]}).to_csv('extra_trees.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 1.0, 'subsample': 0.46816746101562534, 'lambda': 2.0, 'min_child_weight': 4.0, 'max_depth': 15.0, 'colsample_bytree': 0.9838631668484263}\n",
      "0.475060803236\n",
      "{'alpha': 0.45, 'subsample': 0.39663558243906166, 'lambda': 8.0, 'min_child_weight': 4.0, 'max_depth': 9.0, 'colsample_bytree': 0.4660742289623673}\n",
      "0.472894268857\n",
      "{'alpha': 0.35000000000000003, 'subsample': 0.5544694555408947, 'lambda': 7.0, 'min_child_weight': 3.0, 'max_depth': 5.0, 'colsample_bytree': 0.618015664279335}\n",
      "0.471138683313\n",
      "{'alpha': 0.30000000000000004, 'subsample': 0.46058224453877133, 'lambda': 5.0, 'min_child_weight': 3.0, 'max_depth': 11.0, 'colsample_bytree': 0.6452748626167566}\n",
      "0.472712248757\n",
      "{'alpha': 0.6000000000000001, 'subsample': 0.6643561918573215, 'lambda': 9.0, 'min_child_weight': 5.0, 'max_depth': 15.0, 'colsample_bytree': 0.5555735144203602}\n",
      "0.47211594184\n",
      "{'alpha': 0.25, 'subsample': 0.37630382795705897, 'lambda': 6.0, 'min_child_weight': 5.0, 'max_depth': 12.0, 'colsample_bytree': 0.9060860305068899}\n",
      "0.474669954617\n",
      "{'alpha': 0.15000000000000002, 'subsample': 0.8298601757618576, 'lambda': 4.0, 'min_child_weight': 3.0, 'max_depth': 6.0, 'colsample_bytree': 0.337075318372708}\n",
      "0.469954512459\n",
      "{'alpha': 0.8, 'subsample': 0.6999354596223593, 'lambda': 8.0, 'min_child_weight': 2.0, 'max_depth': 12.0, 'colsample_bytree': 0.9463096720240971}\n",
      "0.470169060465\n",
      "{'alpha': 0.5, 'subsample': 0.7952959521778318, 'lambda': 3.0, 'min_child_weight': 5.0, 'max_depth': 13.0, 'colsample_bytree': 0.3776898592145012}\n",
      "0.471535990285\n",
      "{'alpha': 0.25, 'subsample': 0.9159258189231709, 'lambda': 3.0, 'min_child_weight': 1.0, 'max_depth': 13.0, 'colsample_bytree': 0.47198439783813034}\n",
      "0.468836762855\n",
      "{'alpha': 0.4, 'subsample': 0.6555372387413394, 'lambda': 9.0, 'min_child_weight': 3.0, 'max_depth': 14.0, 'colsample_bytree': 0.34588012360698384}\n",
      "0.471633165756\n",
      "{'alpha': 0.5, 'subsample': 0.4551346490857083, 'lambda': 9.0, 'min_child_weight': 4.0, 'max_depth': 8.0, 'colsample_bytree': 0.857640275750825}\n",
      "0.472206681397\n",
      "{'alpha': 0.45, 'subsample': 0.6843428773605469, 'lambda': 7.0, 'min_child_weight': 1.0, 'max_depth': 9.0, 'colsample_bytree': 0.47321617239586344}\n",
      "0.470267925586\n",
      "{'alpha': 0.05, 'subsample': 0.5672600079364729, 'lambda': 1.0, 'min_child_weight': 4.0, 'max_depth': 3.0, 'colsample_bytree': 0.7606205262750233}\n",
      "0.472144821611\n",
      "{'alpha': 0.1, 'subsample': 0.5250277676093769, 'lambda': 7.0, 'min_child_weight': 5.0, 'max_depth': 9.0, 'colsample_bytree': 0.30546011141375023}\n",
      "0.471732639329\n",
      "{'alpha': 0.8, 'subsample': 0.39871016851044266, 'lambda': 1.0, 'min_child_weight': 2.0, 'max_depth': 12.0, 'colsample_bytree': 0.3822266440517426}\n",
      "0.47588631368\n",
      "{'alpha': 0.0, 'subsample': 0.7247790858997996, 'lambda': 6.0, 'min_child_weight': 3.0, 'max_depth': 4.0, 'colsample_bytree': 0.49348985599544126}\n",
      "0.471231693054\n",
      "{'alpha': 0.9500000000000001, 'subsample': 0.6382218330845921, 'lambda': 10.0, 'min_child_weight': 3.0, 'max_depth': 4.0, 'colsample_bytree': 0.6175390035867565}\n",
      "0.471460709545\n",
      "{'alpha': 0.9, 'subsample': 0.7256655839596812, 'lambda': 4.0, 'min_child_weight': 4.0, 'max_depth': 9.0, 'colsample_bytree': 0.5923555528233764}\n",
      "0.470679093276\n",
      "{'alpha': 0.65, 'subsample': 0.9624199751168772, 'lambda': 6.0, 'min_child_weight': 2.0, 'max_depth': 6.0, 'colsample_bytree': 0.31767866643841974}\n",
      "0.469971073069\n",
      "{'alpha': 0.2, 'subsample': 0.9591783819982012, 'lambda': 3.0, 'min_child_weight': 1.0, 'max_depth': 7.0, 'colsample_bytree': 0.7171282660653778}\n",
      "0.469357850412\n",
      "{'alpha': 0.2, 'subsample': 0.9745196525948302, 'lambda': 2.0, 'min_child_weight': 1.0, 'max_depth': 7.0, 'colsample_bytree': 0.7344554873952682}\n",
      "0.469166551703\n",
      "{'alpha': 0.2, 'subsample': 0.8890429026343354, 'lambda': 2.0, 'min_child_weight': 1.0, 'max_depth': 10.0, 'colsample_bytree': 0.7217208753384912}\n",
      "0.469067133701\n",
      "{'alpha': 0.30000000000000004, 'subsample': 0.9171654220607607, 'lambda': 2.0, 'min_child_weight': 1.0, 'max_depth': 11.0, 'colsample_bytree': 0.8406086123734342}\n",
      "0.468940556402\n",
      "{'alpha': 0.30000000000000004, 'subsample': 0.8858097648620118, 'lambda': 3.0, 'min_child_weight': 2.0, 'max_depth': 14.0, 'colsample_bytree': 0.8004132398062052}\n",
      "0.470404835516\n",
      "{'alpha': 0.6000000000000001, 'subsample': 0.8951791543003962, 'lambda': 4.0, 'min_child_weight': 1.0, 'max_depth': 11.0, 'colsample_bytree': 0.5410611171544717}\n",
      "0.468774608107\n",
      "{'alpha': 0.6000000000000001, 'subsample': 0.8026400179516484, 'lambda': 4.0, 'min_child_weight': 2.0, 'max_depth': 13.0, 'colsample_bytree': 0.5359676248045518}\n",
      "0.470469855673\n",
      "{'alpha': 0.75, 'subsample': 0.30180736795798774, 'lambda': 5.0, 'min_child_weight': 1.0, 'max_depth': 11.0, 'colsample_bytree': 0.42509431403882747}\n",
      "0.475971838734\n",
      "{'alpha': 0.6000000000000001, 'subsample': 0.838970098096715, 'lambda': 3.0, 'min_child_weight': 2.0, 'max_depth': 15.0, 'colsample_bytree': 0.5351624413977588}\n",
      "0.471171660716\n",
      "{'alpha': 0.7000000000000001, 'subsample': 0.7668319479465684, 'lambda': 5.0, 'min_child_weight': 1.0, 'max_depth': 14.0, 'colsample_bytree': 0.41335532433270666}\n",
      "0.471708784418\n",
      "{'alpha': 0.55, 'subsample': 0.8861830149425811, 'lambda': 4.0, 'min_child_weight': 2.0, 'max_depth': 10.0, 'colsample_bytree': 0.6794089760077937}\n",
      "0.468004718202\n",
      "{'alpha': 0.55, 'subsample': 0.8529141442472556, 'lambda': 4.0, 'min_child_weight': 2.0, 'max_depth': 10.0, 'colsample_bytree': 0.6695866088742699}\n",
      "0.468531704128\n",
      "{'alpha': 0.4, 'subsample': 0.9908371904050987, 'lambda': 5.0, 'min_child_weight': 2.0, 'max_depth': 10.0, 'colsample_bytree': 0.6882747013275106}\n",
      "0.468639276518\n",
      "{'alpha': 0.55, 'subsample': 0.8443982318501028, 'lambda': 4.0, 'min_child_weight': 2.0, 'max_depth': 8.0, 'colsample_bytree': 0.6841855407902915}\n",
      "0.468478628213\n",
      "{'alpha': 0.8500000000000001, 'subsample': 0.7724033274265469, 'lambda': 1.0, 'min_child_weight': 2.0, 'max_depth': 8.0, 'colsample_bytree': 0.7787464592191352}\n",
      "0.469949613282\n",
      "{'alpha': 0.7000000000000001, 'subsample': 0.6057946850898229, 'lambda': 6.0, 'min_child_weight': 3.0, 'max_depth': 8.0, 'colsample_bytree': 0.6421755536779532}\n",
      "0.46979829683\n",
      "{'alpha': 1.0, 'subsample': 0.7448389429460632, 'lambda': 5.0, 'min_child_weight': 2.0, 'max_depth': 7.0, 'colsample_bytree': 0.9797486507695627}\n",
      "0.470052585194\n",
      "{'alpha': 0.4, 'subsample': 0.8533937828764386, 'lambda': 4.0, 'min_child_weight': 3.0, 'max_depth': 6.0, 'colsample_bytree': 0.5825159189212328}\n",
      "0.469880390338\n",
      "{'alpha': 0.55, 'subsample': 0.9392605512310639, 'lambda': 2.0, 'min_child_weight': 2.0, 'max_depth': 5.0, 'colsample_bytree': 0.8973472948513901}\n",
      "0.470524930805\n",
      "{'alpha': 0.45, 'subsample': 0.8057506115637711, 'lambda': 7.0, 'min_child_weight': 3.0, 'max_depth': 8.0, 'colsample_bytree': 0.8122801633360964}\n",
      "0.469200915387\n",
      "{'alpha': 0.35000000000000003, 'subsample': 0.615308790563377, 'lambda': 3.0, 'min_child_weight': 2.0, 'max_depth': 10.0, 'colsample_bytree': 0.6795219775254913}\n",
      "0.470846947281\n",
      "{'alpha': 0.7000000000000001, 'subsample': 0.6910923686630873, 'lambda': 8.0, 'min_child_weight': 3.0, 'max_depth': 12.0, 'colsample_bytree': 0.8719331779318675}\n",
      "0.47093239235\n",
      "{'alpha': 0.55, 'subsample': 0.9978437106447908, 'lambda': 5.0, 'min_child_weight': 1.0, 'max_depth': 5.0, 'colsample_bytree': 0.9355615824210222}\n",
      "0.4707685876\n",
      "{'alpha': 0.5, 'subsample': 0.5243061885568739, 'lambda': 7.0, 'min_child_weight': 2.0, 'max_depth': 6.0, 'colsample_bytree': 0.5060098429469265}\n",
      "0.471074895946\n",
      "{'alpha': 0.8, 'subsample': 0.8688414673771446, 'lambda': 8.0, 'min_child_weight': 3.0, 'max_depth': 9.0, 'colsample_bytree': 0.7531482292325755}\n",
      "0.469394098118\n",
      "{'alpha': 0.65, 'subsample': 0.9324386271164732, 'lambda': 6.0, 'min_child_weight': 4.0, 'max_depth': 8.0, 'colsample_bytree': 0.6297558175381933}\n",
      "0.468575662945\n",
      "{'alpha': 0.35000000000000003, 'subsample': 0.7622265612714688, 'lambda': 1.0, 'min_child_weight': 2.0, 'max_depth': 7.0, 'colsample_bytree': 0.58535903964556}\n",
      "0.469314416276\n",
      "{'alpha': 0.75, 'subsample': 0.8240047339573482, 'lambda': 4.0, 'min_child_weight': 1.0, 'max_depth': 10.0, 'colsample_bytree': 0.7047227340380058}\n",
      "0.469173941757\n",
      "{'alpha': 0.45, 'subsample': 0.3054511453210766, 'lambda': 3.0, 'min_child_weight': 4.0, 'max_depth': 12.0, 'colsample_bytree': 0.8089182342977466}\n",
      "0.476050481093\n",
      "{'alpha': 0.9, 'subsample': 0.7129727239287411, 'lambda': 10.0, 'min_child_weight': 3.0, 'max_depth': 3.0, 'colsample_bytree': 0.6624849770534286}\n",
      "0.472030285317\n",
      "{'alpha': 0.1, 'subsample': 0.42379749152833684, 'lambda': 6.0, 'min_child_weight': 5.0, 'max_depth': 9.0, 'colsample_bytree': 0.6112505322074585}\n",
      "0.472103406451\n",
      "{'alpha': 0.65, 'subsample': 0.5707354346455655, 'lambda': 2.0, 'min_child_weight': 1.0, 'max_depth': 11.0, 'colsample_bytree': 0.8367781587631887}\n",
      "0.471708036639\n",
      "{'alpha': 0.45, 'subsample': 0.6700632128575619, 'lambda': 3.0, 'min_child_weight': 2.0, 'max_depth': 4.0, 'colsample_bytree': 0.4460530776484327}\n",
      "0.471721032394\n",
      "{'alpha': 0.35000000000000003, 'subsample': 0.9164393789651507, 'lambda': 5.0, 'min_child_weight': 1.0, 'max_depth': 13.0, 'colsample_bytree': 0.7698949671875533}\n",
      "0.469255808808\n",
      "{'alpha': 0.25, 'subsample': 0.8190152718810015, 'lambda': 8.0, 'min_child_weight': 3.0, 'max_depth': 5.0, 'colsample_bytree': 0.5639361465430593}\n",
      "0.470028327549\n",
      "{'alpha': 0.8500000000000001, 'subsample': 0.9519000412189733, 'lambda': 1.0, 'min_child_weight': 2.0, 'max_depth': 7.0, 'colsample_bytree': 0.5055799410981046}\n",
      "0.469057953695\n",
      "{'alpha': 0.55, 'subsample': 0.7801386975382578, 'lambda': 4.0, 'min_child_weight': 1.0, 'max_depth': 11.0, 'colsample_bytree': 0.9536299723155361}\n",
      "0.46921773754\n",
      "{'alpha': 0.30000000000000004, 'subsample': 0.745154541591162, 'lambda': 2.0, 'min_child_weight': 3.0, 'max_depth': 10.0, 'colsample_bytree': 0.6959100566376422}\n",
      "0.469267924558\n",
      "{'alpha': 1.0, 'subsample': 0.4923125105960527, 'lambda': 6.0, 'min_child_weight': 2.0, 'max_depth': 9.0, 'colsample_bytree': 0.7416658083811621}\n",
      "0.471054358948\n",
      "{'alpha': 0.5, 'subsample': 0.8784752221883113, 'lambda': 9.0, 'min_child_weight': 4.0, 'max_depth': 12.0, 'colsample_bytree': 0.36159925753138955}\n",
      "0.468342481633\n",
      "{'alpha': 0.0, 'subsample': 0.8770643158590309, 'lambda': 9.0, 'min_child_weight': 5.0, 'max_depth': 14.0, 'colsample_bytree': 0.3390563021230947}\n",
      "0.469732590195\n",
      "{'alpha': 0.4, 'subsample': 0.6392108293092803, 'lambda': 10.0, 'min_child_weight': 4.0, 'max_depth': 15.0, 'colsample_bytree': 0.39614988693389064}\n",
      "0.473420020391\n",
      "{'alpha': 0.15000000000000002, 'subsample': 0.9765890328620953, 'lambda': 9.0, 'min_child_weight': 4.0, 'max_depth': 13.0, 'colsample_bytree': 0.3624350849953786}\n",
      "0.468465535607\n",
      "{'alpha': 0.5, 'subsample': 0.36927258664881163, 'lambda': 9.0, 'min_child_weight': 5.0, 'max_depth': 12.0, 'colsample_bytree': 0.4671764563302725}\n",
      "0.474138607723\n",
      "{'alpha': 0.75, 'subsample': 0.8992663132407225, 'lambda': 8.0, 'min_child_weight': 4.0, 'max_depth': 13.0, 'colsample_bytree': 0.31248525556089485}\n",
      "0.46932860332\n",
      "{'alpha': 0.15000000000000002, 'subsample': 0.984146602682407, 'lambda': 9.0, 'min_child_weight': 4.0, 'max_depth': 15.0, 'colsample_bytree': 0.3537714917148007}\n",
      "0.470412731567\n",
      "{'alpha': 0.05, 'subsample': 0.9717428759298035, 'lambda': 10.0, 'min_child_weight': 4.0, 'max_depth': 14.0, 'colsample_bytree': 0.36660481486618557}\n",
      "0.469152893158\n",
      "{'alpha': 0.25, 'subsample': 0.9135009234445208, 'lambda': 9.0, 'min_child_weight': 5.0, 'max_depth': 12.0, 'colsample_bytree': 0.44083977138916836}\n",
      "0.469282506401\n",
      "{'alpha': 0.65, 'subsample': 0.9427513804947503, 'lambda': 7.0, 'min_child_weight': 4.0, 'max_depth': 13.0, 'colsample_bytree': 0.39916471957153044}\n",
      "0.468432650859\n",
      "{'alpha': 0.65, 'subsample': 0.8664261641032382, 'lambda': 7.0, 'min_child_weight': 5.0, 'max_depth': 14.0, 'colsample_bytree': 0.4133788379555408}\n",
      "0.470182214955\n",
      "{'alpha': 0.6000000000000001, 'subsample': 0.8107489306392677, 'lambda': 8.0, 'min_child_weight': 5.0, 'max_depth': 11.0, 'colsample_bytree': 0.3242252125421759}\n",
      "0.469448982926\n"
     ]
    }
   ],
   "source": [
    "best = fmin(objective, space, algo=tpe.suggest, max_evals=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'alpha': 0.75,\n",
       " 'colsample_bytree': 0.38560597281374503,\n",
       " 'lambda': 7.0,\n",
       " 'max_depth': 12.0,\n",
       " 'min_samples_leaf': 4.0,\n",
       " 'subsample': 0.9966200526403689}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "best2 = best\n",
    "best2['min_child_weight'] = best2['min_samples_leaf']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'min_samples_leaf': 4.0, 'alpha': 0.75, 'subsample': 0.9966200526403689, 'lambda': 7.0, 'min_child_weight': 4.0, 'max_depth': 12.0, 'colsample_bytree': 0.38560597281374503}\n",
      "0.467628613154\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.46762861315378496"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "objective(best2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "params = {\n",
    "    'nthread': 12,\n",
    "    'eta': .01,\n",
    "    'max_depth': 12,\n",
    "    'colsample_bytree': .4,\n",
    "    'subsample': 1.,\n",
    "    'min_child_weight': 4.,\n",
    "    'lambda': 7.,\n",
    "    'alpha': .75,\n",
    "    'seed': 0,\n",
    "    'objective': 'binary:logistic',\n",
    "    'eval_metric': 'logloss',\n",
    "    'silent': 1\n",
    "}\n",
    "\n",
    "gbm = xgb.train(params, dtrain, num_boost_round=3000, early_stopping_rounds=300, evals=watchlist,\n",
    "          verbose_eval=True)\n",
    "\n",
    "pred = gbm.predict(dval, ntree_limit=gbm.best_iteration)\n",
    "\n",
    "#print(space)\n",
    "print(log_loss(y_test, pred))\n",
    "#return(log_loss(y_test, pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'Booster' object has no attribute 'feature_importances_'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-13-082010587aa9>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mgbm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfeature_importances_\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m: 'Booster' object has no attribute 'feature_importances_'"
     ]
    }
   ],
   "source": [
    "def create_feature_map(fmap_filename, features):\n",
    "\"\"\"\n",
    "features: enumerable of feature names\n",
    "\"\"\"\n",
    "    outfile = open(fmap_filename, 'w')\n",
    "    for i, feat in enumerate(features):\n",
    "        outfile.write('{0}\\t{1}\\tq\\n'.format(i, feat))\n",
    "    outfile.close()\n",
    "\n",
    "create_feature_map('xgb.fmap', features) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/libfun/vrt/lib/python3.4/site-packages/ipykernel/__main__.py:16: FutureWarning: pd.rolling_apply is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(center=False,window=2).apply(kwargs=<dict>,args=<tuple>,func=<function>)\n"
     ]
    }
   ],
   "source": [
    "train = pd.read_csv(\"data/train.csv\")\n",
    "target = train['target'].values\n",
    "test = pd.read_csv(\"data/test.csv\")\n",
    "\n",
    "X_train = train.copy()\n",
    "X_test = test.copy()\n",
    "\n",
    "X_train = X_train.drop(['ID','target','v8','v23','v25','v31','v36','v37',\n",
    "                    'v46','v51','v53','v54','v63','v73','v75','v79',\n",
    "                    'v81','v82','v89','v92','v95','v105','v107','v108',\n",
    "                    'v109','v110','v116','v117','v118','v119','v123','v124','v128'],axis=1)\n",
    "X_test = X_test.drop(['ID', 'v8','v23','v25','v31','v36','v37','v46','v51',\n",
    "                  'v53','v54','v63','v73','v75','v79','v81','v82','v89',\n",
    "                  'v92','v95','v105','v107','v108','v109','v110','v116',\n",
    "                  'v117','v118','v119','v123','v124','v128'],axis=1)\n",
    "num_vars = ['v1', 'v2', 'v4', 'v5', 'v6', 'v7', 'v9', 'v10', 'v11',\n",
    "            'v12', 'v13', 'v14', 'v15', 'v16', 'v17', 'v18', 'v19', 'v20',\n",
    "            'v21', 'v26', 'v27', 'v28', 'v29', 'v32', 'v33', 'v34', 'v35', 'v38',\n",
    "            'v39', 'v40', 'v41', 'v42', 'v43', 'v44', 'v45', 'v48', 'v49', 'v50',\n",
    "            'v55', 'v57', 'v58', 'v59', 'v60', 'v61', 'v62', 'v64', 'v65', 'v67',\n",
    "            'v68', 'v69', 'v70', 'v72', 'v76', 'v77', 'v78', 'v80', 'v83', 'v84', \n",
    "            'v85', 'v86', 'v87', 'v88', 'v90', 'v93', 'v94', 'v96', 'v97', 'v98', \n",
    "            'v99', 'v100', 'v101', 'v102', 'v103', 'v104', 'v106', 'v111', 'v114',\n",
    "            'v115', 'v120', 'v121', 'v122', 'v126', 'v127', 'v129', 'v130', 'v131']\n",
    "\n",
    "dropcols = []\n",
    "traincat = np.zeros((X_train.shape[0], 1))\n",
    "testcat = np.zeros((X_test.shape[0], 1))\n",
    "\n",
    "vs = pd.concat([X_train, X_test])\n",
    "for c in num_vars:\n",
    "    if c not in X_train.columns:\n",
    "        continue\n",
    "\n",
    "    X_train.loc[X_train[c].round(5) == 0, c] = 0\n",
    "    X_test.loc[X_test[c].round(5) == 0, c] = 0\n",
    "\n",
    "    delimiter = find_delimiter(vs, c)\n",
    "    X_train[c] *= 1/delimiter\n",
    "    X_test[c] *= 1/delimiter\n",
    "\n",
    "for (train_name, train_series), (test_name, test_series) in zip(X_train.iteritems(), X_test.iteritems()):\n",
    "    if train_series.dtype == 'O' or train_series.dtype == 'int64':\n",
    "        dropcols.append(train_name)\n",
    "        if train_name != 'v22':\n",
    "            X_train[train_name], tmp_indexer = pd.factorize(X_train[train_name])\n",
    "            X_test[test_name] = tmp_indexer.get_indexer(X_test[test_name])\n",
    "            ohe = OneHotEncoder()\n",
    "            tr = ohe.fit_transform(X_train[train_name].values.reshape(-1, 1) + 1).toarray()\n",
    "            ts = ohe.transform(X_test[test_name].values.reshape(-1, 1) + 1).toarray()\n",
    "        else:\n",
    "            X_train[train_name], tmp_indexer = pd.factorize(X_train[train_name])\n",
    "            X_test[test_name] = tmp_indexer.get_indexer(X_test[test_name])\n",
    "            a, b = pd.factorize(X_train[train_name].value_counts()[X_train[train_name].value_counts() > 100].index)\n",
    "            trv = b.get_indexer(X_train[train_name])\n",
    "            tsv = b.get_indexer(X_test[test_name])\n",
    "            ohe = OneHotEncoder()\n",
    "            tr = ohe.fit_transform(trv.reshape(-1, 1) + 1).toarray()\n",
    "            ts = ohe.transform(tsv.reshape(-1, 1) + 1).toarray()\n",
    "            tr = np.hstack((X_train[train_name].values.reshape(-1, 1), tr))\n",
    "            ts = np.hstack((X_test[test_name].values.reshape(-1, 1), ts))\n",
    "\n",
    "        traincat = np.hstack((traincat, tr))\n",
    "        testcat = np.hstack((testcat, ts))\n",
    "\n",
    "X_train_f = np.hstack((X_train.values, traincat))\n",
    "X_test_f = np.hstack((X_test.values, testcat))\n",
    "\n",
    "X_train_n = np.hstack((X_train[X_train.columns.difference(dropcols)].values, traincat))\n",
    "X_test_n = np.hstack((X_test[X_test.columns.difference(dropcols)].values, testcat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dtrain = xgb.DMatrix(X_train_f, label=target)\n",
    "dtest = xgb.DMatrix(X_test_f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "params = {\n",
    "    'nthread': 12,\n",
    "    'eta': .01,\n",
    "    'max_depth': 12,\n",
    "    'colsample_bytree': .4,\n",
    "    'subsample': 1.,\n",
    "    'min_child_weight': 4.,\n",
    "    'lambda': 7.,\n",
    "    'alpha': .75,\n",
    "    'seed': 0,\n",
    "    'objective': 'binary:logistic',\n",
    "    'eval_metric': 'logloss',\n",
    "    'silent': 1\n",
    "}\n",
    "\n",
    "gbm = xgb.train(params, dtrain, num_boost_round=1500, verbose_eval=False)\n",
    "\n",
    "pred = gbm.predict(dtest)\n",
    "\n",
    "sub = pd.read_csv('data/sample_submission.csv', index_col='ID')\n",
    "sub.PredictedProb = pred\n",
    "sub.to_csv('submission/xgb_ohe_ndc.csv', index_label='ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dtrain_n = xgb.DMatrix(X_train_n, label=target)\n",
    "dtest_n = xgb.DMatrix(X_test_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "params = {\n",
    "    'nthread': 12,\n",
    "    'eta': .01,\n",
    "    'max_depth': 12,\n",
    "    'colsample_bytree': .4,\n",
    "    'subsample': 1.,\n",
    "    'min_child_weight': 4.,\n",
    "    'lambda': 7.,\n",
    "    'alpha': .75,\n",
    "    'seed': 0,\n",
    "    'objective': 'binary:logistic',\n",
    "    'eval_metric': 'logloss',\n",
    "    'silent': 1\n",
    "}\n",
    "\n",
    "gbm = xgb.train(params, dtrain_n, num_boost_round=1500, verbose_eval=False)\n",
    "\n",
    "pred = gbm.predict(dtest_n)\n",
    "\n",
    "sub = pd.read_csv('data/sample_submission.csv', index_col='ID')\n",
    "sub.PredictedProb = pred\n",
    "sub.to_csv('submission/xgb_ohe_dc.csv', index_label='ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "zx = np.array(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "z = zx[:, :, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(114393,)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z[0, :].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(114393,)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z.mean(axis=0).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub = pd.read_csv('data/sample_submission.csv', index_col='ID')\n",
    "sub.PredictedProb = z[0, :]\n",
    "sub.to_csv('submission/et_ch_one.csv', index_label='ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub = pd.read_csv('data/sample_submission.csv', index_col='ID')\n",
    "sub.PredictedProb = z.mean(axis=0)\n",
    "sub.to_csv('submission/et_ch_bag.csv', index_label='ID')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}