xgb.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import cross_validation\n",
    "import xgboost as xgb\n",
    "\n",
    "def ToWeight(y):\n",
    "    w = np.zeros(y.shape, dtype=float)\n",
    "    ind = y != 0\n",
    "    w[ind] = 1./(y[ind]**2)\n",
    "    return w\n",
    "\n",
    "def rmspe(yhat, y):\n",
    "    w = ToWeight(y)\n",
    "    rmspe = np.sqrt(np.mean( w * (y - yhat)**2 ))\n",
    "    return rmspe\n",
    "\n",
    "def rmspe_xg(yhat, y):\n",
    "    # y = y.values\n",
    "    y = y.get_label()\n",
    "    y = np.exp(y) - 1\n",
    "    yhat = np.exp(yhat) - 1\n",
    "    w = ToWeight(y)\n",
    "    rmspe = np.sqrt(np.mean(w * (y - yhat)**2))\n",
    "    return \"rmspe\", rmspe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('train_nnew.csv', low_memory=False)\n",
    "test = pd.read_csv('test_nnew.csv', low_memory=False)\n",
    "features = [u'Open', u'Promo', u'SchoolHoliday', u'StateHoliday_0',\n",
    "       u'StateHoliday_a', u'DayOfWeek_1', u'DayOfWeek_2', u'DayOfWeek_3',\n",
    "       u'DayOfWeek_4', u'DayOfWeek_5', u'DayOfWeek_6', u'DayOfWeek_7',\n",
    "       u'CompetitionDistance', u'Promo2', 'year', 'Mean_Sales', 'month', 'day',\n",
    "       u'StoreType_a', u'StoreType_b', u'StoreType_c', u'StoreType_d',\n",
    "       u'Assortment_a', u'Assortment_b', u'Assortment_c', u'CompetitionOpen']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['year'] = train.Date.apply(lambda x: x.split('-')[0])\n",
    "train['year'] = train['year'].astype(float)\n",
    "train['month'] = train.Date.apply(lambda x: x.split('-')[1])\n",
    "train['month'] = train['month'].astype(float)\n",
    "train['day'] = train.Date.apply(lambda x: x.split('-')[2])\n",
    "train['day'] = train['day'].astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test['year'] = test.Date.apply(lambda x: x.split('-')[0])\n",
    "test['year'] = test['year'].astype(float)\n",
    "test['month'] = test.Date.apply(lambda x: x.split('-')[1])\n",
    "test['month'] = test['month'].astype(float)\n",
    "test['day'] = test.Date.apply(lambda x: x.split('-')[2])\n",
    "test['day'] = test['day'].astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train['StoreType'] = train['StoreType_a'] + 2*train['StoreType_b'] + 3*train['StoreType_c'] + 4*train['StoreType_d']\n",
    "\n",
    "test['StoreType'] = test['StoreType_a'] + 2*test['StoreType_b'] + 3*test['StoreType_c'] + 4*test['StoreType_d']\n",
    "\n",
    "train['Assortment'] = train['Assortment_a'] + 2*train['Assortment_b'] + 3*train['Assortment_c']\n",
    "\n",
    "test['Assortment'] = test['Assortment_a'] + 2*test['Assortment_b'] + 3*test['Assortment_c']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features = [u'Open', u'Promo', u'SchoolHoliday', u'StateHoliday_0',\n",
    "       u'StateHoliday_a', u'DayOfWeek',\n",
    "       u'CompetitionDistance', u'Promo2', 'year', 'Mean_Sales', 'month', 'day',\n",
    "       u'StoreType',\n",
    "       u'Assortment', u'CompetitionOpen']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train['Date'] = train['Date'].astype('datetime64')\n",
    "test['Date'] = test['Date'].astype('datetime64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "params = {\"objective\": \"reg:linear\", \n",
    "          \"eta\": 0.05, \n",
    "          \"max_depth\": 20, \n",
    "          \"subsample\": 0.9, \n",
    "          \"colsample_bytree\": 0.9, \n",
    "          \"silent\": 1, \n",
    "          \"lambda\" : 1000, \n",
    "          \"alpha\" : 1 } \n",
    "num_trees = 900"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train a XGBoost model\n",
      "1017208    2013-01-01\n",
      "Name: Date, dtype: object\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "all feature_names must be alphanumerics",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-3c50ee0dc1d9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcross_validation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_test_split\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.05\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;31m#X_train, X_test = train.head(len(train) - val_size), train.tail(val_size)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mdtrain\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mxgb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDMatrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Open'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Open'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Sales\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0mdvalid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mxgb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDMatrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Open'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Open'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Sales\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0mdtest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mxgb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDMatrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/xgboost-0.4-py2.7.egg/xgboost/core.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data, label, missing, weight, silent, feature_names, feature_types)\u001b[0m\n\u001b[1;32m    230\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_weight\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    231\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 232\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeature_names\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfeature_names\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    233\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeature_types\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfeature_types\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/xgboost-0.4-py2.7.egg/xgboost/core.pyc\u001b[0m in \u001b[0;36mfeature_names\u001b[0;34m(self, feature_names)\u001b[0m\n\u001b[1;32m    524\u001b[0m             if not all(isinstance(f, STRING_TYPES) and f.isalnum()\n\u001b[1;32m    525\u001b[0m                        for f in feature_names):\n\u001b[0;32m--> 526\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'all feature_names must be alphanumerics'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    527\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    528\u001b[0m             \u001b[0;31m# reset feature_types also\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: all feature_names must be alphanumerics"
     ]
    }
   ],
   "source": [
    "print(\"Train a XGBoost model\")\n",
    "val_size = 50000\n",
    "#train = train.sort(['Date'])\n",
    "print(train.tail(1)['Date'])\n",
    "X_train, X_test = cross_validation.train_test_split(train, test_size=0.05, random_state = 1)\n",
    "#X_train, X_test = train.head(len(train) - val_size), train.tail(val_size)\n",
    "dtrain = xgb.DMatrix(X_train[X_train['Open'] > 0][features], np.log(X_train[X_train['Open'] > 0][\"Sales\"] + 1))\n",
    "dvalid = xgb.DMatrix(X_test[X_test['Open'] > 0][features], np.log(X_test[X_test['Open'] > 0][\"Sales\"] + 1))\n",
    "dtest = xgb.DMatrix(test[features])\n",
    "watchlist = [(dvalid, 'eval'), (dtrain, 'train')]\n",
    "gbm = xgb.train(params, dtrain, num_trees, evals=watchlist, early_stopping_rounds=50, \n",
    "                feval=rmspe_xg)#, verbose_eval=True)\n",
    "\n",
    "print(\"Validating\")\n",
    "train_probs = gbm.predict(xgb.DMatrix(X_test[features]))\n",
    "indices = train_probs < 0\n",
    "train_probs[indices] = 0\n",
    "error = rmspe(np.exp(train_probs) - 1, X_test['Sales'].values)\n",
    "print('error', error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Will train until train error hasn't decreased in 50 rounds.\n",
      "[0]\ttrain-rmspe:0.999713\n",
      "[1]\ttrain-rmspe:0.999508\n",
      "[2]\ttrain-rmspe:0.999227\n",
      "[3]\ttrain-rmspe:0.998845\n",
      "[4]\ttrain-rmspe:0.998335\n",
      "[5]\ttrain-rmspe:0.997666\n",
      "[6]\ttrain-rmspe:0.996800\n",
      "[7]\ttrain-rmspe:0.995700\n",
      "[8]\ttrain-rmspe:0.994319\n",
      "[9]\ttrain-rmspe:0.992611\n",
      "[10]\ttrain-rmspe:0.990528\n",
      "[11]\ttrain-rmspe:0.988014\n",
      "[12]\ttrain-rmspe:0.985019\n",
      "[13]\ttrain-rmspe:0.981496\n",
      "[14]\ttrain-rmspe:0.977343\n",
      "[15]\ttrain-rmspe:0.972636\n",
      "[16]\ttrain-rmspe:0.967263\n",
      "[17]\ttrain-rmspe:0.961145\n",
      "[18]\ttrain-rmspe:0.954329\n",
      "[19]\ttrain-rmspe:0.946743\n",
      "[20]\ttrain-rmspe:0.938246\n",
      "[21]\ttrain-rmspe:0.929084\n",
      "[22]\ttrain-rmspe:0.919124\n",
      "[23]\ttrain-rmspe:0.908363\n",
      "[24]\ttrain-rmspe:0.896658\n",
      "[25]\ttrain-rmspe:0.884170\n",
      "[26]\ttrain-rmspe:0.870981\n",
      "[27]\ttrain-rmspe:0.857263\n",
      "[28]\ttrain-rmspe:0.842884\n",
      "[29]\ttrain-rmspe:0.827893\n",
      "[30]\ttrain-rmspe:0.812328\n",
      "[31]\ttrain-rmspe:0.796061\n",
      "[32]\ttrain-rmspe:0.779516\n",
      "[33]\ttrain-rmspe:0.762581\n",
      "[34]\ttrain-rmspe:0.745251\n",
      "[35]\ttrain-rmspe:0.727498\n",
      "[36]\ttrain-rmspe:0.709693\n",
      "[37]\ttrain-rmspe:0.691705\n",
      "[38]\ttrain-rmspe:0.673640\n",
      "[39]\ttrain-rmspe:0.655501\n",
      "[40]\ttrain-rmspe:0.637408\n",
      "[41]\ttrain-rmspe:0.619348\n",
      "[42]\ttrain-rmspe:0.601466\n",
      "[43]\ttrain-rmspe:0.583658\n",
      "[44]\ttrain-rmspe:0.566156\n",
      "[45]\ttrain-rmspe:0.548810\n",
      "[46]\ttrain-rmspe:0.531819\n",
      "[47]\ttrain-rmspe:0.515130\n",
      "[48]\ttrain-rmspe:0.498777\n",
      "[49]\ttrain-rmspe:0.482836\n",
      "[50]\ttrain-rmspe:0.467338\n",
      "[51]\ttrain-rmspe:0.452294\n",
      "[52]\ttrain-rmspe:0.437726\n",
      "[53]\ttrain-rmspe:0.423612\n",
      "[54]\ttrain-rmspe:0.410024\n",
      "[55]\ttrain-rmspe:0.396942\n",
      "[56]\ttrain-rmspe:0.384356\n",
      "[57]\ttrain-rmspe:0.372347\n",
      "[58]\ttrain-rmspe:0.360809\n",
      "[59]\ttrain-rmspe:0.349808\n",
      "[60]\ttrain-rmspe:0.339336\n",
      "[61]\ttrain-rmspe:0.329475\n",
      "[62]\ttrain-rmspe:0.320027\n",
      "[63]\ttrain-rmspe:0.311114\n",
      "[64]\ttrain-rmspe:0.302683\n",
      "[65]\ttrain-rmspe:0.294648\n",
      "[66]\ttrain-rmspe:0.287073\n",
      "[67]\ttrain-rmspe:0.279954\n",
      "[68]\ttrain-rmspe:0.273241\n",
      "[69]\ttrain-rmspe:0.267214\n",
      "[70]\ttrain-rmspe:0.261531\n",
      "[71]\ttrain-rmspe:0.256276\n",
      "[72]\ttrain-rmspe:0.251198\n",
      "[73]\ttrain-rmspe:0.246592\n",
      "[74]\ttrain-rmspe:0.242521\n",
      "[75]\ttrain-rmspe:0.238674\n",
      "[76]\ttrain-rmspe:0.235235\n",
      "[77]\ttrain-rmspe:0.231791\n",
      "[78]\ttrain-rmspe:0.228921\n",
      "[79]\ttrain-rmspe:0.226086\n",
      "[80]\ttrain-rmspe:0.223536\n",
      "[81]\ttrain-rmspe:0.221243\n",
      "[82]\ttrain-rmspe:0.218935\n",
      "[83]\ttrain-rmspe:0.217122\n",
      "[84]\ttrain-rmspe:0.215353\n",
      "[85]\ttrain-rmspe:0.213695\n",
      "[86]\ttrain-rmspe:0.212312\n",
      "[87]\ttrain-rmspe:0.210828\n",
      "[88]\ttrain-rmspe:0.209549\n",
      "[89]\ttrain-rmspe:0.208356\n",
      "[90]\ttrain-rmspe:0.207207\n",
      "[91]\ttrain-rmspe:0.206114\n",
      "[92]\ttrain-rmspe:0.205205\n",
      "[93]\ttrain-rmspe:0.204204\n",
      "[94]\ttrain-rmspe:0.203531\n",
      "[95]\ttrain-rmspe:0.202860\n",
      "[96]\ttrain-rmspe:0.202202\n",
      "[97]\ttrain-rmspe:0.201625\n",
      "[98]\ttrain-rmspe:0.201126\n",
      "[99]\ttrain-rmspe:0.200341\n",
      "[100]\ttrain-rmspe:0.199721\n",
      "[101]\ttrain-rmspe:0.199291\n",
      "[102]\ttrain-rmspe:0.198821\n",
      "[103]\ttrain-rmspe:0.198630\n",
      "[104]\ttrain-rmspe:0.198028\n",
      "[105]\ttrain-rmspe:0.197686\n",
      "[106]\ttrain-rmspe:0.197459\n",
      "[107]\ttrain-rmspe:0.196976\n",
      "[108]\ttrain-rmspe:0.196698\n",
      "[109]\ttrain-rmspe:0.196294\n",
      "[110]\ttrain-rmspe:0.195906\n",
      "[111]\ttrain-rmspe:0.195850\n",
      "[112]\ttrain-rmspe:0.195716\n",
      "[113]\ttrain-rmspe:0.195617\n",
      "[114]\ttrain-rmspe:0.195389\n",
      "[115]\ttrain-rmspe:0.195072\n",
      "[116]\ttrain-rmspe:0.194733\n",
      "[117]\ttrain-rmspe:0.194431\n",
      "[118]\ttrain-rmspe:0.194090\n",
      "[119]\ttrain-rmspe:0.193916\n",
      "[120]\ttrain-rmspe:0.193782\n",
      "[121]\ttrain-rmspe:0.193552\n",
      "[122]\ttrain-rmspe:0.193283\n",
      "[123]\ttrain-rmspe:0.192985\n",
      "[124]\ttrain-rmspe:0.192725\n",
      "[125]\ttrain-rmspe:0.192511\n",
      "[126]\ttrain-rmspe:0.192306\n",
      "[127]\ttrain-rmspe:0.192207\n",
      "[128]\ttrain-rmspe:0.192081\n",
      "[129]\ttrain-rmspe:0.191927\n",
      "[130]\ttrain-rmspe:0.191730\n",
      "[131]\ttrain-rmspe:0.191627\n",
      "[132]\ttrain-rmspe:0.191369\n",
      "[133]\ttrain-rmspe:0.191278\n",
      "[134]\ttrain-rmspe:0.191206\n",
      "[135]\ttrain-rmspe:0.191065\n",
      "[136]\ttrain-rmspe:0.190920\n",
      "[137]\ttrain-rmspe:0.190790\n",
      "[138]\ttrain-rmspe:0.190746\n",
      "[139]\ttrain-rmspe:0.190637\n",
      "[140]\ttrain-rmspe:0.190555\n",
      "[141]\ttrain-rmspe:0.190368\n",
      "[142]\ttrain-rmspe:0.190195\n",
      "[143]\ttrain-rmspe:0.190032\n",
      "[144]\ttrain-rmspe:0.189892\n",
      "[145]\ttrain-rmspe:0.189702\n",
      "[146]\ttrain-rmspe:0.189624\n",
      "[147]\ttrain-rmspe:0.189460\n",
      "[148]\ttrain-rmspe:0.189271\n",
      "[149]\ttrain-rmspe:0.189102\n",
      "[150]\ttrain-rmspe:0.188902\n",
      "[151]\ttrain-rmspe:0.188706\n",
      "[152]\ttrain-rmspe:0.188637\n",
      "[153]\ttrain-rmspe:0.188491\n",
      "[154]\ttrain-rmspe:0.188390\n",
      "[155]\ttrain-rmspe:0.188233\n",
      "[156]\ttrain-rmspe:0.188119\n",
      "[157]\ttrain-rmspe:0.187942\n",
      "[158]\ttrain-rmspe:0.187774\n",
      "[159]\ttrain-rmspe:0.187614\n",
      "[160]\ttrain-rmspe:0.187407\n",
      "[161]\ttrain-rmspe:0.187254\n",
      "[162]\ttrain-rmspe:0.187105\n",
      "[163]\ttrain-rmspe:0.186958\n",
      "[164]\ttrain-rmspe:0.186842\n",
      "[165]\ttrain-rmspe:0.186693\n",
      "[166]\ttrain-rmspe:0.186561\n",
      "[167]\ttrain-rmspe:0.186468\n",
      "[168]\ttrain-rmspe:0.186306\n",
      "[169]\ttrain-rmspe:0.186185\n",
      "[170]\ttrain-rmspe:0.186134\n",
      "[171]\ttrain-rmspe:0.186024\n",
      "[172]\ttrain-rmspe:0.185841\n",
      "[173]\ttrain-rmspe:0.185700\n",
      "[174]\ttrain-rmspe:0.185582\n"
     ]
    }
   ],
   "source": [
    "dtrain = xgb.DMatrix(train[train['Open'] > 0][features], np.log(train[train['Open'] > 0][\"Sales\"] + 1))\n",
    "dtest = xgb.DMatrix(test[features])\n",
    "watchlist = [(dtrain, 'train')]\n",
    "gbm = xgb.train(params, dtrain, num_trees, evals=watchlist, early_stopping_rounds=50, \n",
    "                feval=rmspe_xg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(\"Make predictions on the test set\")\n",
    "test_probs = gbm.predict(xgb.DMatrix(test[features]))\n",
    "indices = test_probs < 0\n",
    "test_probs[indices] = 0\n",
    "submission = pd.DataFrame({\"Id\": test[\"Id\"], \"Sales\": np.exp(test_probs) - 1})\n",
    "submission.to_csv(\"xgboost_lb_all.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}