notebook_jupyter_desktop.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_cell_guid": "b1b1c30b-5503-4896-a6ae-3fde8c58a857",
    "_execution_state": "idle",
    "_uuid": "6253fc3613cae37331065f8a7b510ab3ee469e8a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fastest_routes_test.csv\n",
      "fastest_routes_train_part_1.csv\n",
      "fastest_routes_train_part_2.csv\n",
      "sample_submission.csv\n",
      "test.csv\n",
      "test_data.csv\n",
      "train.csv\n",
      "train_data.csv\n",
      "weather_data_nyc_centralpark_2016.csv\n",
      "xgb_model.sav\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import numpy as np \n",
    "import pandas as pd\n",
    "#import seaborn as sns\n",
    "#import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "import xgboost\n",
    "%matplotlib inline\n",
    "\n",
    "from subprocess import check_output\n",
    "print(check_output([\"ls\", \"/home/ankit/Downloads/all\"]).decode(\"utf8\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_cell_guid": "2731cf63-63f6-493e-8aec-ad0b8510b11e",
    "_execution_state": "idle",
    "_uuid": "7c3dd1c5411eb81518d34c98ed6843c12b6238bc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "75.11792755126953\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('/home/ankit/Downloads/all/train.csv')\n",
    "df.passenger_count = df.passenger_count.astype(np.uint8)\n",
    "df.vendor_id = df.vendor_id.astype(np.uint8)\n",
    "df.trip_duration = df.trip_duration.astype(np.uint32)\n",
    "for c in [c for c in df.columns if c.endswith('tude')]:\n",
    "    df.loc[:,c] = df[c].astype(np.float32)\n",
    "print(df.memory_usage().sum()/2**20)\n",
    "df.pickup_datetime=pd.to_datetime(df.pickup_datetime)\n",
    "df.dropoff_datetime=pd.to_datetime(df.dropoff_datetime)\n",
    "df['pu_hour'] = df.pickup_datetime.dt.hour\n",
    "df['yday'] = df.pickup_datetime.dt.dayofyear\n",
    "df['wday'] = df.pickup_datetime.dt.dayofweek\n",
    "df['month'] = df.pickup_datetime.dt.month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#sns.set_style('white')\n",
    "#sns.set_context(\"paper\",font_scale=2)\n",
    "corr = df.corr()\n",
    "mask = np.zeros_like(corr, dtype=np.bool)\n",
    "mask[np.triu_indices_from(mask)] = True\n",
    "#f, ax = plt.subplots(figsize=(11,9))\n",
    "#cmap = sns.diverging_palette(220, 10, as_cmap=True)\n",
    "#sns.heatmap(corr, mask=mask, cmap=cmap, vmax=0.3, center=0,\n",
    " #          square=True, linewidths=0.5, cbar_kws={\"shrink\":0.5})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fig, ax = plt.subplots(ncols=1, nrows=1)\n",
    "#sns.distplot(df['trip_duration']/3600,ax=ax,bins=100,kde=False,hist_kws={'log':True})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fig, ax = plt.subplots(ncols=1, nrows=1)\n",
    "#ax.set_xlim(0,30)\n",
    "#sns.distplot(df['trip_duration']/3600,ax=ax,bins=1000,kde=False,hist_kws={'log':True})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "_cell_guid": "0096fc06-fdc4-4e69-800b-ef8612d3a87a",
    "_execution_state": "idle",
    "_uuid": "24be42e88b116df8b0ccdc9d5275b33aafd52934",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def haversine(lon1, lat1, lon2, lat2):\n",
    "\n",
    "    lon1, lat1, lon2, lat2 = map(np.radians, [lon1, lat1, lon2, lat2])\n",
    "\n",
    "    dlon = lon2 - lon1\n",
    "    dlat = lat2 - lat1\n",
    "\n",
    "    a = np.sin(dlat/2.0)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon/2.0)**2\n",
    "\n",
    "    c = 2 * np.arcsin(np.sqrt(a))\n",
    "    km = 6367 * c\n",
    "    miles = km *  0.621371\n",
    "    return miles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "_cell_guid": "e328c381-57ff-4eb1-a422-24c7c746de60",
    "_execution_state": "idle",
    "_uuid": "37c0d27a3f54f11cb4505b31dab8211d9f50b5d5"
   },
   "outputs": [],
   "source": [
    "df['distance'] = haversine(df.pickup_longitude, df.pickup_latitude,\n",
    "                                           df.dropoff_longitude, df.dropoff_latitude)\n",
    "#df['distance']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "_cell_guid": "bc07643c-8d28-4bc6-adf6-c42a16951ad0",
    "_execution_state": "idle",
    "_uuid": "c58addc802a84283bdb6995f87c916f091469cff",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "wdf = pd.read_csv('/home/ankit/Downloads/all/weather_data_nyc_centralpark_2016.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "_cell_guid": "5460a272-365c-4cf3-b762-73fc2355581b",
    "_execution_state": "idle",
    "_uuid": "c899eabb0c5c723112f1c67c7b9d472d3a733872",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "wdf['date']=pd.to_datetime(wdf.date,format='%d-%m-%Y')\n",
    "wdf['yday'] = wdf.date.dt.dayofyear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "_cell_guid": "502ddf18-340f-4d4b-a8bd-c0ca8cd3da2e",
    "_execution_state": "idle",
    "_uuid": "d7cb7e9d02379be0d4b36ddfeea80fb82a24c615"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>maximum temperature</th>\n",
       "      <th>minimum temperature</th>\n",
       "      <th>average temperature</th>\n",
       "      <th>precipitation</th>\n",
       "      <th>snow fall</th>\n",
       "      <th>snow depth</th>\n",
       "      <th>yday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>42</td>\n",
       "      <td>34</td>\n",
       "      <td>38.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2016-01-02</td>\n",
       "      <td>40</td>\n",
       "      <td>32</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016-01-03</td>\n",
       "      <td>45</td>\n",
       "      <td>35</td>\n",
       "      <td>40.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2016-01-04</td>\n",
       "      <td>36</td>\n",
       "      <td>14</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2016-01-05</td>\n",
       "      <td>29</td>\n",
       "      <td>11</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        date  maximum temperature  minimum temperature  average temperature  \\\n",
       "0 2016-01-01                   42                   34                 38.0   \n",
       "1 2016-01-02                   40                   32                 36.0   \n",
       "2 2016-01-03                   45                   35                 40.0   \n",
       "3 2016-01-04                   36                   14                 25.0   \n",
       "4 2016-01-05                   29                   11                 20.0   \n",
       "\n",
       "  precipitation snow fall snow depth  yday  \n",
       "0          0.00       0.0          0     1  \n",
       "1          0.00       0.0          0     2  \n",
       "2          0.00       0.0          0     3  \n",
       "3          0.00       0.0          0     4  \n",
       "4          0.00       0.0          0     5  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "wdf['snowfall'] = wdf['snow fall'].replace(['T'],0.05).astype(np.float32)\n",
    "wdf['precipitation'] = wdf['precipitation'].replace(['T'],0.05).astype(np.float32)\n",
    "wdf['snowdepth'] = wdf['snow depth'].replace(['T'],0.05).astype(np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "_cell_guid": "f589e6d7-659b-411d-a755-778b6aaf1e3f",
    "_execution_state": "idle",
    "_uuid": "10924046e6f8106f4fdeba52b1f4225e5a824ee5",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.merge(df,wdf,on='yday')\n",
    "df = df.drop(['date','maximum temperature','minimum temperature'],axis=1)\n",
    "#df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sns.set_style('white')\n",
    "#sns.set_context(\"paper\",font_scale=2)\n",
    "corr = df.corr()\n",
    "mask = np.zeros_like(corr, dtype=np.bool)\n",
    "mask[np.triu_indices_from(mask)] = True\n",
    "#f, ax = plt.subplots(figsize=(11,9))\n",
    "#cmap = sns.diverging_palette(220, 10, as_cmap=True)\n",
    "#sns.heatmap(corr, mask=mask, cmap=cmap, vmax=0.3, center=0,\n",
    " #          square=True, linewidths=0.5, cbar_kws={\"shrink\":0.5})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vendor_id</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>trip_duration</th>\n",
       "      <th>pu_hour</th>\n",
       "      <th>yday</th>\n",
       "      <th>wday</th>\n",
       "      <th>month</th>\n",
       "      <th>distance</th>\n",
       "      <th>average temperature</th>\n",
       "      <th>precipitation</th>\n",
       "      <th>snowfall</th>\n",
       "      <th>snowdepth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>vendor_id</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.287415</td>\n",
       "      <td>0.007820</td>\n",
       "      <td>0.001742</td>\n",
       "      <td>0.001528</td>\n",
       "      <td>0.004496</td>\n",
       "      <td>0.020304</td>\n",
       "      <td>0.009299</td>\n",
       "      <td>-0.006025</td>\n",
       "      <td>0.001311</td>\n",
       "      <td>-0.006221</td>\n",
       "      <td>0.008109</td>\n",
       "      <td>-0.005760</td>\n",
       "      <td>0.001385</td>\n",
       "      <td>0.000207</td>\n",
       "      <td>0.006036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>passenger_count</th>\n",
       "      <td>0.287415</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.002169</td>\n",
       "      <td>-0.005125</td>\n",
       "      <td>-0.000343</td>\n",
       "      <td>-0.002762</td>\n",
       "      <td>0.008471</td>\n",
       "      <td>0.009101</td>\n",
       "      <td>-0.001784</td>\n",
       "      <td>0.025191</td>\n",
       "      <td>-0.002143</td>\n",
       "      <td>0.010306</td>\n",
       "      <td>-0.001203</td>\n",
       "      <td>0.001415</td>\n",
       "      <td>0.000510</td>\n",
       "      <td>0.000733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pickup_longitude</th>\n",
       "      <td>0.007820</td>\n",
       "      <td>0.002169</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.022568</td>\n",
       "      <td>0.783582</td>\n",
       "      <td>0.100190</td>\n",
       "      <td>0.026542</td>\n",
       "      <td>0.010150</td>\n",
       "      <td>0.004263</td>\n",
       "      <td>-0.016356</td>\n",
       "      <td>0.004474</td>\n",
       "      <td>0.259780</td>\n",
       "      <td>0.005547</td>\n",
       "      <td>0.001418</td>\n",
       "      <td>-0.003288</td>\n",
       "      <td>0.003602</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pickup_latitude</th>\n",
       "      <td>0.001742</td>\n",
       "      <td>-0.005125</td>\n",
       "      <td>0.022568</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.114884</td>\n",
       "      <td>0.494038</td>\n",
       "      <td>-0.029204</td>\n",
       "      <td>0.010603</td>\n",
       "      <td>-0.002516</td>\n",
       "      <td>-0.028976</td>\n",
       "      <td>-0.001439</td>\n",
       "      <td>-0.210354</td>\n",
       "      <td>-0.002011</td>\n",
       "      <td>-0.004180</td>\n",
       "      <td>-0.005058</td>\n",
       "      <td>-0.000112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <td>0.001528</td>\n",
       "      <td>-0.000343</td>\n",
       "      <td>0.783582</td>\n",
       "      <td>0.114884</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.124873</td>\n",
       "      <td>0.014678</td>\n",
       "      <td>-0.022455</td>\n",
       "      <td>0.004612</td>\n",
       "      <td>-0.001130</td>\n",
       "      <td>0.004758</td>\n",
       "      <td>0.134033</td>\n",
       "      <td>0.005502</td>\n",
       "      <td>0.001660</td>\n",
       "      <td>0.000774</td>\n",
       "      <td>0.001092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <td>0.004496</td>\n",
       "      <td>-0.002762</td>\n",
       "      <td>0.100190</td>\n",
       "      <td>0.494038</td>\n",
       "      <td>0.124873</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.020677</td>\n",
       "      <td>0.013612</td>\n",
       "      <td>-0.001554</td>\n",
       "      <td>-0.021762</td>\n",
       "      <td>-0.000607</td>\n",
       "      <td>-0.142128</td>\n",
       "      <td>-0.000997</td>\n",
       "      <td>-0.002710</td>\n",
       "      <td>-0.002434</td>\n",
       "      <td>0.000454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>trip_duration</th>\n",
       "      <td>0.020304</td>\n",
       "      <td>0.008471</td>\n",
       "      <td>0.026542</td>\n",
       "      <td>-0.029204</td>\n",
       "      <td>0.014678</td>\n",
       "      <td>-0.020677</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.003690</td>\n",
       "      <td>0.006625</td>\n",
       "      <td>-0.000708</td>\n",
       "      <td>0.006607</td>\n",
       "      <td>0.094777</td>\n",
       "      <td>0.004574</td>\n",
       "      <td>-0.002590</td>\n",
       "      <td>-0.000190</td>\n",
       "      <td>0.002716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pu_hour</th>\n",
       "      <td>0.009299</td>\n",
       "      <td>0.009101</td>\n",
       "      <td>0.010150</td>\n",
       "      <td>0.010603</td>\n",
       "      <td>-0.022455</td>\n",
       "      <td>0.013612</td>\n",
       "      <td>0.003690</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.003340</td>\n",
       "      <td>-0.084255</td>\n",
       "      <td>-0.003438</td>\n",
       "      <td>-0.015813</td>\n",
       "      <td>-0.004948</td>\n",
       "      <td>-0.028656</td>\n",
       "      <td>-0.048589</td>\n",
       "      <td>0.014215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yday</th>\n",
       "      <td>-0.006025</td>\n",
       "      <td>-0.001784</td>\n",
       "      <td>0.004263</td>\n",
       "      <td>-0.002516</td>\n",
       "      <td>0.004612</td>\n",
       "      <td>-0.001554</td>\n",
       "      <td>0.006625</td>\n",
       "      <td>-0.003340</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.013615</td>\n",
       "      <td>0.985657</td>\n",
       "      <td>0.016443</td>\n",
       "      <td>0.837212</td>\n",
       "      <td>-0.031407</td>\n",
       "      <td>-0.077248</td>\n",
       "      <td>-0.229351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wday</th>\n",
       "      <td>0.001311</td>\n",
       "      <td>0.025191</td>\n",
       "      <td>-0.016356</td>\n",
       "      <td>-0.028976</td>\n",
       "      <td>-0.001130</td>\n",
       "      <td>-0.021762</td>\n",
       "      <td>-0.000708</td>\n",
       "      <td>-0.084255</td>\n",
       "      <td>-0.013615</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.014544</td>\n",
       "      <td>0.007599</td>\n",
       "      <td>-0.019799</td>\n",
       "      <td>-0.041673</td>\n",
       "      <td>0.028427</td>\n",
       "      <td>-0.034647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month</th>\n",
       "      <td>-0.006221</td>\n",
       "      <td>-0.002143</td>\n",
       "      <td>0.004474</td>\n",
       "      <td>-0.001439</td>\n",
       "      <td>0.004758</td>\n",
       "      <td>-0.000607</td>\n",
       "      <td>0.006607</td>\n",
       "      <td>-0.003438</td>\n",
       "      <td>0.985657</td>\n",
       "      <td>-0.014544</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.015668</td>\n",
       "      <td>0.814295</td>\n",
       "      <td>-0.022469</td>\n",
       "      <td>-0.079563</td>\n",
       "      <td>-0.268631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>distance</th>\n",
       "      <td>0.008109</td>\n",
       "      <td>0.010306</td>\n",
       "      <td>0.259780</td>\n",
       "      <td>-0.210354</td>\n",
       "      <td>0.134033</td>\n",
       "      <td>-0.142128</td>\n",
       "      <td>0.094777</td>\n",
       "      <td>-0.015813</td>\n",
       "      <td>0.016443</td>\n",
       "      <td>0.007599</td>\n",
       "      <td>0.015668</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.018631</td>\n",
       "      <td>0.001317</td>\n",
       "      <td>-0.003175</td>\n",
       "      <td>0.000548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>average temperature</th>\n",
       "      <td>-0.005760</td>\n",
       "      <td>-0.001203</td>\n",
       "      <td>0.005547</td>\n",
       "      <td>-0.002011</td>\n",
       "      <td>0.005502</td>\n",
       "      <td>-0.000997</td>\n",
       "      <td>0.004574</td>\n",
       "      <td>-0.004948</td>\n",
       "      <td>0.837212</td>\n",
       "      <td>-0.019799</td>\n",
       "      <td>0.814295</td>\n",
       "      <td>0.018631</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.020519</td>\n",
       "      <td>-0.094767</td>\n",
       "      <td>-0.170218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precipitation</th>\n",
       "      <td>0.001385</td>\n",
       "      <td>0.001415</td>\n",
       "      <td>0.001418</td>\n",
       "      <td>-0.004180</td>\n",
       "      <td>0.001660</td>\n",
       "      <td>-0.002710</td>\n",
       "      <td>-0.002590</td>\n",
       "      <td>-0.028656</td>\n",
       "      <td>-0.031407</td>\n",
       "      <td>-0.041673</td>\n",
       "      <td>-0.022469</td>\n",
       "      <td>0.001317</td>\n",
       "      <td>0.020519</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.329554</td>\n",
       "      <td>-0.023126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>snowfall</th>\n",
       "      <td>0.000207</td>\n",
       "      <td>0.000510</td>\n",
       "      <td>-0.003288</td>\n",
       "      <td>-0.005058</td>\n",
       "      <td>0.000774</td>\n",
       "      <td>-0.002434</td>\n",
       "      <td>-0.000190</td>\n",
       "      <td>-0.048589</td>\n",
       "      <td>-0.077248</td>\n",
       "      <td>0.028427</td>\n",
       "      <td>-0.079563</td>\n",
       "      <td>-0.003175</td>\n",
       "      <td>-0.094767</td>\n",
       "      <td>0.329554</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.083143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>snowdepth</th>\n",
       "      <td>0.006036</td>\n",
       "      <td>0.000733</td>\n",
       "      <td>0.003602</td>\n",
       "      <td>-0.000112</td>\n",
       "      <td>0.001092</td>\n",
       "      <td>0.000454</td>\n",
       "      <td>0.002716</td>\n",
       "      <td>0.014215</td>\n",
       "      <td>-0.229351</td>\n",
       "      <td>-0.034647</td>\n",
       "      <td>-0.268631</td>\n",
       "      <td>0.000548</td>\n",
       "      <td>-0.170218</td>\n",
       "      <td>-0.023126</td>\n",
       "      <td>0.083143</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     vendor_id  passenger_count  pickup_longitude  \\\n",
       "vendor_id             1.000000         0.287415          0.007820   \n",
       "passenger_count       0.287415         1.000000          0.002169   \n",
       "pickup_longitude      0.007820         0.002169          1.000000   \n",
       "pickup_latitude       0.001742        -0.005125          0.022568   \n",
       "dropoff_longitude     0.001528        -0.000343          0.783582   \n",
       "dropoff_latitude      0.004496        -0.002762          0.100190   \n",
       "trip_duration         0.020304         0.008471          0.026542   \n",
       "pu_hour               0.009299         0.009101          0.010150   \n",
       "yday                 -0.006025        -0.001784          0.004263   \n",
       "wday                  0.001311         0.025191         -0.016356   \n",
       "month                -0.006221        -0.002143          0.004474   \n",
       "distance              0.008109         0.010306          0.259780   \n",
       "average temperature  -0.005760        -0.001203          0.005547   \n",
       "precipitation         0.001385         0.001415          0.001418   \n",
       "snowfall              0.000207         0.000510         -0.003288   \n",
       "snowdepth             0.006036         0.000733          0.003602   \n",
       "\n",
       "                     pickup_latitude  dropoff_longitude  dropoff_latitude  \\\n",
       "vendor_id                   0.001742           0.001528          0.004496   \n",
       "passenger_count            -0.005125          -0.000343         -0.002762   \n",
       "pickup_longitude            0.022568           0.783582          0.100190   \n",
       "pickup_latitude             1.000000           0.114884          0.494038   \n",
       "dropoff_longitude           0.114884           1.000000          0.124873   \n",
       "dropoff_latitude            0.494038           0.124873          1.000000   \n",
       "trip_duration              -0.029204           0.014678         -0.020677   \n",
       "pu_hour                     0.010603          -0.022455          0.013612   \n",
       "yday                       -0.002516           0.004612         -0.001554   \n",
       "wday                       -0.028976          -0.001130         -0.021762   \n",
       "month                      -0.001439           0.004758         -0.000607   \n",
       "distance                   -0.210354           0.134033         -0.142128   \n",
       "average temperature        -0.002011           0.005502         -0.000997   \n",
       "precipitation              -0.004180           0.001660         -0.002710   \n",
       "snowfall                   -0.005058           0.000774         -0.002434   \n",
       "snowdepth                  -0.000112           0.001092          0.000454   \n",
       "\n",
       "                     trip_duration   pu_hour      yday      wday     month  \\\n",
       "vendor_id                 0.020304  0.009299 -0.006025  0.001311 -0.006221   \n",
       "passenger_count           0.008471  0.009101 -0.001784  0.025191 -0.002143   \n",
       "pickup_longitude          0.026542  0.010150  0.004263 -0.016356  0.004474   \n",
       "pickup_latitude          -0.029204  0.010603 -0.002516 -0.028976 -0.001439   \n",
       "dropoff_longitude         0.014678 -0.022455  0.004612 -0.001130  0.004758   \n",
       "dropoff_latitude         -0.020677  0.013612 -0.001554 -0.021762 -0.000607   \n",
       "trip_duration             1.000000  0.003690  0.006625 -0.000708  0.006607   \n",
       "pu_hour                   0.003690  1.000000 -0.003340 -0.084255 -0.003438   \n",
       "yday                      0.006625 -0.003340  1.000000 -0.013615  0.985657   \n",
       "wday                     -0.000708 -0.084255 -0.013615  1.000000 -0.014544   \n",
       "month                     0.006607 -0.003438  0.985657 -0.014544  1.000000   \n",
       "distance                  0.094777 -0.015813  0.016443  0.007599  0.015668   \n",
       "average temperature       0.004574 -0.004948  0.837212 -0.019799  0.814295   \n",
       "precipitation            -0.002590 -0.028656 -0.031407 -0.041673 -0.022469   \n",
       "snowfall                 -0.000190 -0.048589 -0.077248  0.028427 -0.079563   \n",
       "snowdepth                 0.002716  0.014215 -0.229351 -0.034647 -0.268631   \n",
       "\n",
       "                     distance  average temperature  precipitation  snowfall  \\\n",
       "vendor_id            0.008109            -0.005760       0.001385  0.000207   \n",
       "passenger_count      0.010306            -0.001203       0.001415  0.000510   \n",
       "pickup_longitude     0.259780             0.005547       0.001418 -0.003288   \n",
       "pickup_latitude     -0.210354            -0.002011      -0.004180 -0.005058   \n",
       "dropoff_longitude    0.134033             0.005502       0.001660  0.000774   \n",
       "dropoff_latitude    -0.142128            -0.000997      -0.002710 -0.002434   \n",
       "trip_duration        0.094777             0.004574      -0.002590 -0.000190   \n",
       "pu_hour             -0.015813            -0.004948      -0.028656 -0.048589   \n",
       "yday                 0.016443             0.837212      -0.031407 -0.077248   \n",
       "wday                 0.007599            -0.019799      -0.041673  0.028427   \n",
       "month                0.015668             0.814295      -0.022469 -0.079563   \n",
       "distance             1.000000             0.018631       0.001317 -0.003175   \n",
       "average temperature  0.018631             1.000000       0.020519 -0.094767   \n",
       "precipitation        0.001317             0.020519       1.000000  0.329554   \n",
       "snowfall            -0.003175            -0.094767       0.329554  1.000000   \n",
       "snowdepth            0.000548            -0.170218      -0.023126  0.083143   \n",
       "\n",
       "                     snowdepth  \n",
       "vendor_id             0.006036  \n",
       "passenger_count       0.000733  \n",
       "pickup_longitude      0.003602  \n",
       "pickup_latitude      -0.000112  \n",
       "dropoff_longitude     0.001092  \n",
       "dropoff_latitude      0.000454  \n",
       "trip_duration         0.002716  \n",
       "pu_hour               0.014215  \n",
       "yday                 -0.229351  \n",
       "wday                 -0.034647  \n",
       "month                -0.268631  \n",
       "distance              0.000548  \n",
       "average temperature  -0.170218  \n",
       "precipitation        -0.023126  \n",
       "snowfall              0.083143  \n",
       "snowdepth             1.000000  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>total_distance</th>\n",
       "      <th>total_travel_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id2875421</td>\n",
       "      <td>2009.1</td>\n",
       "      <td>164.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id2377394</td>\n",
       "      <td>2513.2</td>\n",
       "      <td>332.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id3504673</td>\n",
       "      <td>1779.4</td>\n",
       "      <td>235.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id2181028</td>\n",
       "      <td>1614.9</td>\n",
       "      <td>140.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id0801584</td>\n",
       "      <td>1393.5</td>\n",
       "      <td>189.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  total_distance  total_travel_time\n",
       "0  id2875421          2009.1              164.9\n",
       "1  id2377394          2513.2              332.0\n",
       "2  id3504673          1779.4              235.8\n",
       "3  id2181028          1614.9              140.1\n",
       "4  id0801584          1393.5              189.4"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fastest1 = pd.read_csv('/home/ankit/Downloads/all/fastest_routes_train_part_1.csv')\n",
    "fastest2 = pd.read_csv('/home/ankit/Downloads/all/fastest_routes_train_part_2.csv')\n",
    "fastest = pd.concat([fastest1,fastest2],ignore_index=True)\n",
    "fastest = fastest.drop(['step_location_list','step_direction','step_maneuvers','travel_time_per_step','distance_per_step','street_for_each_step','number_of_steps','starting_street','end_street'],axis=1)\n",
    "fastest.head() #"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>vendor_id</th>\n",
       "      <th>pickup_datetime</th>\n",
       "      <th>dropoff_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>...</th>\n",
       "      <th>month</th>\n",
       "      <th>distance</th>\n",
       "      <th>average temperature</th>\n",
       "      <th>precipitation</th>\n",
       "      <th>snow fall</th>\n",
       "      <th>snow depth</th>\n",
       "      <th>snowfall</th>\n",
       "      <th>snowdepth</th>\n",
       "      <th>total_distance</th>\n",
       "      <th>total_travel_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id2875421</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-14 17:24:55</td>\n",
       "      <td>2016-03-14 17:32:30</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.982155</td>\n",
       "      <td>40.767937</td>\n",
       "      <td>-73.964630</td>\n",
       "      <td>40.765602</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0.930648</td>\n",
       "      <td>45.5</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2009.1</td>\n",
       "      <td>164.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id2129090</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-03-14 14:05:39</td>\n",
       "      <td>2016-03-14 14:28:05</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.975090</td>\n",
       "      <td>40.758766</td>\n",
       "      <td>-73.953201</td>\n",
       "      <td>40.765068</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>1.224526</td>\n",
       "      <td>45.5</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2782.9</td>\n",
       "      <td>244.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id0256505</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-03-14 15:04:38</td>\n",
       "      <td>2016-03-14 15:16:13</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.994484</td>\n",
       "      <td>40.745087</td>\n",
       "      <td>-73.998993</td>\n",
       "      <td>40.722710</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>1.563173</td>\n",
       "      <td>45.5</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3124.4</td>\n",
       "      <td>419.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id3863815</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-14 04:24:36</td>\n",
       "      <td>2016-03-14 04:37:11</td>\n",
       "      <td>3</td>\n",
       "      <td>-73.944359</td>\n",
       "      <td>40.714489</td>\n",
       "      <td>-73.910530</td>\n",
       "      <td>40.709492</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>1.803944</td>\n",
       "      <td>45.5</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3327.8</td>\n",
       "      <td>280.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id3817493</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-14 14:57:56</td>\n",
       "      <td>2016-03-14 15:15:26</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.952881</td>\n",
       "      <td>40.766468</td>\n",
       "      <td>-73.978630</td>\n",
       "      <td>40.761921</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>1.382826</td>\n",
       "      <td>45.5</td>\n",
       "      <td>0.29</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3354.8</td>\n",
       "      <td>270.3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  vendor_id     pickup_datetime    dropoff_datetime  \\\n",
       "0  id2875421          2 2016-03-14 17:24:55 2016-03-14 17:32:30   \n",
       "1  id2129090          1 2016-03-14 14:05:39 2016-03-14 14:28:05   \n",
       "2  id0256505          1 2016-03-14 15:04:38 2016-03-14 15:16:13   \n",
       "3  id3863815          2 2016-03-14 04:24:36 2016-03-14 04:37:11   \n",
       "4  id3817493          2 2016-03-14 14:57:56 2016-03-14 15:15:26   \n",
       "\n",
       "   passenger_count  pickup_longitude  pickup_latitude  dropoff_longitude  \\\n",
       "0                1        -73.982155        40.767937         -73.964630   \n",
       "1                1        -73.975090        40.758766         -73.953201   \n",
       "2                1        -73.994484        40.745087         -73.998993   \n",
       "3                3        -73.944359        40.714489         -73.910530   \n",
       "4                1        -73.952881        40.766468         -73.978630   \n",
       "\n",
       "   dropoff_latitude store_and_fwd_flag        ...         month  distance  \\\n",
       "0         40.765602                  N        ...             3  0.930648   \n",
       "1         40.765068                  N        ...             3  1.224526   \n",
       "2         40.722710                  N        ...             3  1.563173   \n",
       "3         40.709492                  N        ...             3  1.803944   \n",
       "4         40.761921                  N        ...             3  1.382826   \n",
       "\n",
       "   average temperature  precipitation  snow fall  snow depth  snowfall  \\\n",
       "0                 45.5           0.29        0.0           0       0.0   \n",
       "1                 45.5           0.29        0.0           0       0.0   \n",
       "2                 45.5           0.29        0.0           0       0.0   \n",
       "3                 45.5           0.29        0.0           0       0.0   \n",
       "4                 45.5           0.29        0.0           0       0.0   \n",
       "\n",
       "   snowdepth total_distance total_travel_time  \n",
       "0        0.0         2009.1             164.9  \n",
       "1        0.0         2782.9             244.6  \n",
       "2        0.0         3124.4             419.9  \n",
       "3        0.0         3327.8             280.8  \n",
       "4        0.0         3354.8             270.3  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.merge(df,fastest,on='id',how='outer')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mask = ((df.trip_duration > 60) & (df.distance < 0.05))\n",
    "df = df[~mask]\n",
    "mask = (df.trip_duration < 60) \n",
    "df = df[~mask]\n",
    "mask =  df.trip_duration > 79200\n",
    "df = df[~mask]\n",
    "mask = df.distance/(df.trip_duration/3600) > 60\n",
    "df = df[~mask]\n",
    "df.trip_duration = df.trip_duration.astype(np.uint16)\n",
    "df = df[df.passenger_count > 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "m = df.groupby(['wday','vendor_id'])[['trip_duration']].apply(np.median)\n",
    "m.name = 'trip_duration_median'\n",
    "df = df.join(m, on=['wday','vendor_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sns.lmplot(y='trip_duration_median', x='wday',data=df, fit_reg=False, hue='vendor_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "m = df.groupby(['pu_hour','vendor_id'])[['trip_duration']].apply(np.median)\n",
    "m.name ='trip_duration_median_hour'\n",
    "df = df.join(m, on=['pu_hour','vendor_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#sns.lmplot(y='trip_duration_median_hour', x='pu_hour',data=df, fit_reg=False, hue='vendor_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "jfk_lon = -73.778889\n",
    "jfk_lat = 40.639722\n",
    "lga_lon = -73.872611\n",
    "lga_lat = 40.77725"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['jfk_pickup_dist'] = df.apply(lambda row: haversine(jfk_lon, jfk_lat, row['pickup_longitude'],row['pickup_latitude']), axis=1)\n",
    "df['lga_pickup_dist'] = df.apply(lambda row: haversine(lga_lon, lga_lat, row['pickup_longitude'],row['pickup_latitude']), axis=1)\n",
    "df['jfk_dropoff_dist'] = df.apply(lambda row: haversine(jfk_lon, jfk_lat, row['dropoff_longitude'],row['dropoff_latitude']), axis=1)\n",
    "df['lga_dropoff_dist'] = df.apply(lambda row: haversine(lga_lon, lga_lat, row['dropoff_longitude'],row['dropoff_latitude']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 50)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#fig, ax = plt.subplots(ncols=2, nrows=2, sharex=True)\n",
    "ax[0,0].set_xlim(0,50)\n",
    "\n",
    "#sns.distplot(df['jfk_pickup_dist'],ax=ax[0,0],bins=100,kde=False,hist_kws={'log':True})\n",
    "#sns.distplot(df['jfk_dropoff_dist'],ax=ax[0,1],bins=100,kde=False,hist_kws={'log':True})\n",
    "#sns.distplot(df['lga_pickup_dist'],ax=ax[1,0],bins=100,kde=False,hist_kws={'log':True})\n",
    "#sns.distplot(df['lga_dropoff_dist'],ax=ax[1,1],bins=100,kde=False,hist_kws={'log':True})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>vendor_id</th>\n",
       "      <th>pickup_datetime</th>\n",
       "      <th>dropoff_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>...</th>\n",
       "      <th>snow fall</th>\n",
       "      <th>snow depth</th>\n",
       "      <th>snowfall</th>\n",
       "      <th>snowdepth</th>\n",
       "      <th>total_distance</th>\n",
       "      <th>total_travel_time</th>\n",
       "      <th>trip_duration_median</th>\n",
       "      <th>trip_duration_median_hour</th>\n",
       "      <th>jfk</th>\n",
       "      <th>lga</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id2875421</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-14 17:24:55</td>\n",
       "      <td>2016-03-14 17:32:30</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.982155</td>\n",
       "      <td>40.767937</td>\n",
       "      <td>-73.964630</td>\n",
       "      <td>40.765602</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2009.1</td>\n",
       "      <td>164.9</td>\n",
       "      <td>634.0</td>\n",
       "      <td>709.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id2129090</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-03-14 14:05:39</td>\n",
       "      <td>2016-03-14 14:28:05</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.975090</td>\n",
       "      <td>40.758766</td>\n",
       "      <td>-73.953201</td>\n",
       "      <td>40.765068</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2782.9</td>\n",
       "      <td>244.6</td>\n",
       "      <td>633.0</td>\n",
       "      <td>727.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id0256505</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-03-14 15:04:38</td>\n",
       "      <td>2016-03-14 15:16:13</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.994484</td>\n",
       "      <td>40.745087</td>\n",
       "      <td>-73.998993</td>\n",
       "      <td>40.722710</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3124.4</td>\n",
       "      <td>419.9</td>\n",
       "      <td>633.0</td>\n",
       "      <td>724.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id3863815</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-14 04:24:36</td>\n",
       "      <td>2016-03-14 04:37:11</td>\n",
       "      <td>3</td>\n",
       "      <td>-73.944359</td>\n",
       "      <td>40.714489</td>\n",
       "      <td>-73.910530</td>\n",
       "      <td>40.709492</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3327.8</td>\n",
       "      <td>280.8</td>\n",
       "      <td>634.0</td>\n",
       "      <td>633.5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id3817493</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-14 14:57:56</td>\n",
       "      <td>2016-03-14 15:15:26</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.952881</td>\n",
       "      <td>40.766468</td>\n",
       "      <td>-73.978630</td>\n",
       "      <td>40.761921</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3354.8</td>\n",
       "      <td>270.3</td>\n",
       "      <td>634.0</td>\n",
       "      <td>735.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  vendor_id     pickup_datetime    dropoff_datetime  \\\n",
       "0  id2875421          2 2016-03-14 17:24:55 2016-03-14 17:32:30   \n",
       "1  id2129090          1 2016-03-14 14:05:39 2016-03-14 14:28:05   \n",
       "2  id0256505          1 2016-03-14 15:04:38 2016-03-14 15:16:13   \n",
       "3  id3863815          2 2016-03-14 04:24:36 2016-03-14 04:37:11   \n",
       "4  id3817493          2 2016-03-14 14:57:56 2016-03-14 15:15:26   \n",
       "\n",
       "   passenger_count  pickup_longitude  pickup_latitude  dropoff_longitude  \\\n",
       "0                1        -73.982155        40.767937         -73.964630   \n",
       "1                1        -73.975090        40.758766         -73.953201   \n",
       "2                1        -73.994484        40.745087         -73.998993   \n",
       "3                3        -73.944359        40.714489         -73.910530   \n",
       "4                1        -73.952881        40.766468         -73.978630   \n",
       "\n",
       "   dropoff_latitude store_and_fwd_flag  ...    snow fall  snow depth  \\\n",
       "0         40.765602                  N  ...          0.0           0   \n",
       "1         40.765068                  N  ...          0.0           0   \n",
       "2         40.722710                  N  ...          0.0           0   \n",
       "3         40.709492                  N  ...          0.0           0   \n",
       "4         40.761921                  N  ...          0.0           0   \n",
       "\n",
       "   snowfall  snowdepth  total_distance  total_travel_time  \\\n",
       "0       0.0        0.0          2009.1              164.9   \n",
       "1       0.0        0.0          2782.9              244.6   \n",
       "2       0.0        0.0          3124.4              419.9   \n",
       "3       0.0        0.0          3327.8              280.8   \n",
       "4       0.0        0.0          3354.8              270.3   \n",
       "\n",
       "   trip_duration_median  trip_duration_median_hour    jfk    lga  \n",
       "0                 634.0                      709.0  False  False  \n",
       "1                 633.0                      727.0  False  False  \n",
       "2                 633.0                      724.0  False  False  \n",
       "3                 634.0                      633.5  False  False  \n",
       "4                 634.0                      735.0  False  False  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['jfk'] = ((df['jfk_pickup_dist'] < 2) | (df['jfk_dropoff_dist'] < 2))\n",
    "df['lga'] = ((df['lga_pickup_dist'] < 2) | (df['lga_dropoff_dist'] < 2))\n",
    "df = df.drop(['jfk_pickup_dist','lga_pickup_dist','jfk_dropoff_dist','lga_dropoff_dist'],axis=1)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>vendor_id</th>\n",
       "      <th>pickup_datetime</th>\n",
       "      <th>dropoff_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>...</th>\n",
       "      <th>snow depth</th>\n",
       "      <th>snowfall</th>\n",
       "      <th>snowdepth</th>\n",
       "      <th>total_distance</th>\n",
       "      <th>total_travel_time</th>\n",
       "      <th>trip_duration_median</th>\n",
       "      <th>trip_duration_median_hour</th>\n",
       "      <th>jfk</th>\n",
       "      <th>lga</th>\n",
       "      <th>workday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id2875421</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-14 17:24:55</td>\n",
       "      <td>2016-03-14 17:32:30</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.982155</td>\n",
       "      <td>40.767937</td>\n",
       "      <td>-73.964630</td>\n",
       "      <td>40.765602</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2009.1</td>\n",
       "      <td>164.9</td>\n",
       "      <td>634.0</td>\n",
       "      <td>709.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id2129090</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-03-14 14:05:39</td>\n",
       "      <td>2016-03-14 14:28:05</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.975090</td>\n",
       "      <td>40.758766</td>\n",
       "      <td>-73.953201</td>\n",
       "      <td>40.765068</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2782.9</td>\n",
       "      <td>244.6</td>\n",
       "      <td>633.0</td>\n",
       "      <td>727.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id0256505</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-03-14 15:04:38</td>\n",
       "      <td>2016-03-14 15:16:13</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.994484</td>\n",
       "      <td>40.745087</td>\n",
       "      <td>-73.998993</td>\n",
       "      <td>40.722710</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3124.4</td>\n",
       "      <td>419.9</td>\n",
       "      <td>633.0</td>\n",
       "      <td>724.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id3863815</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-14 04:24:36</td>\n",
       "      <td>2016-03-14 04:37:11</td>\n",
       "      <td>3</td>\n",
       "      <td>-73.944359</td>\n",
       "      <td>40.714489</td>\n",
       "      <td>-73.910530</td>\n",
       "      <td>40.709492</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3327.8</td>\n",
       "      <td>280.8</td>\n",
       "      <td>634.0</td>\n",
       "      <td>633.5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id3817493</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-14 14:57:56</td>\n",
       "      <td>2016-03-14 15:15:26</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.952881</td>\n",
       "      <td>40.766468</td>\n",
       "      <td>-73.978630</td>\n",
       "      <td>40.761921</td>\n",
       "      <td>N</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3354.8</td>\n",
       "      <td>270.3</td>\n",
       "      <td>634.0</td>\n",
       "      <td>735.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  vendor_id     pickup_datetime    dropoff_datetime  \\\n",
       "0  id2875421          2 2016-03-14 17:24:55 2016-03-14 17:32:30   \n",
       "1  id2129090          1 2016-03-14 14:05:39 2016-03-14 14:28:05   \n",
       "2  id0256505          1 2016-03-14 15:04:38 2016-03-14 15:16:13   \n",
       "3  id3863815          2 2016-03-14 04:24:36 2016-03-14 04:37:11   \n",
       "4  id3817493          2 2016-03-14 14:57:56 2016-03-14 15:15:26   \n",
       "\n",
       "   passenger_count  pickup_longitude  pickup_latitude  dropoff_longitude  \\\n",
       "0                1        -73.982155        40.767937         -73.964630   \n",
       "1                1        -73.975090        40.758766         -73.953201   \n",
       "2                1        -73.994484        40.745087         -73.998993   \n",
       "3                3        -73.944359        40.714489         -73.910530   \n",
       "4                1        -73.952881        40.766468         -73.978630   \n",
       "\n",
       "   dropoff_latitude store_and_fwd_flag   ...    snow depth  snowfall  \\\n",
       "0         40.765602                  N   ...             0       0.0   \n",
       "1         40.765068                  N   ...             0       0.0   \n",
       "2         40.722710                  N   ...             0       0.0   \n",
       "3         40.709492                  N   ...             0       0.0   \n",
       "4         40.761921                  N   ...             0       0.0   \n",
       "\n",
       "   snowdepth  total_distance  total_travel_time  trip_duration_median  \\\n",
       "0        0.0          2009.1              164.9                 634.0   \n",
       "1        0.0          2782.9              244.6                 633.0   \n",
       "2        0.0          3124.4              419.9                 633.0   \n",
       "3        0.0          3327.8              280.8                 634.0   \n",
       "4        0.0          3354.8              270.3                 634.0   \n",
       "\n",
       "   trip_duration_median_hour    jfk    lga workday  \n",
       "0                      709.0  False  False    True  \n",
       "1                      727.0  False  False    True  \n",
       "2                      724.0  False  False    True  \n",
       "3                      633.5  False  False   False  \n",
       "4                      735.0  False  False    True  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['workday'] = ((df['pu_hour'] > 8) & (df['pu_hour'] < 18))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "f396143a-380f-4d90-8a25-99b5a2d91454",
    "_execution_state": "idle",
    "_uuid": "53eb8baab05d86229fbaa11c5947f6ec085f9726"
   },
   "source": [
    "## Locations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "_cell_guid": "6067f4ea-649d-438b-8de3-deab1a8438c1",
    "_execution_state": "idle",
    "_uuid": "f700181673094d1b8e3cbce2f285ccc2c5000be7"
   },
   "outputs": [],
   "source": [
    "#fig, ax = plt.subplots(ncols=1, nrows=1,figsize=(12,10))\n",
    "#plt.ylim(40.6, 40.9)\n",
    "#plt.xlim(-74.1,-73.7)\n",
    "#ax.scatter(df['pickup_longitude'],df['pickup_latitude'], s=0.01, alpha=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RMSLE: Evaluation Metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def rmsle(evaluator,X,real):\n",
    "    sum = 0.0\n",
    "    predicted = evaluator.predict(X)\n",
    "    print(\"Number predicted less than 0: {}\".format(np.where(predicted < 0)[0].shape))\n",
    "\n",
    "    predicted[predicted < 0] = 0\n",
    "    for x in range(len(predicted)):\n",
    "        p = np.log(predicted[x]+1)\n",
    "        r = np.log(real[x]+1)\n",
    "        sum = sum + (p-r)**2\n",
    "    return (sum/len(predicted))**0.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "_cell_guid": "f36d65a0-9179-4e0e-b893-fad641510ca5",
    "_execution_state": "idle",
    "_uuid": "ff6a80de90c26ecb63b02f197f72aed5f1fc4414",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tdf = pd.read_csv('/home/ankit/Downloads/all/test.csv')\n",
    "tdf.pickup_datetime=pd.to_datetime(tdf.pickup_datetime)\n",
    "#tdf.dropoff_datetime=pd.to_datetime(tdf.dropoff_datetime)\n",
    "tdf['pu_hour'] = tdf.pickup_datetime.dt.hour\n",
    "tdf['yday'] = tdf.pickup_datetime.dt.dayofyear\n",
    "tdf['wday'] = tdf.pickup_datetime.dt.dayofweek\n",
    "tdf['month'] = tdf.pickup_datetime.dt.month\n",
    "tdf['distance'] = haversine(tdf.pickup_longitude, tdf.pickup_latitude,\n",
    "                                           tdf.dropoff_longitude, tdf.dropoff_latitude)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fastest_test = pd.read_csv('/home/ankit/Downloads/all/fastest_routes_test.csv')\n",
    "tdf = pd.merge(tdf,fastest_test,on='id',how='outer')\n",
    "tdf = tdf.drop(['step_location_list','step_direction','step_maneuvers','travel_time_per_step','distance_per_step','street_for_each_step','number_of_steps','starting_street','end_street'],axis=1)\n",
    "tdf = pd.merge(tdf,wdf,on='yday')\n",
    "tdf = tdf.drop(['date','maximum temperature','minimum temperature'],axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tdf['jfk_pickup_dist'] = tdf.apply(lambda row: haversine(jfk_lon, jfk_lat, row['pickup_longitude'],row['pickup_latitude']), axis=1)\n",
    "tdf['lga_pickup_dist'] = tdf.apply(lambda row: haversine(lga_lon, lga_lat, row['pickup_longitude'],row['pickup_latitude']), axis=1)\n",
    "tdf['jfk_dropoff_dist'] = tdf.apply(lambda row: haversine(jfk_lon, jfk_lat, row['dropoff_longitude'],row['dropoff_latitude']), axis=1)\n",
    "tdf['lga_dropoff_dist'] = tdf.apply(lambda row: haversine(lga_lon, lga_lat, row['dropoff_longitude'],row['dropoff_latitude']), axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tdf['jfk'] = ((tdf['jfk_pickup_dist'] < 2) | (tdf['jfk_dropoff_dist'] < 2))\n",
    "tdf['lga'] = ((tdf['lga_pickup_dist'] < 2) | (tdf['lga_dropoff_dist'] < 2))\n",
    "tdf = tdf.drop(['jfk_pickup_dist','lga_pickup_dist','jfk_dropoff_dist','lga_dropoff_dist'],axis=1)\n",
    "tdf['workday'] = ((tdf['pu_hour'] > 8) & (tdf['pu_hour'] < 18))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tdf['snowfall'] = tdf['snow fall'].replace(['T'],0.05)\n",
    "tdf['precipitation'] = tdf['precipitation'].replace(['T'],0.05)\n",
    "tdf['snowdepth'] = tdf['snow depth'].replace(['T'],0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>vendor_id</th>\n",
       "      <th>pickup_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>pu_hour</th>\n",
       "      <th>...</th>\n",
       "      <th>total_travel_time</th>\n",
       "      <th>average temperature</th>\n",
       "      <th>precipitation</th>\n",
       "      <th>snow fall</th>\n",
       "      <th>snow depth</th>\n",
       "      <th>snowfall</th>\n",
       "      <th>snowdepth</th>\n",
       "      <th>jfk</th>\n",
       "      <th>lga</th>\n",
       "      <th>workday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id3004672</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-30 23:59:58</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.988129</td>\n",
       "      <td>40.732029</td>\n",
       "      <td>-73.990173</td>\n",
       "      <td>40.756680</td>\n",
       "      <td>N</td>\n",
       "      <td>23</td>\n",
       "      <td>...</td>\n",
       "      <td>424.6</td>\n",
       "      <td>76.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id3505355</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-30 23:59:53</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.964203</td>\n",
       "      <td>40.679993</td>\n",
       "      <td>-73.959808</td>\n",
       "      <td>40.655403</td>\n",
       "      <td>N</td>\n",
       "      <td>23</td>\n",
       "      <td>...</td>\n",
       "      <td>200.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id1217141</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-30 23:59:47</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.997437</td>\n",
       "      <td>40.737583</td>\n",
       "      <td>-73.986160</td>\n",
       "      <td>40.729523</td>\n",
       "      <td>N</td>\n",
       "      <td>23</td>\n",
       "      <td>...</td>\n",
       "      <td>193.2</td>\n",
       "      <td>76.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id2150126</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-06-30 23:59:41</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.956070</td>\n",
       "      <td>40.771900</td>\n",
       "      <td>-73.986427</td>\n",
       "      <td>40.730469</td>\n",
       "      <td>N</td>\n",
       "      <td>23</td>\n",
       "      <td>...</td>\n",
       "      <td>494.8</td>\n",
       "      <td>76.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id1598245</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-30 23:59:33</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.970215</td>\n",
       "      <td>40.761475</td>\n",
       "      <td>-73.961510</td>\n",
       "      <td>40.755890</td>\n",
       "      <td>N</td>\n",
       "      <td>23</td>\n",
       "      <td>...</td>\n",
       "      <td>103.2</td>\n",
       "      <td>76.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  vendor_id     pickup_datetime  passenger_count  \\\n",
       "0  id3004672          1 2016-06-30 23:59:58                1   \n",
       "1  id3505355          1 2016-06-30 23:59:53                1   \n",
       "2  id1217141          1 2016-06-30 23:59:47                1   \n",
       "3  id2150126          2 2016-06-30 23:59:41                1   \n",
       "4  id1598245          1 2016-06-30 23:59:33                1   \n",
       "\n",
       "   pickup_longitude  pickup_latitude  dropoff_longitude  dropoff_latitude  \\\n",
       "0        -73.988129        40.732029         -73.990173         40.756680   \n",
       "1        -73.964203        40.679993         -73.959808         40.655403   \n",
       "2        -73.997437        40.737583         -73.986160         40.729523   \n",
       "3        -73.956070        40.771900         -73.986427         40.730469   \n",
       "4        -73.970215        40.761475         -73.961510         40.755890   \n",
       "\n",
       "  store_and_fwd_flag  pu_hour   ...    total_travel_time  average temperature  \\\n",
       "0                  N       23   ...                424.6                 76.0   \n",
       "1                  N       23   ...                200.0                 76.0   \n",
       "2                  N       23   ...                193.2                 76.0   \n",
       "3                  N       23   ...                494.8                 76.0   \n",
       "4                  N       23   ...                103.2                 76.0   \n",
       "\n",
       "   precipitation  snow fall  snow depth  snowfall  snowdepth    jfk    lga  \\\n",
       "0            0.0        0.0           0       0.0          0  False  False   \n",
       "1            0.0        0.0           0       0.0          0  False  False   \n",
       "2            0.0        0.0           0       0.0          0  False  False   \n",
       "3            0.0        0.0           0       0.0          0  False  False   \n",
       "4            0.0        0.0           0       0.0          0  False  False   \n",
       "\n",
       "  workday  \n",
       "0   False  \n",
       "1   False  \n",
       "2   False  \n",
       "3   False  \n",
       "4   False  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.to_csv('/home/ankit/Downloads/all/train_data.csv',index=False)\n",
    "tdf.to_csv('/home/ankit/Downloads/all/test_data.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ankit/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py:2698: DtypeWarning: Columns (18,19) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('/home/ankit/Downloads/all/train_data.csv')\n",
    "tdf = pd.read_csv('/home/ankit/Downloads/all/test_data.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Benchmark Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>trip_duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id0771704</td>\n",
       "      <td>200.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id3274209</td>\n",
       "      <td>141.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id2756455</td>\n",
       "      <td>324.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id3684027</td>\n",
       "      <td>84.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id3101285</td>\n",
       "      <td>294.7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  trip_duration\n",
       "0  id0771704          200.2\n",
       "1  id3274209          141.5\n",
       "2  id2756455          324.6\n",
       "3  id3684027           84.2\n",
       "4  id3101285          294.7"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "benchmark = fastest_test[['id','total_travel_time']]\n",
    "benchmark = benchmark.rename(index=str, columns={\"total_travel_time\": \"trip_duration\"})\n",
    "benchmark.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "benchmark['trip_duration'].isnull().values.any()\n",
    "benchmark.to_csv('benchmark.csv',index=False)\n",
    "#RMSLE=0.990"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "_cell_guid": "da4346dd-73a7-4412-8222-3fc122e4020c",
    "_execution_state": "idle",
    "_uuid": "39ebf018303a0b31cea169a5bc184d9290ea249a",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features = df[['vendor_id','passenger_count','pickup_latitude','pickup_longitude','dropoff_latitude','dropoff_longitude','pu_hour','wday','month','workday','precipitation','snowfall','snowdepth','total_distance','total_travel_time','jfk','lga']]\n",
    "target = df['trip_duration']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "_cell_guid": "58acd8d6-8b32-43ba-bf48-7e16d5c52244",
    "_execution_state": "idle",
    "_uuid": "1035773aa16ec621683056b10197c4ce36bf500a",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tfeatures = tdf[['vendor_id','passenger_count','pickup_latitude','pickup_longitude','dropoff_latitude','dropoff_longitude','pu_hour','wday','month','workday','precipitation','snowfall','snowdepth','total_distance','total_travel_time','jfk','lga']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "b0bbeea7-80a5-450b-864e-6f540293b8e2",
    "_execution_state": "idle",
    "_uuid": "790a8b8ce345eb618bbef88fe8fec940abf8c75c"
   },
   "source": [
    "## Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "_cell_guid": "3efc1ba1-55cb-4039-b8d3-991e43f6c22e",
    "_execution_state": "idle",
    "_uuid": "1785c513750bbef82f98065c4dc9b4deeee2dd5f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number predicted less than 0: (14,)\n",
      "Number predicted less than 0: (9,)\n",
      "Number predicted less than 0: (17,)\n",
      "Number predicted less than 0: (12,)\n",
      "[ 0.43999626  0.44022776  0.43897465  0.44073689]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import linear_model\n",
    "\n",
    "reg = linear_model.LinearRegression()\n",
    "cv = ShuffleSplit(n_splits=4, test_size=0.1, random_state=0)\n",
    "print(cross_val_score(reg, features, np.ravel(target), cv=cv, scoring=rmsle))\n",
    "reg.fit(features, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.43998374750000002"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean([0.43999617,  0.44022755 , 0.43897449,0.44073678])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(625134, 17)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfeatures.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(490,)\n"
     ]
    }
   ],
   "source": [
    "pred = reg.predict(tfeatures)\n",
    "print(np.where(pred < 0)[0].shape)\n",
    "pred[pred < 0]=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "_cell_guid": "1baff0f1-e3cf-40b4-af2d-2a3856dfbc81",
    "_execution_state": "idle",
    "_uuid": "70381e4a5b6a713d2dc671ce72686149210d790f",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tdf['trip_duration']=pred.astype(int)\n",
    "out = tdf[['id','trip_duration']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "_cell_guid": "0721f21f-2888-4ab3-b3bd-f54968156bdb",
    "_execution_state": "idle",
    "_uuid": "74f10d64578769e0eddbe133fb3ca32f6f26e28b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out['trip_duration'].isnull().values.any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "_cell_guid": "31941b96-6946-47d6-8b72-28842f417e30",
    "_execution_state": "idle",
    "_uuid": "3ca4d926dc29d9767614e8e05dd313379ae5bdb5",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "out.to_csv('pred_linear.csv',index=False)\n",
    "#RMSLE=0.535"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## K-nearest Neighbors Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "_cell_guid": "55f8b4ca-315d-402d-a85b-f18d1f521c54",
    "_execution_state": "idle",
    "_uuid": "6db76ebe986844f2484ab081ece300d4b0f8fa8f",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number predicted less than 0: (0,)\n",
      "Number predicted less than 0: (0,)\n",
      "Number predicted less than 0: (0,)\n",
      "Number predicted less than 0: (0,)\n",
      "[ 0.42011203  0.4194096   0.41947696  0.41968832]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "KNeighborsRegressor(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "          metric_params=None, n_jobs=1, n_neighbors=10, p=2,\n",
       "          weights='uniform')"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "neigh = KNeighborsRegressor(n_neighbors=10)\n",
    "cv = ShuffleSplit(n_splits=4, test_size=0.1, random_state=0)\n",
    "print(cross_val_score(neigh, features, np.ravel(target), cv=cv,scoring=rmsle))\n",
    "neigh.fit(features,target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.41967126500000002"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean([0.42010954, 0.41940803, 0.41947931, 0.41968818])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0,)\n"
     ]
    }
   ],
   "source": [
    "pred = neigh.predict(tfeatures)\n",
    "print(np.where(pred < 0)[0].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "_cell_guid": "04e4bf99-8113-42f5-ba0d-a419e2842afe",
    "_execution_state": "idle",
    "_uuid": "849b36bc0db626910c1ad2b4657e6cb944ddc748",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tdf['trip_duration']=pred.astype(int)\n",
    "out = tdf[['id','trip_duration']]\n",
    "out.to_csv('pred_knn.csv',index=False)\n",
    "#RMSLE=0.505"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number predicted less than 0: (0,)\n",
      "Number predicted less than 0: (0,)\n",
      "Number predicted less than 0: (0,)\n",
      "Number predicted less than 0: (0,)\n",
      "[ 0.349922    0.35009407  0.34966408  0.34969   ]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "rf = RandomForestRegressor()\n",
    "cv = ShuffleSplit(n_splits=4, test_size=0.1, random_state=0)\n",
    "print(cross_val_score(rf, features, np.ravel(target), cv=cv,scoring=rmsle))\n",
    "rf = rf.fit(features,np.ravel(target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.35075301749999999"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean([0.35085909, 0.35181223, 0.34976928, 0.35057147])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0,)\n"
     ]
    }
   ],
   "source": [
    "pred = rf.predict(tfeatures)\n",
    "print(np.where(pred < 0)[0].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tdf['trip_duration']=pred.astype(int)\n",
    "out = tdf[['id','trip_duration']]\n",
    "out.to_csv('pred_rf.csv',index=False)\n",
    "#RMSLE=0.473"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "_cell_guid": "6e34d8d0-402d-4115-b00a-4d56414d2085",
    "_execution_state": "idle",
    "_uuid": "12c7db13fe177d3573b0b3ec0c260738ecc56f4c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number predicted less than 0: (1,)\n",
      "Number predicted less than 0: (0,)\n",
      "Number predicted less than 0: (0,)\n",
      "Number predicted less than 0: (0,)\n",
      "[ 0.34347665  0.34339981  0.34196098  0.34286028]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.08, max_delta_step=0,\n",
       "       max_depth=7, min_child_weight=1, missing=None, n_estimators=100,\n",
       "       n_jobs=1, nthread=None, objective='reg:linear', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "       silent=True, subsample=0.75)"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg = xgboost.XGBRegressor(n_estimators=100, learning_rate=0.08, gamma=0, subsample=0.75,\n",
    "                           colsample_bytree=1, max_depth=7)\n",
    "\n",
    "cv = ShuffleSplit(n_splits=4, test_size=0.1, random_state=0)\n",
    "print(cross_val_score(reg, features, np.ravel(target), cv=cv,scoring=rmsle))\n",
    "reg.fit(features,target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0,)\n"
     ]
    }
   ],
   "source": [
    "pred = reg.predict(tfeatures)\n",
    "print(np.where(pred < 0)[0].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "_cell_guid": "ac0340eb-fdfd-4998-a698-a8f440b1e632",
    "_execution_state": "idle",
    "_uuid": "723ceeca2a5f05925711eda23e672d38a01b1d8f",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred[pred < 0] = 0\n",
    "tdf['trip_duration']=pred.astype(int)\n",
    "out = tdf[['id','trip_duration']]\n",
    "out['trip_duration'].isnull().values.any()\n",
    "out.to_csv('pred_xgboost.csv',index=False)\n",
    "#RMSLE=0.463"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f71db5fe390>"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAABcCAYAAAAh+sH1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAABpVJREFUeJzt3T1oE38cx/HvpfUfRMSnDhFTDITAoeIDKqiVLopTs4ZKwEW6Cq5OZhBErINuhS5dlDp2tCIIxShodRAEsSIVHyiisT5QfOh/0JyX3LVN0rv7fS95v5Ymzd3vPk3T++SXpr9ai4uLiwIAgGEJ0wEAABChkAAASlBIAAAVKCQAgAoUEgBABQoJAKAChQQAUIFCAgCoQCEBAFSgkAAAKlBIAAAVKCQAgAoUEgBABQoJiKHBwUHTEYDAWfz7CQCABsyQAAAqUEgAABUoJCBEyaTpBMuYmDCdAKhBIQEhWlgwnWAZ+XxgQ509ezawsdC5eFMDAEAFZkjoSNms6QQA6lFIaDs7d/67fO7cOd9tXryIKAyAhvGSHQBzJidFjh83nQJKUEgAABV4yQ5Ae5iZMZ0Aq8QMCUBnqVRENmwwnQI+mCGhxuSk6QRAyAIoo/v37wcQBPWYIQEAVGCGBABQgUICAKhAIQFY1u7dfz5++PDBbJB2wxsrPPgdkkJPn9auNgAAnYAZkkKUEUwZGTGdAJ2MGRIAQAVmSAZMT0+bjgAA6jBDQuxks6zWDbQjCgkAoAIv2QEAVKCQgAi8efPGdITYKRaLpiMgYrxk16b6+/vl7t27pmMgBvbv3y8PHz40HQOgkNrNjRsig4OmUwABmZ4W2bfPdApEhJfsWjA2NmY6gq/z51svoyNHjgSaBTH0+rXpBF6UUUehkJo0Pz8vp06danq/ri6RixcvhpDon1LJannfiYmJAJM0JpFIiGW1nrled3d3YGO5ff36VUREKpXKqsZJpVKBfr31BgYGWtpv8+bNf3Kl04FlCeLrrB/j+/fvqx6zUS9fvmxpv2QyGXASr/fv3ze9z8DAQO3j4+DBABMFh5fsACU+ffokGzdujOx469evl/n5+ciOB6yEGdISbt40nQCdJsoyEpEVy+jHjx8RJUEz0gHOZKMcuxHMkAAEYt26dc7Lm0ArmCHBiN+/f5uO0Ja6urqMHbvTy2jr1q2mI8QehdSCpk+m4+PhBFlBU7/kv3o1vCA+EgkeemH49etXQ9tt2rQp8GPfvh34kLHy6tVb0xFij7NCC+JyMv3582fjG585E16QdmXgnYlB+fjxY+BjHjsW+JCx8t9/phPEXzzOrDEVwpNQaJLPm04AtBUKKUQhPAldlTt37piOoA5rzEVrZmbGdITwbdnS8q7Xrl1zLq9ZsyaINLHCu+ywpNHRUTl9+rTpGMCKrl+/LidPnjQdA6vEDKlJYa+2sJIDBw5Ediy/Mnr8+HFkxwcapamM5ubmTEeILWZIgMuTJ09k27Zt0tPTYzpKw/L5vJGln5bz7t07SaVSpmMgZjp2hhSTN8rF1qVLl1ra78uXL0vedutWq2kaMzIyInv27JGenh7ZsWNHuAcLSCKRqC2jBp9fHjoUUqC/UqmU3Lt3L9yDwEPrws+NYoYUJ5bV8AknDpJJkYUF0yngkU6rWvn7+XORXM50CkSh4+YJuVxOent7RUTk6NGjzufHxsakXC7LiRMnJJPJiEjtasPVbaufK5fLYlmWcz2bzTqrgFuWVbMycalUci77rYJ84cIF36yFQqHmemb7dufy4cOHJZvNytq1az37DA8PS7FYFNu2pa+vz3Nsd4apqama63Nzc/Ls2TPfvLZte/Ivte1Sa2I9evTI2bZaRuVy2Xdbt/7+fs8x6s3Ozkoul5NyuSyFQkEuX77s3FapVMS2bRER5/uf+3uWGx8fr9nWfbzq/ZtznRHrt63/A+ShoSEn565du+TKlSu+2S3LkqGhIc/3sHpbNWe93t5e+fz5swwPD3seI27FYtFzTPd6efW3Oc9NfcpopdW73eveWZYlU1NTzvW9e/cum+HtW/8/KK3+PFbvesuyPP9F1rIs6evrk+7u7pqM2WzWc/8VCgVnxfbqY6G6rXvfTCbjXP/27ZtnXPdj2G10dLTmev3PRvXr9Lsv6++/Ktu2PeO6z0XpdFpKpZLYti2zs7POWA8ePPCM5f6eLMXvHBElZkgAABU6boYEANCJQgIAqEAhAQBUoJAAACpQSAAAFSgkAIAKFBIAQAUKCQCgAoUEAFCBQgIAqEAhAQBUoJAAACpQSAAAFSgkAIAKFBIAQAUKCQCgAoUEAFCBQgIAqEAhAQBUoJAAACpQSAAAFSgkAIAKFBIAQAUKCQCgAoUEAFCBQgIAqEAhAQBUoJAAACpQSAAAFSgkAIAKFBIAQAUKCQCgAoUEAFDhf5e9h/LYv+jKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from xgboost import plot_tree\n",
    "plot_tree(reg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "pickle.dump(reg, open('xgb_model.sav','wb'),protocol=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}