Modeling and scikit-learn.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:d5b0c10fdc63ce3d2a6f2e16868726a726586d51342c1951eb276665a921c8ff"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "############################################################\n",
      "# This is a basic walkthrough tutorial on building\n",
      "# predictive models with python and sklearn.\n",
      "# Author: Everett Wetchler\n",
      "# Copyright 2014 Bayes Impact\n",
      "############################################################"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%matplotlib inline\n",
      "# Definitely recommend that you create a file like this,\n",
      "# so you don't end up copy/pasting large import blocks\n",
      "# every time you start a new notebook.\n",
      "# If you place this file in ~/.ipython/profile_default/startup\n",
      "# it will run every time by default.\n",
      "%run lib/massimport.ipy"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "############################################################\n",
      "# Part 1: Basics\n",
      "############################################################"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# The first step is to create your dataframe, however you have to.\n",
      "# Normally this will be loading from csv, or querying a database, etc.\n",
      "# This dataframe will become your feature matrix. Here, we'll use\n",
      "# one of sklearn's builtin example datasets.\n",
      "\n",
      "iris = datasets.load_iris()\n",
      "df = pd.DataFrame(iris.data, columns=iris.feature_names)\n",
      "df['species'] = pd.Series(iris['target']).apply(lambda x: iris.target_names[x])\n",
      "df.ix[np.random.choice(df.index, 10)]  # Print a sample of rows"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>sepal length (cm)</th>\n",
        "      <th>sepal width (cm)</th>\n",
        "      <th>petal length (cm)</th>\n",
        "      <th>petal width (cm)</th>\n",
        "      <th>species</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>64 </th>\n",
        "      <td> 5.6</td>\n",
        "      <td> 2.9</td>\n",
        "      <td> 3.6</td>\n",
        "      <td> 1.3</td>\n",
        "      <td> versicolor</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>45 </th>\n",
        "      <td> 4.8</td>\n",
        "      <td> 3.0</td>\n",
        "      <td> 1.4</td>\n",
        "      <td> 0.3</td>\n",
        "      <td>     setosa</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>35 </th>\n",
        "      <td> 5.0</td>\n",
        "      <td> 3.2</td>\n",
        "      <td> 1.2</td>\n",
        "      <td> 0.2</td>\n",
        "      <td>     setosa</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>59 </th>\n",
        "      <td> 5.2</td>\n",
        "      <td> 2.7</td>\n",
        "      <td> 3.9</td>\n",
        "      <td> 1.4</td>\n",
        "      <td> versicolor</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>28 </th>\n",
        "      <td> 5.2</td>\n",
        "      <td> 3.4</td>\n",
        "      <td> 1.4</td>\n",
        "      <td> 0.2</td>\n",
        "      <td>     setosa</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>91 </th>\n",
        "      <td> 6.1</td>\n",
        "      <td> 3.0</td>\n",
        "      <td> 4.6</td>\n",
        "      <td> 1.4</td>\n",
        "      <td> versicolor</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>9  </th>\n",
        "      <td> 4.9</td>\n",
        "      <td> 3.1</td>\n",
        "      <td> 1.5</td>\n",
        "      <td> 0.1</td>\n",
        "      <td>     setosa</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>143</th>\n",
        "      <td> 6.8</td>\n",
        "      <td> 3.2</td>\n",
        "      <td> 5.9</td>\n",
        "      <td> 2.3</td>\n",
        "      <td>  virginica</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>65 </th>\n",
        "      <td> 6.7</td>\n",
        "      <td> 3.1</td>\n",
        "      <td> 4.4</td>\n",
        "      <td> 1.4</td>\n",
        "      <td> versicolor</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>90 </th>\n",
        "      <td> 5.5</td>\n",
        "      <td> 2.6</td>\n",
        "      <td> 4.4</td>\n",
        "      <td> 1.2</td>\n",
        "      <td> versicolor</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 4,
       "text": [
        "     sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)  \\\n",
        "64                 5.6               2.9                3.6               1.3   \n",
        "45                 4.8               3.0                1.4               0.3   \n",
        "35                 5.0               3.2                1.2               0.2   \n",
        "59                 5.2               2.7                3.9               1.4   \n",
        "28                 5.2               3.4                1.4               0.2   \n",
        "91                 6.1               3.0                4.6               1.4   \n",
        "9                  4.9               3.1                1.5               0.1   \n",
        "143                6.8               3.2                5.9               2.3   \n",
        "65                 6.7               3.1                4.4               1.4   \n",
        "90                 5.5               2.6                4.4               1.2   \n",
        "\n",
        "        species  \n",
        "64   versicolor  \n",
        "45       setosa  \n",
        "35       setosa  \n",
        "59   versicolor  \n",
        "28       setosa  \n",
        "91   versicolor  \n",
        "9        setosa  \n",
        "143   virginica  \n",
        "65   versicolor  \n",
        "90   versicolor  "
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# To make this a classification problem, we need a binary output label.\n",
      "# For this dataset there are 3 types of species (setosa, vriginica, and veriscolor)\n",
      "# Let's try to predict if an iris is a setotsa based on the measurements\n",
      "df['is_setosa'] = df['species'] == 'setosa'\n",
      "df.ix[np.random.choice(df.index, 10)]  # Print a sample of rows"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>sepal length (cm)</th>\n",
        "      <th>sepal width (cm)</th>\n",
        "      <th>petal length (cm)</th>\n",
        "      <th>petal width (cm)</th>\n",
        "      <th>species</th>\n",
        "      <th>is_setosa</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>87 </th>\n",
        "      <td> 6.3</td>\n",
        "      <td> 2.3</td>\n",
        "      <td> 4.4</td>\n",
        "      <td> 1.3</td>\n",
        "      <td> versicolor</td>\n",
        "      <td> False</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>69 </th>\n",
        "      <td> 5.6</td>\n",
        "      <td> 2.5</td>\n",
        "      <td> 3.9</td>\n",
        "      <td> 1.1</td>\n",
        "      <td> versicolor</td>\n",
        "      <td> False</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>87 </th>\n",
        "      <td> 6.3</td>\n",
        "      <td> 2.3</td>\n",
        "      <td> 4.4</td>\n",
        "      <td> 1.3</td>\n",
        "      <td> versicolor</td>\n",
        "      <td> False</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>94 </th>\n",
        "      <td> 5.6</td>\n",
        "      <td> 2.7</td>\n",
        "      <td> 4.2</td>\n",
        "      <td> 1.3</td>\n",
        "      <td> versicolor</td>\n",
        "      <td> False</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>104</th>\n",
        "      <td> 6.5</td>\n",
        "      <td> 3.0</td>\n",
        "      <td> 5.8</td>\n",
        "      <td> 2.2</td>\n",
        "      <td>  virginica</td>\n",
        "      <td> False</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>54 </th>\n",
        "      <td> 6.5</td>\n",
        "      <td> 2.8</td>\n",
        "      <td> 4.6</td>\n",
        "      <td> 1.5</td>\n",
        "      <td> versicolor</td>\n",
        "      <td> False</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>75 </th>\n",
        "      <td> 6.6</td>\n",
        "      <td> 3.0</td>\n",
        "      <td> 4.4</td>\n",
        "      <td> 1.4</td>\n",
        "      <td> versicolor</td>\n",
        "      <td> False</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>142</th>\n",
        "      <td> 5.8</td>\n",
        "      <td> 2.7</td>\n",
        "      <td> 5.1</td>\n",
        "      <td> 1.9</td>\n",
        "      <td>  virginica</td>\n",
        "      <td> False</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>138</th>\n",
        "      <td> 6.0</td>\n",
        "      <td> 3.0</td>\n",
        "      <td> 4.8</td>\n",
        "      <td> 1.8</td>\n",
        "      <td>  virginica</td>\n",
        "      <td> False</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>53 </th>\n",
        "      <td> 5.5</td>\n",
        "      <td> 2.3</td>\n",
        "      <td> 4.0</td>\n",
        "      <td> 1.3</td>\n",
        "      <td> versicolor</td>\n",
        "      <td> False</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "     sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)  \\\n",
        "87                 6.3               2.3                4.4               1.3   \n",
        "69                 5.6               2.5                3.9               1.1   \n",
        "87                 6.3               2.3                4.4               1.3   \n",
        "94                 5.6               2.7                4.2               1.3   \n",
        "104                6.5               3.0                5.8               2.2   \n",
        "54                 6.5               2.8                4.6               1.5   \n",
        "75                 6.6               3.0                4.4               1.4   \n",
        "142                5.8               2.7                5.1               1.9   \n",
        "138                6.0               3.0                4.8               1.8   \n",
        "53                 5.5               2.3                4.0               1.3   \n",
        "\n",
        "        species is_setosa  \n",
        "87   versicolor     False  \n",
        "69   versicolor     False  \n",
        "87   versicolor     False  \n",
        "94   versicolor     False  \n",
        "104   virginica     False  \n",
        "54   versicolor     False  \n",
        "75   versicolor     False  \n",
        "142   virginica     False  \n",
        "138   virginica     False  \n",
        "53   versicolor     False  "
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Let's also add a few random noise features for contrast\n",
      "n_noise = 2\n",
      "for i in range(n_noise):\n",
      "    df['rand_noise_%d' % i] = np.round(np.random.rand(len(df)), 3)\n",
      "df.ix[np.random.choice(df.index, 10)]  # Print a sample of rows"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>sepal length (cm)</th>\n",
        "      <th>sepal width (cm)</th>\n",
        "      <th>petal length (cm)</th>\n",
        "      <th>petal width (cm)</th>\n",
        "      <th>species</th>\n",
        "      <th>is_setosa</th>\n",
        "      <th>rand_noise_0</th>\n",
        "      <th>rand_noise_1</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>32 </th>\n",
        "      <td> 5.2</td>\n",
        "      <td> 4.1</td>\n",
        "      <td> 1.5</td>\n",
        "      <td> 0.1</td>\n",
        "      <td>     setosa</td>\n",
        "      <td>  True</td>\n",
        "      <td> 0.401</td>\n",
        "      <td> 0.913</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>77 </th>\n",
        "      <td> 6.7</td>\n",
        "      <td> 3.0</td>\n",
        "      <td> 5.0</td>\n",
        "      <td> 1.7</td>\n",
        "      <td> versicolor</td>\n",
        "      <td> False</td>\n",
        "      <td> 0.770</td>\n",
        "      <td> 0.229</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>13 </th>\n",
        "      <td> 4.3</td>\n",
        "      <td> 3.0</td>\n",
        "      <td> 1.1</td>\n",
        "      <td> 0.1</td>\n",
        "      <td>     setosa</td>\n",
        "      <td>  True</td>\n",
        "      <td> 0.980</td>\n",
        "      <td> 0.950</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>51 </th>\n",
        "      <td> 6.4</td>\n",
        "      <td> 3.2</td>\n",
        "      <td> 4.5</td>\n",
        "      <td> 1.5</td>\n",
        "      <td> versicolor</td>\n",
        "      <td> False</td>\n",
        "      <td> 0.282</td>\n",
        "      <td> 0.509</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5  </th>\n",
        "      <td> 5.4</td>\n",
        "      <td> 3.9</td>\n",
        "      <td> 1.7</td>\n",
        "      <td> 0.4</td>\n",
        "      <td>     setosa</td>\n",
        "      <td>  True</td>\n",
        "      <td> 0.451</td>\n",
        "      <td> 0.418</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>149</th>\n",
        "      <td> 5.9</td>\n",
        "      <td> 3.0</td>\n",
        "      <td> 5.1</td>\n",
        "      <td> 1.8</td>\n",
        "      <td>  virginica</td>\n",
        "      <td> False</td>\n",
        "      <td> 0.472</td>\n",
        "      <td> 0.527</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>132</th>\n",
        "      <td> 6.4</td>\n",
        "      <td> 2.8</td>\n",
        "      <td> 5.6</td>\n",
        "      <td> 2.2</td>\n",
        "      <td>  virginica</td>\n",
        "      <td> False</td>\n",
        "      <td> 0.193</td>\n",
        "      <td> 0.804</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>16 </th>\n",
        "      <td> 5.4</td>\n",
        "      <td> 3.9</td>\n",
        "      <td> 1.3</td>\n",
        "      <td> 0.4</td>\n",
        "      <td>     setosa</td>\n",
        "      <td>  True</td>\n",
        "      <td> 0.062</td>\n",
        "      <td> 0.244</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4  </th>\n",
        "      <td> 5.0</td>\n",
        "      <td> 3.6</td>\n",
        "      <td> 1.4</td>\n",
        "      <td> 0.2</td>\n",
        "      <td>     setosa</td>\n",
        "      <td>  True</td>\n",
        "      <td> 0.031</td>\n",
        "      <td> 0.351</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3  </th>\n",
        "      <td> 4.6</td>\n",
        "      <td> 3.1</td>\n",
        "      <td> 1.5</td>\n",
        "      <td> 0.2</td>\n",
        "      <td>     setosa</td>\n",
        "      <td>  True</td>\n",
        "      <td> 0.244</td>\n",
        "      <td> 0.607</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "     sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)  \\\n",
        "32                 5.2               4.1                1.5               0.1   \n",
        "77                 6.7               3.0                5.0               1.7   \n",
        "13                 4.3               3.0                1.1               0.1   \n",
        "51                 6.4               3.2                4.5               1.5   \n",
        "5                  5.4               3.9                1.7               0.4   \n",
        "149                5.9               3.0                5.1               1.8   \n",
        "132                6.4               2.8                5.6               2.2   \n",
        "16                 5.4               3.9                1.3               0.4   \n",
        "4                  5.0               3.6                1.4               0.2   \n",
        "3                  4.6               3.1                1.5               0.2   \n",
        "\n",
        "        species is_setosa  rand_noise_0  rand_noise_1  \n",
        "32       setosa      True         0.401         0.913  \n",
        "77   versicolor     False         0.770         0.229  \n",
        "13       setosa      True         0.980         0.950  \n",
        "51   versicolor     False         0.282         0.509  \n",
        "5        setosa      True         0.451         0.418  \n",
        "149   virginica     False         0.472         0.527  \n",
        "132   virginica     False         0.193         0.804  \n",
        "16       setosa      True         0.062         0.244  \n",
        "4        setosa      True         0.031         0.351  \n",
        "3        setosa      True         0.244         0.607  "
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# For starters, we can look at pearson correlation coefficients with .corr\n",
      "# All of the original features look promising.\n",
      "df.corr()['is_setosa']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 7,
       "text": [
        "sepal length (cm)   -0.717416\n",
        "sepal width (cm)     0.595601\n",
        "petal length (cm)   -0.922688\n",
        "petal width (cm)    -0.887510\n",
        "is_setosa            1.000000\n",
        "rand_noise_0        -0.080335\n",
        "rand_noise_1         0.068206\n",
        "Name: is_setosa, dtype: float64"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Root mean squared error calculation, used below.\n",
      "# In practice you may have a different target function.\n",
      "def rmse(truth, predictions):\n",
      "    return np.sqrt(sum((truth - predictions) ** 2)/len(truth))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# The basics of modeling is to \"fit\" an estimator, then use it to \"predict\"\n",
      "# sklearn requires 2d numpy matrices, not dataframes, which you can obtain with DataFrame.values\n",
      "features = ['sepal length (cm)',u'sepal width (cm)', 'petal length (cm)', 'petal width (cm)', 'rand_noise_0', 'rand_noise_1']\n",
      "X = df[features].values  # Feature matrix\n",
      "y = df['is_setosa'].values  # Target label vector\n",
      "estimator = linear_model.LogisticRegression()\n",
      "estimator.fit(X, y)\n",
      "predictions = estimator.predict_proba(X)[:,1]\n",
      "print ', '.join(features + ['is_setota', 'p(is_setosa)'])\n",
      "for i in np.random.choice(df.index, 10):\n",
      "    print X[i], y[i], '%.4f' % predictions[i]\n",
      "print 'Root mean squared error:', rmse(y, predictions)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "sepal length (cm), sepal width (cm), petal length (cm), petal width (cm), rand_noise_0, rand_noise_1, is_setota, p(is_setosa)\n",
        "[ 6.     2.2    5.     1.5    0.9    0.341] False 0.0010\n",
        "[ 6.5    3.     5.8    2.2    0.765  0.337] False 0.0003\n",
        "[ 6.3    2.9    5.6    1.8    0.038  0.472] False 0.0006\n",
        "[ 4.6    3.6    1.     0.2    0.572  0.852] True 0.9937\n",
        "[ 6.2    3.4    5.4    2.3    0.966  0.168] False 0.0011\n",
        "[ 7.7    3.     6.1    2.3    0.09   0.779] False 0.0002\n",
        "[ 5.     3.6    1.4    0.2    0.031  0.351] True 0.9848\n",
        "[ 6.4   2.9   4.3   1.3   0.5   0.44] False 0.0195\n",
        "[ 5.4    3.     4.5    1.5    0.671  0.323] False 0.0079\n",
        "[ 5.7    2.6    3.5    1.     0.163  0.227] False 0.0700\n",
        "Root mean squared error: 0.0252721878151\n"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Each estimator can tell you information about itself.\n",
      "# Logistic regression can tell you its coefficients and intercept:\n",
      "print 'Coefficients:'\n",
      "for f, val in zip(features, estimator.coef_[0]):\n",
      "    print '* %s: %.4f' % (f, val)\n",
      "print '* intercept: %.4f' % estimator.intercept_[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Coefficients:\n",
        "* sepal length (cm): 0.3926\n",
        "* sepal width (cm): 1.4530\n",
        "* petal length (cm): -2.2573\n",
        "* petal width (cm): -1.0228\n",
        "* rand_noise_0: 0.0649\n",
        "* rand_noise_1: 0.2290\n",
        "* intercept: 0.2589\n"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# However, we tested our predictions on the same dataset that we trained on.\n",
      "# We need to split out test and train datasets.\n",
      "X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, y, test_size=0.2)\n",
      "print 'Splitting %d samples into %d training and %d test samples' % (\n",
      "    len(X), len(X_train), len(X_test))\n",
      "\n",
      "estimator.fit(X_train, y_train)\n",
      "predictions = estimator.predict_proba(X_test)[:,1]\n",
      "print ', '.join(features + ['is_setota', 'p(is_setosa)'])\n",
      "for i in range(10):\n",
      "    print X_test[i], y_test[i], predictions[i]\n",
      "print 'Root mean squared error:', rmse(y_test, predictions)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Splitting 150 samples into 120 training and 30 test samples\n",
        "sepal length (cm), sepal width (cm), petal length (cm), petal width (cm), rand_noise_0, rand_noise_1, is_setota, p(is_setosa)\n",
        "[ 5.2    4.1    1.5    0.1    0.401  0.913] True 0.991177916568\n",
        "[ 5.4    3.     4.5    1.5    0.671  0.323] False 0.00935911832272\n",
        "[ 5.7    3.8    1.7    0.3    0.865  0.557] True 0.978123225208\n",
        "[ 6.8    3.2    5.9    2.3    0.656  0.514] False 0.00049676893029\n",
        "[ 5.8    4.     1.2    0.2    0.651  0.241] True 0.994621759685\n",
        "[ 7.7    2.8    6.7    2.     0.923  0.858] False 0.000102103555262\n",
        "[ 4.6    3.1    1.5    0.2    0.244  0.607] True 0.950670675609\n",
        "[ 6.1    3.     4.9    1.8    0.653  0.246] False 0.00381344111686\n",
        "[ 6.     2.7    5.1    1.6    0.251  0.123] False 0.00185506366156\n",
        "[ 6.5    3.     5.8    2.2    0.765  0.337] False 0.000443025335224\n",
        "Root mean squared error: 0.0356672107781\n"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# However, this RMSE value might vary a lot, depending on the\n",
      "# random train/test split. We have no idea how much to trust this value.\n",
      "# Let's try many train/test splits. This is the basics of cross-validation.\n",
      "def cross_validate(runs):\n",
      "    errors = []\n",
      "    for i in range(runs):\n",
      "        X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, y, test_size=0.2)\n",
      "        estimator.fit(X_train, y_train)\n",
      "        predictions = estimator.predict_proba(X_test)[:,1]\n",
      "        errors.append(rmse(y_test, predictions))\n",
      "    mu = np.mean(errors)\n",
      "    sig = np.std(errors)\n",
      "    print 'Average RMSE: %.4f. StdDev: %.4f' % (mu, sig)\n",
      "    print '95%% CI: [%.4f, %.4f]' % (mu - 2*sig, mu + 2*sig)\n",
      "\n",
      "cross_validate(100)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Average RMSE: 0.0290. StdDev: 0.0057\n",
        "95% CI: [0.0176, 0.0404]\n"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Dealing with categorical variables -- let's add one to our original frame\n",
      "df['random_categorical'] = np.random.choice(['apple', 'banana', 'cherry'], len(df))\n",
      "df.head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>sepal length (cm)</th>\n",
        "      <th>sepal width (cm)</th>\n",
        "      <th>petal length (cm)</th>\n",
        "      <th>petal width (cm)</th>\n",
        "      <th>species</th>\n",
        "      <th>is_setosa</th>\n",
        "      <th>rand_noise_0</th>\n",
        "      <th>rand_noise_1</th>\n",
        "      <th>random_categorical</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 5.1</td>\n",
        "      <td> 3.5</td>\n",
        "      <td> 1.4</td>\n",
        "      <td> 0.2</td>\n",
        "      <td> setosa</td>\n",
        "      <td> True</td>\n",
        "      <td> 0.443</td>\n",
        "      <td> 0.318</td>\n",
        "      <td>  apple</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 4.9</td>\n",
        "      <td> 3.0</td>\n",
        "      <td> 1.4</td>\n",
        "      <td> 0.2</td>\n",
        "      <td> setosa</td>\n",
        "      <td> True</td>\n",
        "      <td> 0.818</td>\n",
        "      <td> 0.835</td>\n",
        "      <td> banana</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 4.7</td>\n",
        "      <td> 3.2</td>\n",
        "      <td> 1.3</td>\n",
        "      <td> 0.2</td>\n",
        "      <td> setosa</td>\n",
        "      <td> True</td>\n",
        "      <td> 0.950</td>\n",
        "      <td> 0.988</td>\n",
        "      <td> cherry</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 4.6</td>\n",
        "      <td> 3.1</td>\n",
        "      <td> 1.5</td>\n",
        "      <td> 0.2</td>\n",
        "      <td> setosa</td>\n",
        "      <td> True</td>\n",
        "      <td> 0.244</td>\n",
        "      <td> 0.607</td>\n",
        "      <td> banana</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> 5.0</td>\n",
        "      <td> 3.6</td>\n",
        "      <td> 1.4</td>\n",
        "      <td> 0.2</td>\n",
        "      <td> setosa</td>\n",
        "      <td> True</td>\n",
        "      <td> 0.031</td>\n",
        "      <td> 0.351</td>\n",
        "      <td>  apple</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 13,
       "text": [
        "   sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)  \\\n",
        "0                5.1               3.5                1.4               0.2   \n",
        "1                4.9               3.0                1.4               0.2   \n",
        "2                4.7               3.2                1.3               0.2   \n",
        "3                4.6               3.1                1.5               0.2   \n",
        "4                5.0               3.6                1.4               0.2   \n",
        "\n",
        "  species is_setosa  rand_noise_0  rand_noise_1 random_categorical  \n",
        "0  setosa      True         0.443         0.318              apple  \n",
        "1  setosa      True         0.818         0.835             banana  \n",
        "2  setosa      True         0.950         0.988             cherry  \n",
        "3  setosa      True         0.244         0.607             banana  \n",
        "4  setosa      True         0.031         0.351              apple  "
       ]
      }
     ],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We need to \"one-hot-encode\" categorical variables as a series of binary ones.\n",
      "# Pandas can easily create these for us with get_dummies\n",
      "dummy_df = pd.get_dummies(df['random_categorical'])\n",
      "df = df.merge(dummy_df, left_index=True, right_index=True)\n",
      "df.head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>sepal length (cm)</th>\n",
        "      <th>sepal width (cm)</th>\n",
        "      <th>petal length (cm)</th>\n",
        "      <th>petal width (cm)</th>\n",
        "      <th>species</th>\n",
        "      <th>is_setosa</th>\n",
        "      <th>rand_noise_0</th>\n",
        "      <th>rand_noise_1</th>\n",
        "      <th>random_categorical</th>\n",
        "      <th>apple</th>\n",
        "      <th>banana</th>\n",
        "      <th>cherry</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 5.1</td>\n",
        "      <td> 3.5</td>\n",
        "      <td> 1.4</td>\n",
        "      <td> 0.2</td>\n",
        "      <td> setosa</td>\n",
        "      <td> True</td>\n",
        "      <td> 0.443</td>\n",
        "      <td> 0.318</td>\n",
        "      <td>  apple</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 4.9</td>\n",
        "      <td> 3.0</td>\n",
        "      <td> 1.4</td>\n",
        "      <td> 0.2</td>\n",
        "      <td> setosa</td>\n",
        "      <td> True</td>\n",
        "      <td> 0.818</td>\n",
        "      <td> 0.835</td>\n",
        "      <td> banana</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 4.7</td>\n",
        "      <td> 3.2</td>\n",
        "      <td> 1.3</td>\n",
        "      <td> 0.2</td>\n",
        "      <td> setosa</td>\n",
        "      <td> True</td>\n",
        "      <td> 0.950</td>\n",
        "      <td> 0.988</td>\n",
        "      <td> cherry</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 4.6</td>\n",
        "      <td> 3.1</td>\n",
        "      <td> 1.5</td>\n",
        "      <td> 0.2</td>\n",
        "      <td> setosa</td>\n",
        "      <td> True</td>\n",
        "      <td> 0.244</td>\n",
        "      <td> 0.607</td>\n",
        "      <td> banana</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> 5.0</td>\n",
        "      <td> 3.6</td>\n",
        "      <td> 1.4</td>\n",
        "      <td> 0.2</td>\n",
        "      <td> setosa</td>\n",
        "      <td> True</td>\n",
        "      <td> 0.031</td>\n",
        "      <td> 0.351</td>\n",
        "      <td>  apple</td>\n",
        "      <td> 1</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 14,
       "text": [
        "   sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)  \\\n",
        "0                5.1               3.5                1.4               0.2   \n",
        "1                4.9               3.0                1.4               0.2   \n",
        "2                4.7               3.2                1.3               0.2   \n",
        "3                4.6               3.1                1.5               0.2   \n",
        "4                5.0               3.6                1.4               0.2   \n",
        "\n",
        "  species is_setosa  rand_noise_0  rand_noise_1 random_categorical  apple  \\\n",
        "0  setosa      True         0.443         0.318              apple      1   \n",
        "1  setosa      True         0.818         0.835             banana      0   \n",
        "2  setosa      True         0.950         0.988             cherry      0   \n",
        "3  setosa      True         0.244         0.607             banana      0   \n",
        "4  setosa      True         0.031         0.351              apple      1   \n",
        "\n",
        "   banana  cherry  \n",
        "0       0       0  \n",
        "1       1       0  \n",
        "2       0       1  \n",
        "3       1       0  \n",
        "4       0       0  "
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df.corr()['is_setosa']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 15,
       "text": [
        "sepal length (cm)   -0.717416\n",
        "sepal width (cm)     0.595601\n",
        "petal length (cm)   -0.922688\n",
        "petal width (cm)    -0.887510\n",
        "is_setosa            1.000000\n",
        "rand_noise_0        -0.080335\n",
        "rand_noise_1         0.068206\n",
        "apple                0.079243\n",
        "banana              -0.050814\n",
        "cherry              -0.029854\n",
        "Name: is_setosa, dtype: float64"
       ]
      }
     ],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We fit a model with these features, but we leave one out (which the intercept will imply)\n",
      "features = ['apple', 'banana', 'sepal length (cm)',u'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']\n",
      "X = df[features].values\n",
      "y = df['is_setosa'].values\n",
      "estimator = linear_model.LogisticRegression()\n",
      "cross_validate(100)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Average RMSE: 0.0293. StdDev: 0.0067\n",
        "95% CI: [0.0159, 0.0427]\n"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Does a random forest do any better?\n",
      "estimator = ensemble.RandomForestClassifier(n_estimators=40)\n",
      "cross_validate(100)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Average RMSE: 0.0186. StdDev: 0.0116\n",
        "95% CI: [-0.0045, 0.0417]\n"
       ]
      }
     ],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 17
    }
   ],
   "metadata": {}
  }
 ]
}