performance-short/5. fast methods.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# How to make pandas go <font color='green'>fast</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The Cython extension is already loaded. To reload it, use:\n",
      "  %reload_ext Cython\n"
     ]
    }
   ],
   "source": [
    "pd.set_option('max_rows',12)\n",
    "import bottleneck as bn\n",
    "import numexpr as ne\n",
    "import numpy as np\n",
    "from numba import jit\n",
    "%load_ext Cython"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# computation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(np.random.randn(10000000,2),columns=list('AB'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "def f_vectorize_numpy(df):\n",
    "    return (df['A'].values*2 + df['B'].values + 1).sum()\n",
    "def f_vectorize_pandas(df):\n",
    "    return (df['A']*2 + df['B'] + 1).sum()\n",
    "def f_numexpr(df):\n",
    "    a = df['A'].values\n",
    "    b = df['B'].values\n",
    "    return ne.evaluate('sum(a*2 + b + 1)',local_dict = { 'a' : a, 'b' : b }).item()\n",
    "def f_bottleneck(df):\n",
    "    return bn.nansum(df['A'].values*2 + df['B'].values + 1)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "%%cython\n",
    "cdef _cython(double[:] a, double[:] b):\n",
    "    cdef:\n",
    "        double result = 0.0\n",
    "        int i\n",
    "    \n",
    "    assert len(a) == len(b)\n",
    "    for i in range(len(a)):\n",
    "        result += a[i]*2 + b[i] + 1.0\n",
    "    return result\n",
    "        \n",
    "def f_cython(df):\n",
    "    return _cython(df['A'].values,df['B'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "@jit\n",
    "def _numba(a, b):\n",
    "    result = 0.0\n",
    "    assert len(a) == len(b)\n",
    "    for i in range(len(a)):\n",
    "        result += a[i]*2 + b[i] + 1.0\n",
    "    return result\n",
    "\n",
    "def f_numba(df):\n",
    "    return _numba(df['A'].values,df['B'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.8225525255655128e-13"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.var([f_vectorize_numpy(df),f_vectorize_pandas(df),f_numexpr(df),f_bottleneck(df),f_cython(df),f_numba(df)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 loops, best of 3: 107 ms per loop\n"
     ]
    }
   ],
   "source": [
    "%timeit f_vectorize_numpy(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 loops, best of 3: 85.3 ms per loop\n"
     ]
    }
   ],
   "source": [
    "%timeit f_vectorize_pandas(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 loops, best of 3: 37.7 ms per loop\n"
     ]
    }
   ],
   "source": [
    "%timeit f_numexpr(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 loops, best of 3: 110 ms per loop\n"
     ]
    }
   ],
   "source": [
    "%timeit f_bottleneck(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 loops, best of 3: 25.3 ms per loop\n"
     ]
    }
   ],
   "source": [
    "%timeit f_cython(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 loops, best of 3: 15 ms per loop\n"
     ]
    }
   ],
   "source": [
    "%timeit f_numba(df)"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}