Latest numpy and pandas

.gitignore

@@ -55,6 +55,9 @@ nosetests.xml
 # Built documentation
 docs/_build/*
 
+# Un-tarred example data input. We should only commit the tarball.
+tests/resources/example_data/*
+
 # database of vbench
 benchmarks.db
 

.travis.yml

@@ -9,8 +9,7 @@ env:
     # ANACONDA_TOKEN with api:write
     - secure: "RJJjjQloUjCSkhII93QM+YAsr6YYq7hPFvlbnT07ogn1NeUPsVCyJ97oiZfKtPgdbd24hdQP/CHfB0HgTTES8n996tN3QWc4hZj0e10kFyIlas9qnkrRYRR1jxGShBDXrLdx/tGh8z1qnnCm1fy+fDhAF7Zerouwy4EA2YEzxdE="
   matrix:
-    - PANDAS_VERSION=0.16.1 NUMPY_VERSION=1.9.2 SCIPY_VERSION=0.15.1
-    - PANDAS_VERSION=0.17.1 NUMPY_VERSION=1.10.4 SCIPY_VERSION=0.16.1
+    - NUMPY_VERSION=1.11.1 SCIPY_VERSION=0.17.1
 cache:
   directories:
     - $HOME/.cache/.pip/
@@ -20,14 +19,11 @@ before_install:
   - chmod +x miniconda.sh
   - ./miniconda.sh -b -p $HOME/miniconda
   - export PATH="$HOME/miniconda/bin:$PATH"
-  - sed -i "s/numpy==.*/numpy==$NUMPY_VERSION/" etc/requirements.txt
-  - sed -i "s/pandas==.*/pandas==$PANDAS_VERSION/" etc/requirements.txt
-  - sed -i "s/scipy==.*/scipy==$SCIPY_VERSION/" etc/requirements.txt
 install:
   - conda install conda=4.1.11 conda-build=1.21.11 anaconda-client=1.5.1 --yes
 
   - TALIB_VERSION=$(cat ./etc/requirements_talib.txt | sed "s/TA-Lib==\(.*\)/\1/")
-  - conda create -n testenv --yes -c quantopian pip python=$TRAVIS_PYTHON_VERSION numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION libgfortran=1.0 ta-lib=$TALIB_VERSION
+  - conda create -n testenv --yes -c quantopian pip python=$TRAVIS_PYTHON_VERSION numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION libgfortran=3.0 ta-lib=$TALIB_VERSION
   - source activate testenv
   - IFS='.' read -r -a NPY_VERSION_ARR <<< "$NUMPY_VERSION"
   - CONDA_NPY=${NPY_VERSION_ARR[0]}${NPY_VERSION_ARR[1]}

appveyor.yml

@@ -15,27 +15,15 @@ environment:
   matrix:
     - PYTHON_VERSION: "2.7"
       PYTHON_ARCH: "64"
-      PANDAS_VERSION: "0.16.1"
-      NUMPY_VERSION: "1.9.2"
-      SCIPY_VERSION: "0.15.1"
+      PANDAS_VERSION: "0.18.1"
+      NUMPY_VERSION: "1.11.1"
+      SCIPY_VERSION: "0.17.1"
 
     - PYTHON_VERSION: "3.4"
       PYTHON_ARCH: "64"
-      PANDAS_VERSION: "0.16.1"
-      NUMPY_VERSION: "1.9.2"
-      SCIPY_VERSION: "0.15.1"
-
-    - PYTHON_VERSION: "2.7"
-      PYTHON_ARCH: "64"
-      PANDAS_VERSION: "0.17.1"
-      NUMPY_VERSION: "1.10.4"
-      SCIPY_VERSION: "0.16.1"
-
-    - PYTHON_VERSION: "3.4"
-      PYTHON_ARCH: "64"
-      PANDAS_VERSION: "0.17.1"
-      NUMPY_VERSION: "1.10.4"
-      SCIPY_VERSION: "0.16.1"
+      PANDAS_VERSION: "0.18.1"
+      NUMPY_VERSION: "1.11.1"
+      SCIPY_VERSION: "0.17.1"
 
 # We always use a 64-bit machine, but can build x86 distributions
 # with the PYTHON_ARCH variable (which is used by CMD_IN_ENV).

etc/requirements.txt

@@ -8,13 +8,13 @@ Logbook==0.12.5
 
 # Scientific Libraries
 
-pytz==2015.4
-numpy==1.9.2
+pytz==2016.4
+numpy==1.11.1
 
 # scipy and pandas are required for statsmodels,
 # statsmodels in turn is required for some pandas packages
-scipy==0.15.1
-pandas==0.16.1
+scipy==0.17.1
+pandas==0.18.1
 pandas-datareader==0.2.1
 # Needed for parts of pandas.stats
 patsy==0.4.0
@@ -38,6 +38,7 @@ contextlib2==0.4.0
 
 # networkx requires decorator
 decorator==4.0.0
+
 # Graph algorithms used by zipline.pipeline
 networkx==1.9.1
 

etc/requirements_blaze.txt

@@ -1,3 +1,3 @@
 -e git://github.com/quantopian/datashape.git@bf06a41dc0908baf7c324aeacadba8820468ee78#egg=datashape-dev
--e git://github.com/quantopian/odo.git@9e16310b5f2c3f05162145200db7e7908f0a866e#egg=odo-dev
--e git://github.com/quantopian/blaze.git@7b8f70c15279c7ebacf2ac2f5f74acf08b300d71#egg=blaze-dev
+-e git://github.com/quantopian/odo.git@da7f26d87702f5d293763e8ed54c7e25fd3af386#egg=odo-dev
+-e git://github.com/quantopian/blaze.git@929afbd3466df7efa0cf720b74dba223e0b3ac4a#egg=blaze-dev

setup.py

@@ -148,7 +148,7 @@ def _filter_requirements(lines_iter, filter_names=None,
 
 REQ_UPPER_BOUNDS = {
     'bcolz': '<1',
-    'pandas': '<0.18',
+    'pandas': '<0.19',
 }
 
 

tests/data/test_minute_bars.py

@@ -590,7 +590,7 @@ def test_nans(self):
                 'high': full(9, nan),
                 'low': full(9, nan),
                 'close': full(9, nan),
-                'volume': full(9, 0),
+                'volume': full(9, 0.0),
             },
             index=[minutes])
         self.writer.write_sid(sid, data)
@@ -633,7 +633,7 @@ def test_differing_nans(self):
                 view(float64),
                 'close': ((0b11111111111 << 52) + arange(31, 40, dtype=int64)).
                 view(float64),
-                'volume': full(9, 0),
+                'volume': full(9, 0.0),
             },
             index=[minutes])
         self.writer.write_sid(sid, data)

tests/events/test_events.py

@@ -16,6 +16,7 @@
 from inspect import isabstract
 import random
 from unittest import TestCase
+import warnings
 
 from nose_parameterized import parameterized
 import pandas as pd
@@ -439,6 +440,38 @@ def test_ComposedRule(self):
             self.assertIs(composed.second, rule2)
             self.assertFalse(any(map(should_trigger, minute)))
 
+    @parameterized.expand([
+        ('month_start', NthTradingDayOfMonth),
+        ('month_end', NDaysBeforeLastTradingDayOfMonth),
+        ('week_start', NthTradingDayOfWeek),
+        ('week_end', NthTradingDayOfWeek),
+    ])
+    def test_pass_float_to_day_of_period_rule(self, name, rule_type):
+        with warnings.catch_warnings(record=True) as raised_warnings:
+            warnings.simplefilter('always')
+            rule_type(n=3)    # Shouldn't trigger a warning.
+            rule_type(n=3.0)  # Should trigger a warning about float coercion.
+
+        self.assertEqual(len(raised_warnings), 1)
+
+        # We only implicitly convert from float to int when there's no loss of
+        # precision.
+        with self.assertRaises(TypeError):
+            rule_type(3.1)
+
+    def test_invalid_offsets(self):
+        with self.assertRaises(ValueError):
+            NthTradingDayOfWeek(5)
+
+        with self.assertRaises(ValueError):
+            NthTradingDayOfWeek(-1)
+
+        with self.assertRaises(ValueError):
+            NthTradingDayOfMonth(-1)
+
+        with self.assertRaises(ValueError):
+            NthTradingDayOfMonth(24)
+
 
 class StatefulRulesTests(RuleTestCase):
     CALENDAR_STRING = "NYSE"

tests/pipeline/test_blaze.py

@@ -784,6 +784,7 @@ def _test_id_macro(self, df, dshape, expected, finder, add):
             expr,
             loader=loader,
             no_deltas_rule='ignore',
+            no_checkpoints_rule='ignore',
             missing_values=self.missing_values,
         )
 

tests/pipeline/test_column.py

@@ -11,6 +11,7 @@
 from zipline.pipeline import Pipeline
 from zipline.pipeline.data.testing import TestingDataSet as TDS
 from zipline.testing import chrange, temp_pipeline_engine
+from zipline.utils.pandas_utils import ignore_pandas_nan_categorical_warning
 
 
 class LatestTestCase(TestCase):
@@ -71,6 +72,8 @@ def test_latest(self):
             dates_to_test[-1],
         )
         for column in columns:
-            col_result = result[column.name].unstack()
+            with ignore_pandas_nan_categorical_warning():
+                col_result = result[column.name].unstack()
+
             expected_col_result = self.expected_latest(column, cal_slice)
             assert_frame_equal(col_result, expected_col_result)

tests/pipeline/test_engine.py

@@ -25,12 +25,8 @@
     Categorical,
     DataFrame,
     date_range,
-    ewma,
-    ewmstd,
     Int64Index,
     MultiIndex,
-    rolling_apply,
-    rolling_mean,
     Series,
     Timestamp,
 )
@@ -1008,15 +1004,17 @@ def test_SMA(self):
         # Shift back the raw inputs by a trading day because we expect our
         # computed results to be computed using values anchored on the
         # **previous** day's data.
-        expected_raw = rolling_mean(
+        expected_raw = DataFrame(
             expected_bar_values_2d(
                 dates - self.trading_calendar.day,
                 self.equity_info,
                 'close',
             ),
+        ).rolling(
             window_length,
             min_periods=1,
-        )
+        ).mean(
+        ).values
 
         expected = DataFrame(
             # Truncate off the extra rows needed to compute the SMAs.
@@ -1122,19 +1120,31 @@ def init_class_fixtures(cls):
     def expected_ewma(self, window_length, decay_rate):
         alpha = 1 - decay_rate
         span = (2 / alpha) - 1
-        return rolling_apply(
-            self.raw_data,
-            window_length,
-            lambda window: ewma(window, span=span)[-1],
+
+        # XXX: This is a comically inefficient way to compute a windowed EWMA.
+        # Don't use it outside of testing.  We're using rolling-apply of an
+        # ewma (which is itself a rolling-window function) because we only want
+        # to look at ``window_length`` rows at a time.
+        return self.raw_data.rolling(window_length).apply(
+            lambda subarray: (DataFrame(subarray)
+                              .ewm(span=span)
+                              .mean()
+                              .values[-1])
         )[window_length:]
 
     def expected_ewmstd(self, window_length, decay_rate):
         alpha = 1 - decay_rate
         span = (2 / alpha) - 1
-        return rolling_apply(
-            self.raw_data,
-            window_length,
-            lambda window: ewmstd(window, span=span)[-1],
+
+        # XXX: This is a comically inefficient way to compute a windowed
+        # EWMSTD.  Don't use it outside of testing.  We're using rolling-apply
+        # of an ewma (which is itself a rolling-window function) because we
+        # only want to look at ``window_length`` rows at a time.
+        return self.raw_data.rolling(window_length).apply(
+            lambda subarray: (DataFrame(subarray)
+                              .ewm(span=span)
+                              .std()
+                              .values[-1])
         )[window_length:]
 
     @parameterized.expand([
@@ -1259,7 +1269,7 @@ def test_dollar_volume(self):
         expected_1 = (self.raw_data[5:] ** 2) * 2
         assert_frame_equal(results['dv1'].unstack(), expected_1)
 
-        expected_5 = rolling_mean((self.raw_data ** 2) * 2, window=5)[5:]
+        expected_5 = ((self.raw_data ** 2) * 2).rolling(5).mean()[5:]
         assert_frame_equal(results['dv5'].unstack(), expected_5)
 
         # The following two use USEquityPricing.open and .volume as inputs.
@@ -1269,9 +1279,11 @@ def test_dollar_volume(self):
                           * self.raw_data[5:] * 2).fillna(0)
         assert_frame_equal(results['dv1_nan'].unstack(), expected_1_nan)
 
-        expected_5_nan = rolling_mean((self.raw_data_with_nans
-                                       * self.raw_data * 2).fillna(0),
-                                      window=5)[5:]
+        expected_5_nan = ((self.raw_data_with_nans * self.raw_data * 2)
+                          .fillna(0)
+                          .rolling(5).mean()
+                          [5:])
+
         assert_frame_equal(results['dv5_nan'].unstack(), expected_5_nan)
 
 

tests/pipeline/test_events.py

@@ -153,7 +153,7 @@ class EventIndexerTestCase(ZiplineTestCase):
     @classmethod
     def init_class_fixtures(cls):
         super(EventIndexerTestCase, cls).init_class_fixtures()
-        cls.events = make_events(add_nulls=False).sort('event_date')
+        cls.events = make_events(add_nulls=False).sort_values('event_date')
         cls.events.reset_index(inplace=True)
 
     def test_previous_event_indexer(self):
@@ -354,7 +354,7 @@ def check_previous_value_results(self, column, results):
         # from pandas won't be tz_localized.
         dates = self.trading_days.tz_localize(None)
 
-        for asset, asset_result in results.iterkv():
+        for asset, asset_result in results.iteritems():
             relevant_events = events[events.sid == asset.sid]
             self.assertEqual(len(relevant_events), 2)
 
@@ -397,7 +397,7 @@ def check_next_value_results(self, column, results):
         # Remove timezone info from trading days, since the outputs
         # from pandas won't be tz_localized.
         dates = self.trading_days.tz_localize(None)
-        for asset, asset_result in results.iterkv():
+        for asset, asset_result in results.iteritems():
             relevant_events = events[events.sid == asset.sid]
             self.assertEqual(len(relevant_events), 2)
 

tests/pipeline/test_pipeline_algo.py

@@ -346,7 +346,7 @@ class PipelineAlgorithmTestCase(WithBcolzEquityDailyBarReaderFromCSVs,
     AAPL = 1
     MSFT = 2
     BRK_A = 3
-    assets = ASSET_FINDER_EQUITY_SIDS = AAPL, MSFT, BRK_A
+    ASSET_FINDER_EQUITY_SIDS = AAPL, MSFT, BRK_A
     ASSET_FINDER_EQUITY_SYMBOLS = 'AAPL', 'MSFT', 'BRK_A'
     START_DATE = Timestamp('2014')
     END_DATE = Timestamp('2015')
@@ -403,6 +403,9 @@ def init_class_fixtures(cls):
         )
         cls.dates = cls.raw_data[cls.AAPL].index.tz_localize('UTC')
         cls.AAPL_split_date = Timestamp("2014-06-09", tz='UTC')
+        cls.assets = cls.asset_finder.retrieve_all(
+            cls.ASSET_FINDER_EQUITY_SIDS
+        )
 
     def compute_expected_vwaps(self, window_lengths):
         AAPL, MSFT, BRK_A = self.AAPL, self.MSFT, self.BRK_A
@@ -488,7 +491,7 @@ def compute_expected_vwaps(self, window_lengths):
         (False,),
     ])
     def test_handle_adjustment(self, set_screen):
-        AAPL, MSFT, BRK_A = assets = self.AAPL, self.MSFT, self.BRK_A
+        AAPL, MSFT, BRK_A = assets = self.assets
 
         window_lengths = [1, 2, 5, 10]
         vwaps = self.compute_expected_vwaps(window_lengths)