Merge remote-tracking branch 'upstream/master' into fix-panel-fillna

* upstream/master: (48 commits)
  BUG: Categorical comparison with unordered (pandas-dev#16339)
  ENH: Adding 'protocol' parameter to 'to_pickle'.
  PERF: improve MultiIndex get_loc performance (pandas-dev#16346)
  TST: remove pandas-datareader xfail as 0.4.0 works (pandas-dev#16374)
  TST: followup to pandas-dev#16364, catch errstate warnings (pandas-dev#16373)
  DOC: new oauth token
  TST: Add test for clip-na (pandas-dev#16369)
  ENH: Draft metadata specification doc for Apache Parquet (pandas-dev#16315)
  MAINT: Add .iml to .gitignore (pandas-dev#16368)
  BUG/API: Categorical constructor scalar categories (pandas-dev#16340)
  ENH: Provide dict object for to_dict() pandas-dev#16122 (pandas-dev#16220)
  PERF: improved clip performance (pandas-dev#16364)
  DOC: try new token for docs
  DOC: try with new secure token
  DOC: add developer section to the docs
  DEPS: Drop Python 3.4 support (pandas-dev#16303)
  DOC: remove credential helper
  DOC: force fetch on build docs
  DOC: redo dev docs access token
  DOC: add dataframe construction in merge_asof example (pandas-dev#16348)
  ...

.gitignore

@@ -7,6 +7,7 @@
 *$
 *.bak
 *flymake*
+*.iml
 *.kdev4
 *.log
 *.swp

.travis.yml

@@ -17,8 +17,13 @@ cache:
 env:
   global:
 
-  # pandas-docs-travis GH
-  - secure: "YvvTc+FrSYHgdxqoxn9s8VOaCWjvZzlkaf6k55kkmQqCYR9dPiLMsot1F96/N7o3YlD1s0znPQCak93Du8HHi/8809zAXloTaMSZrWz4R4qn96xlZFRE88O/w/Z1t3VVYpKX3MHlCggBc8MtXrqmvWKJMAqXyysZ4TTzoiJDPvE="
+  # pandas-docs/pandas-docs-travis GH #
+  #
+  # create a github personal access token
+  # cd pandas-docs/pandas-docs-travis
+  # travis encrypt
+  # PANDAS_GH_TOKEN=personal_access_token
+  secure: "S49Tn5dzBRu6QaQcSV8MoCeX9rn7l8xuHFJbFsT9jPm1l0YPb94S8iDk0Isw71SqvHBgh+j2cms9jgYn2N3VCArh5MpA0oKwTKRZEX3iLQv248dCY2C6LdzAKLA+8m2naDGcfc0qMLeNieCGZICccs0EKIGDt8m7VQBMqeT0YU0="
 
 git:
     # for cloning
@@ -123,7 +128,7 @@ after_success:
 
 after_script:
   - echo "after_script start"
-  - source activate pandas && python -c "import pandas; pandas.show_versions();"
+  - source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
   - if [ -e /tmp/single.xml ]; then
     ci/print_skipped.py /tmp/single.xml;
     fi

asv_bench/benchmarks/indexing.py

@@ -19,6 +19,9 @@ def time_getitem_list_like(self):
     def time_getitem_array(self):
         self.s[np.arange(10000)]
 
+    def time_getitem_lists(self):
+        self.s[np.arange(10000).tolist()]
+
     def time_iloc_array(self):
         self.s.iloc[np.arange(10000)]
 
@@ -190,9 +193,15 @@ def setup(self):
              np.arange(1000)], names=['one', 'two'])
 
         import string
-        self.mistring = MultiIndex.from_product(
-            [np.arange(1000),
-             np.arange(20), list(string.ascii_letters)],
+
+        self.mi_large = MultiIndex.from_product(
+            [np.arange(1000), np.arange(20), list(string.ascii_letters)],
+            names=['one', 'two', 'three'])
+        self.mi_med = MultiIndex.from_product(
+            [np.arange(1000), np.arange(10), list('A')],
+            names=['one', 'two', 'three'])
+        self.mi_small = MultiIndex.from_product(
+            [np.arange(100), list('A'), list('A')],
             names=['one', 'two', 'three'])
 
     def time_series_xs_mi_ix(self):
@@ -215,8 +224,26 @@ def time_multiindex_get_indexer(self):
                       (0, 16), (0, 17), (0, 18),
                       (0, 19)], dtype=object))
 
+    def time_multiindex_large_get_loc(self):
+        self.mi_large.get_loc((999, 19, 'Z'))
+
+    def time_multiindex_large_get_loc_warm(self):
+        for _ in range(1000):
+            self.mi_large.get_loc((999, 19, 'Z'))
+
+    def time_multiindex_med_get_loc(self):
+        self.mi_med.get_loc((999, 9, 'A'))
+
+    def time_multiindex_med_get_loc_warm(self):
+        for _ in range(1000):
+            self.mi_med.get_loc((999, 9, 'A'))
+
     def time_multiindex_string_get_loc(self):
-        self.mistring.get_loc((999, 19, 'Z'))
+        self.mi_small.get_loc((99, 'A', 'A'))
+
+    def time_multiindex_small_get_loc_warm(self):
+        for _ in range(1000):
+            self.mi_small.get_loc((99, 'A', 'A'))
 
     def time_is_monotonic(self):
         self.miint.is_monotonic

asv_bench/benchmarks/series_methods.py

@@ -111,6 +111,7 @@ def setup(self):
     def time_series_dropna_int64(self):
         self.s.dropna()
 
+
 class series_dropna_datetime(object):
     goal_time = 0.2
 
@@ -120,3 +121,13 @@ def setup(self):
 
     def time_series_dropna_datetime(self):
         self.s.dropna()
+
+
+class series_clip(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.s = pd.Series(np.random.randn(50))
+
+    def time_series_dropna_datetime(self):
+        self.s.clip(0, 1)

ci/build_docs.sh

@@ -40,10 +40,10 @@ if [ "$DOC" ]; then
     cd /tmp/doc/build/html
     git config --global user.email "pandas-docs-bot@localhost.foo"
     git config --global user.name "pandas-docs-bot"
-    git config --global credential.helper cache
 
     # create the repo
     git init
+
     touch README
     git add README
     git commit -m "Initial commit" --allow-empty
@@ -52,8 +52,12 @@ if [ "$DOC" ]; then
     touch .nojekyll
     git add --all .
     git commit -m "Version" --allow-empty
+
     git remote remove origin
     git remote add origin "https://${PANDAS_GH_TOKEN}@github.com/pandas-docs/pandas-docs-travis.git"
+    git fetch origin
+    git remote -v
+
     git push origin gh-pages -f
 fi
 

ci/install_travis.sh

@@ -119,15 +119,7 @@ if [ "$COVERAGE" ]; then
 fi
 
 echo
-if [ "$BUILD_TEST" ]; then
-
-    # build & install testing
-    echo ["Starting installation test."]
-    bash ci/install_release_build.sh
-    conda uninstall -y cython
-    time pip install dist/*tar.gz || exit 1
-
-else
+if [ -z "$BUILD_TEST" ]; then
 
     # build but don't install
     echo "[build em]"
@@ -163,9 +155,22 @@ fi
 # w/o removing anything else
 echo
 echo "[removing installed pandas]"
-conda remove pandas --force
+conda remove pandas -y --force
 
-if [ -z "$BUILD_TEST" ]; then
+if [ "$BUILD_TEST" ]; then
+
+    # remove any installation
+    pip uninstall -y pandas
+    conda list pandas
+    pip list --format columns |grep pandas
+
+    # build & install testing
+    echo ["building release"]
+    bash scripts/build_dist_for_release.sh
+    conda uninstall -y cython
+    time pip install dist/*tar.gz || exit 1
+
+else
 
     # install our pandas
     echo

ci/requirements-3.4_SLOW.build → ci/requirements-3.6_LOCALE.build

@@ -1,6 +1,6 @@
-python=3.4*
+python=3.6*
 python-dateutil
 pytz
 nomkl
-numpy=1.10*
+numpy
 cython

ci/requirements-3.6_LOCALE.run

@@ -0,0 +1,22 @@
+python-dateutil
+pytz
+numpy
+scipy
+openpyxl
+xlsxwriter
+xlrd
+xlwt
+numexpr
+pytables
+matplotlib
+lxml
+html5lib
+jinja2
+sqlalchemy
+pymysql
+# feather-format (not available on defaults ATM)
+# psycopg2 (not avail on defaults ATM)
+beautifulsoup4
+s3fs
+xarray
+ipython

ci/requirements-3.6_LOCALE_SLOW.build

@@ -0,0 +1,6 @@
+python=3.6*
+python-dateutil
+pytz
+nomkl
+numpy
+cython

ci/requirements-3.6_LOCALE_SLOW.run

@@ -0,0 +1,22 @@
+python-dateutil
+pytz
+numpy
+scipy
+openpyxl
+xlsxwriter
+xlrd
+xlwt
+numexpr
+pytables
+matplotlib
+lxml
+html5lib
+jinja2
+sqlalchemy
+pymysql
+# feather-format (not available on defaults ATM)
+# psycopg2 (not available on defaults ATM)
+beautifulsoup4
+s3fs
+xarray
+ipython

ci/script_multi.sh

@@ -19,20 +19,26 @@ export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 429496
 echo PYTHONHASHSEED=$PYTHONHASHSEED
 
 if [ "$BUILD_TEST" ]; then
-    echo "build-test"
+    echo "[build-test]"
+
+    echo "[env]"
+    pip list --format columns |grep pandas
+
+    echo "[running]"
     cd /tmp
-    pwd
-    conda list pandas
-    echo "running"
-    python -c "import pandas; pandas.test(['-n 2'])"
+    unset PYTHONPATH
+    python -c 'import pandas; pandas.test(["-n 2", "--skip-slow", "--skip-network", "-r xX", "-m not single"])'
+
 elif [ "$DOC" ]; then
     echo "We are not running pytest as this is a doc-build"
+
 elif [ "$COVERAGE" ]; then
     echo pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
     pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
+
 else
-    echo pytest -n 2 -m "not single" --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
-    pytest -n 2 -m "not single" --junitxml=/tmp/multiple.xml $TEST_ARGS pandas # TODO: doctest
+    echo pytest -n 2 -r xX -m "not single" --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
+    pytest -n 2 -r xX -m "not single" --junitxml=/tmp/multiple.xml $TEST_ARGS pandas # TODO: doctest
 fi
 
 RET="$?"

ci/script_single.sh

@@ -20,8 +20,8 @@ elif [ "$COVERAGE" ]; then
     echo pytest -s -m "single" --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas
     pytest -s -m "single" --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas
 else
-    echo pytest -m "single" --junitxml=/tmp/single.xml $TEST_ARGS pandas
-    pytest -m "single" --junitxml=/tmp/single.xml $TEST_ARGS pandas # TODO: doctest
+    echo pytest -m "single" -r xX --junitxml=/tmp/single.xml $TEST_ARGS pandas
+    pytest -m "single" -r xX  --junitxml=/tmp/single.xml $TEST_ARGS pandas # TODO: doctest
 fi
 
 RET="$?"

circle.yml

@@ -23,9 +23,9 @@ dependencies:
        0)
          sudo apt-get install language-pack-it && ./ci/install_circle.sh JOB="2.7_COMPAT" LOCALE_OVERRIDE="it_IT.UTF-8" ;;
        1)
-         sudo apt-get install language-pack-zh-hans && ./ci/install_circle.sh JOB="3.4_SLOW" LOCALE_OVERRIDE="zh_CN.UTF-8" ;;
+         sudo apt-get install language-pack-zh-hans && ./ci/install_circle.sh JOB="3.6_LOCALE" LOCALE_OVERRIDE="zh_CN.UTF-8" ;;
        2)
-         sudo apt-get install language-pack-zh-hans && ./ci/install_circle.sh JOB="3.4" LOCALE_OVERRIDE="zh_CN.UTF-8" ;;
+         sudo apt-get install language-pack-zh-hans && ./ci/install_circle.sh JOB="3.6_LOCALE_SLOW" LOCALE_OVERRIDE="zh_CN.UTF-8" ;;
        3)
          ./ci/install_circle.sh JOB="3.5_ASCII" LOCALE_OVERRIDE="C" ;;
       esac

doc/make.py

@@ -115,15 +115,28 @@ def maybe_exclude_notebooks():
     notebooks = [os.path.join(base, 'source', nb)
                  for nb in ['style.ipynb']]
     contents = {}
-    try:
-        import nbconvert
-        nbconvert.utils.pandoc.get_pandoc_version()
-    except (ImportError, nbconvert.utils.pandoc.PandocMissing):
-        print("Warning: Pandoc is not installed. Skipping Notebooks.")
+
+    def _remove_notebooks():
         for nb in notebooks:
             with open(nb, 'rt') as f:
                 contents[nb] = f.read()
             os.remove(nb)
+
+    # Skip notebook conversion if
+    # 1. nbconvert isn't installed, or
+    # 2. nbconvert is installed, but pandoc isn't
+    try:
+        import nbconvert
+    except ImportError:
+        print("Warning: nbconvert not installed. Skipping notebooks.")
+        _remove_notebooks()
+    else:
+        try:
+            nbconvert.utils.pandoc.get_pandoc_version()
+        except nbconvert.utils.pandoc.PandocMissing:
+            print("Warning: Pandoc is not installed. Skipping notebooks.")
+            _remove_notebooks()
+
     yield
     for nb, content in contents.items():
         with open(nb, 'wt') as f: