CI: Migrate some CircleCI jobs to Azure (pandas-dev#22992)

.circleci/config.yml

@@ -1,39 +1,6 @@
 version: 2
 jobs:
 
-  # --------------------------------------------------------------------------
-  # 0. py27_compat
-  # --------------------------------------------------------------------------
-  py27_compat:
-    docker:
-      - image: continuumio/miniconda:latest
-      # databases configuration
-      - image: circleci/postgres:9.6.5-alpine-ram
-        environment:
-            POSTGRES_USER: postgres
-            POSTGRES_DB: pandas_nosetest
-      - image: circleci/mysql:8-ram
-        environment:
-          MYSQL_USER: "root"
-          MYSQL_HOST: "localhost"
-          MYSQL_ALLOW_EMPTY_PASSWORD: "true"
-          MYSQL_DATABASE: "pandas_nosetest"
-    environment:
-      JOB: "2.7_COMPAT"
-      ENV_FILE: "ci/circle-27-compat.yaml"
-      LOCALE_OVERRIDE: "it_IT.UTF-8"
-      MINICONDA_DIR: /home/ubuntu/miniconda3
-    steps:
-      - checkout
-      - run:
-          name: build
-          command: |
-            ./ci/install_circle.sh
-            ./ci/show_circle.sh
-      - run:
-          name: test
-          command: ./ci/run_circle.sh  --skip-slow --skip-network
-
   # --------------------------------------------------------------------------
   # 1. py36_locale
   # --------------------------------------------------------------------------
@@ -62,86 +29,14 @@ jobs:
       - run:
           name: build
           command: |
-            ./ci/install_circle.sh
-            ./ci/show_circle.sh
+            ./ci/circle/install_circle.sh
+            ./ci/circle/show_circle.sh
       - run:
           name: test
-          command: ./ci/run_circle.sh  --skip-slow --skip-network
-
-  # --------------------------------------------------------------------------
-  # 2. py36_locale_slow
-  # --------------------------------------------------------------------------
-  py36_locale_slow:
-    docker:
-      - image: continuumio/miniconda:latest
-      # databases configuration
-      - image: circleci/postgres:9.6.5-alpine-ram
-        environment:
-            POSTGRES_USER: postgres
-            POSTGRES_DB: pandas_nosetest
-      - image: circleci/mysql:8-ram
-        environment:
-          MYSQL_USER: "root"
-          MYSQL_HOST: "localhost"
-          MYSQL_ALLOW_EMPTY_PASSWORD: "true"
-          MYSQL_DATABASE: "pandas_nosetest"
-
-    environment:
-      JOB: "3.6_LOCALE_SLOW"
-      ENV_FILE: "ci/circle-36-locale_slow.yaml"
-      LOCALE_OVERRIDE: "zh_CN.UTF-8"
-      MINICONDA_DIR: /home/ubuntu/miniconda3
-    steps:
-      - checkout
-      - run:
-          name: build
-          command: |
-            ./ci/install_circle.sh
-            ./ci/show_circle.sh
-      - run:
-          name: test
-          command: ./ci/run_circle.sh  --only-slow --skip-network
-
-  # --------------------------------------------------------------------------
-  # 3. py35_ascii
-  # --------------------------------------------------------------------------
-  py35_ascii:
-    docker:
-      - image: continuumio/miniconda:latest
-      # databases configuration
-      - image: circleci/postgres:9.6.5-alpine-ram
-        environment:
-            POSTGRES_USER: postgres
-            POSTGRES_DB: pandas_nosetest
-      - image: circleci/mysql:8-ram
-        environment:
-          MYSQL_USER: "root"
-          MYSQL_HOST: "localhost"
-          MYSQL_ALLOW_EMPTY_PASSWORD: "true"
-          MYSQL_DATABASE: "pandas_nosetest"
-
-    environment:
-      JOB: "3.5_ASCII"
-      ENV_FILE: "ci/circle-35-ascii.yaml"
-      LOCALE_OVERRIDE: "C"
-      MINICONDA_DIR: /home/ubuntu/miniconda3
-    steps:
-      - checkout
-      - run:
-          name: build
-          command: |
-            ./ci/install_circle.sh
-            ./ci/show_circle.sh
-      - run:
-          name: test
-          command: ./ci/run_circle.sh  --skip-slow --skip-network
-
+          command: ./ci/circle/run_circle.sh  --skip-slow --skip-network
 
 workflows:
   version: 2
   build_and_test:
     jobs:
-      - py27_compat
       - py36_locale
-      - py36_locale_slow
-      - py35_ascii

.travis.yml

@@ -116,10 +116,10 @@ after_success:
 after_script:
   - echo "after_script start"
   - source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
-  - if [ -e /tmp/single.xml ]; then
-    ci/print_skipped.py /tmp/single.xml;
+  - if [ -e test-data-single.xml ]; then
+    ci/print_skipped.py test-data-single.xml;
     fi
-  - if [ -e /tmp/multiple.xml ]; then
-    ci/print_skipped.py /tmp/multiple.xml;
+  - if [ -e test-data-multiple.xml ]; then
+    ci/print_skipped.py test-data-multiple.xml;
     fi
   - echo "after_script done"

azure-pipelines.yml

@@ -7,10 +7,10 @@ jobs:
   parameters:
     name: macOS
     vmImage: xcode9-macos10.13
-# - template: ci/azure/linux.yml
-#   parameters:
-#     name: Linux
-#     vmImage: ubuntu-16.04
+- template: ci/azure/linux.yml
+  parameters:
+    name: Linux
+    vmImage: ubuntu-16.04
 
 # Windows Python 2.7 needs VC 9.0 installed, and not sure
 # how to make that a conditional task, so for now these are

ci/azure-37-locale.yaml

@@ -0,0 +1,35 @@
+name: pandas
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - beautifulsoup4
+  - cython>=0.28.2
+  - html5lib
+  - ipython
+  - jinja2
+  - lxml
+  - matplotlib
+  - nomkl
+  - numexpr
+  - numpy
+  - openpyxl=2.5.5
+  - psycopg2
+  - pymysql
+  - pytables
+  - python-dateutil
+  - python=3.6*
+  - pytz
+  - s3fs
+  - scipy
+  - sqlalchemy
+  - xarray
+  - xlrd
+  - xlsxwriter
+  - xlwt
+  # universal
+  - pytest
+  - pytest-xdist
+  - moto
+  - pip:
+    - hypothesis>=3.58.0

ci/azure/linux.yml

@@ -0,0 +1,56 @@
+parameters:
+  name: ''
+  vmImage: ''
+
+jobs:
+- job: ${{ parameters.name }}
+  pool:
+    vmImage: ${{ parameters.vmImage }}
+  strategy:
+    maxParallel: 11
+    matrix:
+      py27_np_19:
+        ENV_FILE: ci/azure-27-compat.yaml
+        CONDA_PY: "27"
+        CONDA_ENV: pandas
+        TEST_ARGS: "--skip-slow --skip-network"
+
+      py36_locale:
+        ENV_FILE: ci/azure-37-locale.yaml
+        CONDA_PY: "37"
+        CONDA_ENV: pandas
+        TEST_ARGS: "--skip-slow --skip-network"
+        LOCALE_OVERRIDE: "zh_CN.UTF-8"
+
+      py36_locale_slow:
+        ENV_FILE: ci/azure-36-locale_slow.yaml
+        CONDA_PY: "36"
+        CONDA_ENV: pandas
+        TEST_ARGS: "--only-slow --skip-network"
+
+  steps:
+    - script: |
+        if [ "$(uname)" == "Linux" ]; then sudo apt-get install -y libc6-dev-i386; fi
+        echo "Installing Miniconda"{
+        ci/incremental/install_miniconda.sh
+        export PATH=$HOME/miniconda3/bin:$PATH
+        echo "Setting up Conda environment"
+        ci/incremental/setup_conda_environment.sh
+      displayName: 'Before Install'
+    - script: |
+        export PATH=$HOME/miniconda3/bin:$PATH
+        ci/incremental/build.sh
+      displayName: 'Build'
+    - script: |
+        export PATH=$HOME/miniconda3/bin:$PATH
+        ci/script_single.sh
+        ci/script_multi.sh
+        echo "[Test done]"
+      displayName: 'Test'
+    - script: |
+        export PATH=$HOME/miniconda3/bin:$PATH
+        source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
+    - task: PublishTestResults@2
+      inputs:
+        testResultsFiles: 'test-data-*.xml'
+        testRunTitle: 'Linux'

ci/azure/macos.yml

@@ -39,5 +39,5 @@ jobs:
         source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
     - task: PublishTestResults@2
       inputs:
-        testResultsFiles: '/tmp/*.xml'
+        testResultsFiles: 'test-data-*.xml'
         testRunTitle: 'MacOS-35'

ci/incremental/setup_conda_environment.sh

@@ -27,13 +27,17 @@ set -v
 # w/o removing anything else
 echo
 echo "[removing installed pandas]"
-conda remove pandas -y --force
-pip uninstall -y pandas
+conda remove pandas -y --force || true
+pip uninstall -y pandas || true
 
 echo
 echo "[no installed pandas]"
 conda list pandas
 
+if [ -n "$LOCALE_OVERRIDE" ]; then
+    sudo locale-gen "$LOCALE_OVERRIDE"
+fi
+
 # # Install the compiler toolchain
 # if [[ $(uname) == Linux ]]; then
 #     if [[ "$CONDA_SUBDIR" == "linux-32" || "$BITS32" == "yes" ]] ; then

ci/script_multi.sh

@@ -27,17 +27,17 @@ if [ "$DOC" ]; then
     echo "We are not running pytest as this is a doc-build"
 
 elif [ "$COVERAGE" ]; then
-    echo pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas
-    pytest      -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas
+    echo pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas
+    pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas
 
 elif [ "$SLOW" ]; then
     TEST_ARGS="--only-slow --skip-network"
-    echo pytest -m "not single and slow" -v --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas
-    pytest      -m "not single and slow" -v --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas
+    echo pytest -m "not single and slow" -v --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas
+    pytest      -m "not single and slow" -v --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas
 
 else
-    echo pytest -n 2 -m "not single" --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas
-    pytest      -n 2 -m "not single" --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas # TODO: doctest
+    echo pytest -n 2 -m "not single" --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas
+    pytest      -n 2 -m "not single" --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas # TODO: doctest
 
 fi
 

ci/script_single.sh

@@ -5,8 +5,9 @@ echo "[script_single]"
 source activate pandas
 
 if [ -n "$LOCALE_OVERRIDE" ]; then
+    echo "Setting LC_ALL and LANG to $LOCALE_OVERRIDE"
     export LC_ALL="$LOCALE_OVERRIDE";
-    echo "Setting LC_ALL to $LOCALE_OVERRIDE"
+    export LANG="$LOCALE_OVERRIDE";
 
     pycmd='import pandas; print("pandas detected console encoding: %s" % pandas.get_option("display.encoding"))'
     python -c "$pycmd"
@@ -25,14 +26,13 @@ if [ "$DOC" ]; then
     echo "We are not running pytest as this is a doc-build"
 
 elif [ "$COVERAGE" ]; then
-    echo pytest -s -m "single" --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas
-    pytest      -s -m "single" --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas
-
+    echo pytest -s -m "single" --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=test-data-single.xml $TEST_ARGS pandas
+    pytest      -s -m "single" --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=test-data-single.xml $TEST_ARGS pandas
     echo pytest -s --strict scripts
     pytest      -s --strict scripts
 else
-    echo pytest -m "single" --junitxml=/tmp/single.xml --strict $TEST_ARGS pandas
-    pytest      -m "single" --junitxml=/tmp/single.xml --strict $TEST_ARGS pandas # TODO: doctest
+    echo pytest -m "single" --junitxml=test-data-single.xml --strict $TEST_ARGS pandas
+    pytest      -m "single" --junitxml=test-data-single.xml --strict $TEST_ARGS pandas
 
 fi
 

doc/source/contributing.rst

@@ -684,7 +684,7 @@ Test-driven development/code writing
 ------------------------------------
 
 *pandas* is serious about testing and strongly encourages contributors to embrace
-`test-driven development (TDD) <http://en.wikipedia.org/wiki/Test-driven_development>`_.
+`test-driven development (TDD) <https://en.wikipedia.org/wiki/Test-driven_development>`_.
 This development process "relies on the repetition of a very short development cycle:
 first the developer writes an (initially failing) automated test case that defines a desired
 improvement or new function, then produces the minimum amount of code to pass that test."