From 4524b316494652410c6236e5579456552e51d03b Mon Sep 17 00:00:00 2001
From: "azure-pipelines[bot]" <azure-pipelines[bot]@users.noreply.github.com>
Date: Wed, 19 Sep 2018 10:42:24 -0500
Subject: [PATCH] Set up CI with Azure Pipelines (#22760)

---
 .travis.yml                                   |  5 -
 appveyor.yml                                  | 91 ------------------
 azure-pipelines.yml                           | 25 +++++
 ci/{travis-35-osx.yaml => azure-macos-35.yml} |  0
 ...appveyor-27.yaml => azure-windows-27.yaml} |  0
 ...appveyor-36.yaml => azure-windows-36.yaml} |  0
 ci/azure/macos.yml                            | 39 ++++++++
 ci/azure/windows-py27.yml                     | 41 +++++++++
 ci/azure/windows.yml                          | 32 +++++++
 ci/incremental/build.cmd                      | 10 ++
 ci/incremental/build.sh                       | 18 ++++
 ci/incremental/install_miniconda.sh           | 19 ++++
 ci/incremental/setup_conda_environment.cmd    | 21 +++++
 ci/incremental/setup_conda_environment.sh     | 48 ++++++++++
 ci/install.ps1                                | 92 -------------------
 15 files changed, 253 insertions(+), 188 deletions(-)
 delete mode 100644 appveyor.yml
 create mode 100644 azure-pipelines.yml
 rename ci/{travis-35-osx.yaml => azure-macos-35.yml} (100%)
 rename ci/{appveyor-27.yaml => azure-windows-27.yaml} (100%)
 rename ci/{appveyor-36.yaml => azure-windows-36.yaml} (100%)
 create mode 100644 ci/azure/macos.yml
 create mode 100644 ci/azure/windows-py27.yml
 create mode 100644 ci/azure/windows.yml
 create mode 100644 ci/incremental/build.cmd
 create mode 100755 ci/incremental/build.sh
 create mode 100755 ci/incremental/install_miniconda.sh
 create mode 100644 ci/incremental/setup_conda_environment.cmd
 create mode 100755 ci/incremental/setup_conda_environment.sh
 delete mode 100644 ci/install.ps1

diff --git a/.travis.yml b/.travis.yml
index 76f4715a4abb2e..a180e83eeec216 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -30,11 +30,6 @@ matrix:
     exclude:
       # Exclude the default Python 3.5 build
       - python: 3.5
-    include:
-    - os: osx
-      language: generic
-      env:
-        - JOB="3.5, OSX" ENV_FILE="ci/travis-35-osx.yaml" TEST_ARGS="--skip-slow --skip-network"
 
     - dist: trusty
       env:
diff --git a/appveyor.yml b/appveyor.yml
deleted file mode 100644
index c6199c1493f22a..00000000000000
--- a/appveyor.yml
+++ /dev/null
@@ -1,91 +0,0 @@
-# With infos from
-# http://tjelvarolsson.com/blog/how-to-continuously-test-your-python-code-on-windows-using-appveyor/
-# https://packaging.python.org/en/latest/appveyor/
-# https://github.com/rmcgibbo/python-appveyor-conda-example
-
-# Backslashes in quotes need to be escaped: \ -> "\\"
-
-matrix:
-  fast_finish: true     # immediately finish build once one of the jobs fails.
-
-environment:
-  global:
-    # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
-    # /E:ON and /V:ON options are not enabled in the batch script interpreter
-    # See: http://stackoverflow.com/a/13751649/163740
-    CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\ci\\run_with_env.cmd"
-    clone_folder: C:\projects\pandas
-    PANDAS_TESTING_MODE: "deprecate"
-
-  matrix:
-
-    - CONDA_ROOT: "C:\\Miniconda3_64"
-      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
-      PYTHON_VERSION: "3.6"
-      PYTHON_ARCH: "64"
-      CONDA_PY: "36"
-      CONDA_NPY: "113"
-
-    - CONDA_ROOT: "C:\\Miniconda3_64"
-      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
-      PYTHON_VERSION: "2.7"
-      PYTHON_ARCH: "64"
-      CONDA_PY: "27"
-      CONDA_NPY: "110"
-
-# We always use a 64-bit machine, but can build x86 distributions
-# with the PYTHON_ARCH variable (which is used by CMD_IN_ENV).
-platform:
-    - x64
-
-# all our python builds have to happen in tests_script...
-build: false
-
-install:
-  # cancel older builds for the same PR
-  - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
-        https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
-        Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
-        throw "There are newer queued builds for this pull request, failing early." }
-
-  # this installs the appropriate Miniconda (Py2/Py3, 32/64 bit)
-  # updates conda & installs: conda-build jinja2 anaconda-client
-  - powershell .\ci\install.ps1
-  - SET PATH=%CONDA_ROOT%;%CONDA_ROOT%\Scripts;%PATH%
-  - echo "install"
-  - cd
-  - ls -ltr
-  - git tag --sort v:refname
-
-  # this can conflict with git
-  - cmd: rmdir C:\cygwin /s /q
-
-  # install our build environment
-  - cmd: conda config --set show_channel_urls true --set always_yes true --set changeps1 false
-  - cmd: conda update -q conda
-  - cmd: conda config --set ssl_verify false
-
-  # add the pandas channel *before* defaults to have defaults take priority
-  - cmd: conda config --add channels conda-forge
-  - cmd: conda config --add channels pandas
-  - cmd: conda config --remove channels defaults
-  - cmd: conda config --add channels defaults
-
-  # this is now the downloaded conda...
-  - cmd: conda info -a
-
-  # create our env
-  - cmd: conda env create -q -n pandas --file=ci\appveyor-%CONDA_PY%.yaml
-  - cmd: activate pandas
-  - cmd: conda list -n pandas
-  # uninstall pandas if it's present
-  - cmd: conda remove pandas -y --force & exit 0
-  - cmd: pip uninstall -y pandas & exit 0
-
-  # build em using the local source checkout in the correct windows env
-  - cmd: '%CMD_IN_ENV% python setup.py build_ext --inplace'
-
-test_script:
-  # tests
-  - cmd: activate pandas
-  - cmd: test.bat
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
new file mode 100644
index 00000000000000..c82dafa2249613
--- /dev/null
+++ b/azure-pipelines.yml
@@ -0,0 +1,25 @@
+# Adapted from https://github.com/numba/numba/blob/master/azure-pipelines.yml
+jobs:
+# Mac and Linux could potentially use the same template
+# except it isn't clear how to use a different build matrix
+# for each, so for now they are separate
+- template: ci/azure/macos.yml
+  parameters:
+    name: macOS
+    vmImage: xcode9-macos10.13
+# - template: ci/azure/linux.yml
+#   parameters:
+#     name: Linux
+#     vmImage: ubuntu-16.04
+
+# Windows Python 2.7 needs VC 9.0 installed, and not sure
+# how to make that a conditional task, so for now these are
+# separate templates as well
+- template: ci/azure/windows.yml
+  parameters:
+    name: Windows
+    vmImage: vs2017-win2017
+- template: ci/azure/windows-py27.yml
+  parameters:
+    name: WindowsPy27
+    vmImage: vs2017-win2017
diff --git a/ci/travis-35-osx.yaml b/ci/azure-macos-35.yml
similarity index 100%
rename from ci/travis-35-osx.yaml
rename to ci/azure-macos-35.yml
diff --git a/ci/appveyor-27.yaml b/ci/azure-windows-27.yaml
similarity index 100%
rename from ci/appveyor-27.yaml
rename to ci/azure-windows-27.yaml
diff --git a/ci/appveyor-36.yaml b/ci/azure-windows-36.yaml
similarity index 100%
rename from ci/appveyor-36.yaml
rename to ci/azure-windows-36.yaml
diff --git a/ci/azure/macos.yml b/ci/azure/macos.yml
new file mode 100644
index 00000000000000..25b66615dac7e5
--- /dev/null
+++ b/ci/azure/macos.yml
@@ -0,0 +1,39 @@
+parameters:
+  name: ''
+  vmImage: ''
+
+jobs:
+- job: ${{ parameters.name }}
+  pool:
+    vmImage: ${{ parameters.vmImage }}
+  strategy:
+    maxParallel: 11
+    matrix:
+      py35_np_110:
+        ENV_FILE: ci/azure-macos-35.yml
+        CONDA_PY: "35"
+        CONDA_ENV: pandas
+        TEST_ARGS: "--skip-slow --skip-network"
+
+  steps:
+    - script: |
+        if [ "$(uname)" == "Linux" ]; then sudo apt-get install -y libc6-dev-i386; fi
+        echo "Installing Miniconda"
+        ci/incremental/install_miniconda.sh
+        export PATH=$HOME/miniconda3/bin:$PATH
+        echo "Setting up Conda environment"
+        ci/incremental/setup_conda_environment.sh
+      displayName: 'Before Install'
+    - script: |
+        export PATH=$HOME/miniconda3/bin:$PATH
+        ci/incremental/build.sh
+      displayName: 'Build'
+    - script: |
+        export PATH=$HOME/miniconda3/bin:$PATH
+        ci/script_single.sh
+        ci/script_multi.sh
+        echo "[Test done]"
+      displayName: 'Test'
+    - script: |
+        export PATH=$HOME/miniconda3/bin:$PATH
+        source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
diff --git a/ci/azure/windows-py27.yml b/ci/azure/windows-py27.yml
new file mode 100644
index 00000000000000..e60844896b71cf
--- /dev/null
+++ b/ci/azure/windows-py27.yml
@@ -0,0 +1,41 @@
+parameters:
+  name: ''
+  vmImage: ''
+
+jobs:
+- job: ${{ parameters.name }}
+  pool:
+    vmImage: ${{ parameters.vmImage }}
+  strategy:
+    maxParallel: 11
+    matrix:
+      py36_np14:
+        ENV_FILE: ci/azure-windows-27.yml
+        CONDA_PY: "27"
+        CONDA_ENV: pandas
+
+  steps:
+    - task: CondaEnvironment@1
+      inputs:
+        updateConda: no
+        packageSpecs: ''
+
+    # Need to install VC 9.0 only for Python 2.7
+    # Once we understand how to do tasks conditional on build matrix variables
+    # we could merge this into azure-windows.yml
+    - powershell: |
+        $wc = New-Object net.webclient
+        $wc.Downloadfile("https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi", "VCForPython27.msi")
+        Start-Process "VCForPython27.msi" /qn -Wait
+      displayName: 'Install VC 9.0'
+
+    - script: |
+        ci\\incremental\\setup_conda_environment.cmd
+      displayName: 'Before Install'
+    - script: |
+        ci\\incremental\\build.cmd
+      displayName: 'Build'
+    - script: |
+        call activate %CONDA_ENV%
+        pytest --skip-slow --skip-network pandas -n 2 -r sxX --strict %*
+      displayName: 'Test'
diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml
new file mode 100644
index 00000000000000..6090139fb4f3e3
--- /dev/null
+++ b/ci/azure/windows.yml
@@ -0,0 +1,32 @@
+parameters:
+  name: ''
+  vmImage: ''
+
+jobs:
+- job: ${{ parameters.name }}
+  pool:
+    vmImage: ${{ parameters.vmImage }}
+  strategy:
+    maxParallel: 11
+    matrix:
+      py36_np14:
+        ENV_FILE: ci/azure-windows-36.yml
+        CONDA_PY: "36"
+        CONDA_ENV: pandas
+
+  steps:
+    - task: CondaEnvironment@1
+      inputs:
+        updateConda: no
+        packageSpecs: ''
+
+    - script: |
+        ci\\incremental\\setup_conda_environment.cmd
+      displayName: 'Before Install'
+    - script: |
+        ci\\incremental\\build.cmd
+      displayName: 'Build'
+    - script: |
+        call activate %CONDA_ENV%
+        pytest --skip-slow --skip-network pandas -n 2 -r sxX --strict %*
+      displayName: 'Test'
diff --git a/ci/incremental/build.cmd b/ci/incremental/build.cmd
new file mode 100644
index 00000000000000..d2fd06d7d9e508
--- /dev/null
+++ b/ci/incremental/build.cmd
@@ -0,0 +1,10 @@
+@rem https://github.com/numba/numba/blob/master/buildscripts/incremental/build.cmd
+call activate %CONDA_ENV%
+
+@rem Build numba extensions without silencing compile errors
+python setup.py build_ext -q --inplace
+
+@rem Install pandas locally
+python -m pip install -e .
+
+if %errorlevel% neq 0 exit /b %errorlevel%
diff --git a/ci/incremental/build.sh b/ci/incremental/build.sh
new file mode 100755
index 00000000000000..8f2301a3b7ef50
--- /dev/null
+++ b/ci/incremental/build.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+source activate $CONDA_ENV
+
+# Make sure any error below is reported as such
+set -v -e
+
+echo "[building extensions]"
+python setup.py build_ext -q --inplace
+python -m pip install -e .
+
+echo
+echo "[show environment]"
+conda list
+
+echo
+echo "[done]"
+exit 0
diff --git a/ci/incremental/install_miniconda.sh b/ci/incremental/install_miniconda.sh
new file mode 100755
index 00000000000000..a47dfdb324b348
--- /dev/null
+++ b/ci/incremental/install_miniconda.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -v -e
+
+# Install Miniconda
+unamestr=`uname`
+if [[ "$unamestr" == 'Linux' ]]; then
+    if [[ "$BITS32" == "yes" ]]; then
+        wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86.sh -O miniconda.sh
+    else
+        wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+    fi
+elif [[ "$unamestr" == 'Darwin' ]]; then
+    wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh
+else
+  echo Error
+fi
+chmod +x miniconda.sh
+./miniconda.sh -b
diff --git a/ci/incremental/setup_conda_environment.cmd b/ci/incremental/setup_conda_environment.cmd
new file mode 100644
index 00000000000000..b4446c49fabd3a
--- /dev/null
+++ b/ci/incremental/setup_conda_environment.cmd
@@ -0,0 +1,21 @@
+@rem https://github.com/numba/numba/blob/master/buildscripts/incremental/setup_conda_environment.cmd
+@rem The cmd /C hack circumvents a regression where conda installs a conda.bat
+@rem script in non-root environments.
+set CONDA_INSTALL=cmd /C conda install -q -y
+set PIP_INSTALL=pip install -q
+
+@echo on
+
+@rem Deactivate any environment
+call deactivate
+@rem Display root environment (for debugging)
+conda list
+@rem Clean up any left-over from a previous build
+conda remove --all -q -y -n %CONDA_ENV%
+@rem Scipy, CFFI, jinja2 and IPython are optional dependencies, but exercised in the test suite
+conda env create -n %CONDA_ENV% --file=ci\azure-windows-%CONDA_PY%.yaml
+
+call activate %CONDA_ENV%
+conda list
+
+if %errorlevel% neq 0 exit /b %errorlevel%
diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh
new file mode 100755
index 00000000000000..c716a391386448
--- /dev/null
+++ b/ci/incremental/setup_conda_environment.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -v -e
+
+CONDA_INSTALL="conda install -q -y"
+PIP_INSTALL="pip install -q"
+
+# Deactivate any environment
+source deactivate
+# Display root environment (for debugging)
+conda list
+# Clean up any left-over from a previous build
+# (note workaround for https://github.com/conda/conda/issues/2679:
+#  `conda env remove` issue)
+conda remove --all -q -y -n $CONDA_ENV
+
+echo
+echo "[create env]"
+time conda env create -q -n "${CONDA_ENV}" --file="${ENV_FILE}" || exit 1
+
+# Activate first
+set +v
+source activate $CONDA_ENV
+set -v
+
+# remove any installed pandas package
+# w/o removing anything else
+echo
+echo "[removing installed pandas]"
+conda remove pandas -y --force
+pip uninstall -y pandas
+
+echo
+echo "[no installed pandas]"
+conda list pandas
+
+# # Install the compiler toolchain
+# if [[ $(uname) == Linux ]]; then
+#     if [[ "$CONDA_SUBDIR" == "linux-32" || "$BITS32" == "yes" ]] ; then
+#         $CONDA_INSTALL gcc_linux-32 gxx_linux-32
+#     else
+#         $CONDA_INSTALL gcc_linux-64 gxx_linux-64
+#     fi
+# elif  [[ $(uname) == Darwin ]]; then
+#     $CONDA_INSTALL clang_osx-64 clangxx_osx-64
+#     # Install llvm-openmp and intel-openmp on OSX too
+#     $CONDA_INSTALL llvm-openmp intel-openmp
+# fi
diff --git a/ci/install.ps1 b/ci/install.ps1
deleted file mode 100644
index 64ec7f81884cd1..00000000000000
--- a/ci/install.ps1
+++ /dev/null
@@ -1,92 +0,0 @@
-# Sample script to install Miniconda under Windows
-# Authors: Olivier Grisel, Jonathan Helmus and Kyle Kastner, Robert McGibbon
-# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
-
-$MINICONDA_URL = "http://repo.continuum.io/miniconda/"
-
-
-function DownloadMiniconda ($python_version, $platform_suffix) {
-    $webclient = New-Object System.Net.WebClient
-    $filename = "Miniconda3-latest-Windows-" + $platform_suffix + ".exe"
-    $url = $MINICONDA_URL + $filename
-
-    $basedir = $pwd.Path + "\"
-    $filepath = $basedir + $filename
-    if (Test-Path $filename) {
-        Write-Host "Reusing" $filepath
-        return $filepath
-    }
-
-    # Download and retry up to 3 times in case of network transient errors.
-    Write-Host "Downloading" $filename "from" $url
-    $retry_attempts = 2
-    for($i=0; $i -lt $retry_attempts; $i++){
-        try {
-            $webclient.DownloadFile($url, $filepath)
-            break
-        }
-        Catch [Exception]{
-            Start-Sleep 1
-        }
-   }
-   if (Test-Path $filepath) {
-       Write-Host "File saved at" $filepath
-   } else {
-       # Retry once to get the error message if any at the last try
-       $webclient.DownloadFile($url, $filepath)
-   }
-   return $filepath
-}
-
-
-function InstallMiniconda ($python_version, $architecture, $python_home) {
-    Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home
-    if (Test-Path $python_home) {
-        Write-Host $python_home "already exists, skipping."
-        return $false
-    }
-    if ($architecture -match "32") {
-        $platform_suffix = "x86"
-    } else {
-        $platform_suffix = "x86_64"
-    }
-
-    $filepath = DownloadMiniconda $python_version $platform_suffix
-    Write-Host "Installing" $filepath "to" $python_home
-    $install_log = $python_home + ".log"
-    $args = "/S /D=$python_home"
-    Write-Host $filepath $args
-    Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru
-    if (Test-Path $python_home) {
-        Write-Host "Python $python_version ($architecture) installation complete"
-    } else {
-        Write-Host "Failed to install Python in $python_home"
-        Get-Content -Path $install_log
-        Exit 1
-    }
-}
-
-
-function InstallCondaPackages ($python_home, $spec) {
-    $conda_path = $python_home + "\Scripts\conda.exe"
-    $args = "install --yes " + $spec
-    Write-Host ("conda " + $args)
-    Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru
-}
-
-function UpdateConda ($python_home) {
-    $conda_path = $python_home + "\Scripts\conda.exe"
-    Write-Host "Updating conda..."
-    $args = "update --yes conda"
-    Write-Host $conda_path $args
-    Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru
-}
-
-
-function main () {
-    InstallMiniconda "3.5" $env:PYTHON_ARCH $env:CONDA_ROOT
-    UpdateConda $env:CONDA_ROOT
-    InstallCondaPackages $env:CONDA_ROOT "conda-build jinja2 anaconda-client"
-}
-
-main