diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index c6199c1493f22a..00000000000000 --- a/appveyor.yml +++ /dev/null @@ -1,91 +0,0 @@ -# With infos from -# http://tjelvarolsson.com/blog/how-to-continuously-test-your-python-code-on-windows-using-appveyor/ -# https://packaging.python.org/en/latest/appveyor/ -# https://github.com/rmcgibbo/python-appveyor-conda-example - -# Backslashes in quotes need to be escaped: \ -> "\\" - -matrix: - fast_finish: true # immediately finish build once one of the jobs fails. - -environment: - global: - # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the - # /E:ON and /V:ON options are not enabled in the batch script interpreter - # See: http://stackoverflow.com/a/13751649/163740 - CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\ci\\run_with_env.cmd" - clone_folder: C:\projects\pandas - PANDAS_TESTING_MODE: "deprecate" - - matrix: - - - CONDA_ROOT: "C:\\Miniconda3_64" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - PYTHON_VERSION: "3.6" - PYTHON_ARCH: "64" - CONDA_PY: "36" - CONDA_NPY: "113" - - - CONDA_ROOT: "C:\\Miniconda3_64" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - PYTHON_VERSION: "2.7" - PYTHON_ARCH: "64" - CONDA_PY: "27" - CONDA_NPY: "110" - -# We always use a 64-bit machine, but can build x86 distributions -# with the PYTHON_ARCH variable (which is used by CMD_IN_ENV). -platform: - - x64 - -# all our python builds have to happen in tests_script... -build: false - -install: - # cancel older builds for the same PR - - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` - https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` - Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` - throw "There are newer queued builds for this pull request, failing early." } - - # this installs the appropriate Miniconda (Py2/Py3, 32/64 bit) - # updates conda & installs: conda-build jinja2 anaconda-client - - powershell .\ci\install.ps1 - - SET PATH=%CONDA_ROOT%;%CONDA_ROOT%\Scripts;%PATH% - - echo "install" - - cd - - ls -ltr - - git tag --sort v:refname - - # this can conflict with git - - cmd: rmdir C:\cygwin /s /q - - # install our build environment - - cmd: conda config --set show_channel_urls true --set always_yes true --set changeps1 false - - cmd: conda update -q conda - - cmd: conda config --set ssl_verify false - - # add the pandas channel *before* defaults to have defaults take priority - - cmd: conda config --add channels conda-forge - - cmd: conda config --add channels pandas - - cmd: conda config --remove channels defaults - - cmd: conda config --add channels defaults - - # this is now the downloaded conda... - - cmd: conda info -a - - # create our env - - cmd: conda env create -q -n pandas --file=ci\appveyor-%CONDA_PY%.yaml - - cmd: activate pandas - - cmd: conda list -n pandas - # uninstall pandas if it's present - - cmd: conda remove pandas -y --force & exit 0 - - cmd: pip uninstall -y pandas & exit 0 - - # build em using the local source checkout in the correct windows env - - cmd: '%CMD_IN_ENV% python setup.py build_ext --inplace' - -test_script: - # tests - - cmd: activate pandas - - cmd: test.bat diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000000000..c82dafa2249613 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,25 @@ +# Adapted from https://github.com/numba/numba/blob/master/azure-pipelines.yml +jobs: +# Mac and Linux could potentially use the same template +# except it isn't clear how to use a different build matrix +# for each, so for now they are separate +- template: ci/azure/macos.yml + parameters: + name: macOS + vmImage: xcode9-macos10.13 +# - template: ci/azure/linux.yml +# parameters: +# name: Linux +# vmImage: ubuntu-16.04 + +# Windows Python 2.7 needs VC 9.0 installed, and not sure +# how to make that a conditional task, so for now these are +# separate templates as well +- template: ci/azure/windows.yml + parameters: + name: Windows + vmImage: vs2017-win2017 +- template: ci/azure/windows-py27.yml + parameters: + name: WindowsPy27 + vmImage: vs2017-win2017 diff --git a/ci/azure-macos-35.yml b/ci/azure-macos-35.yml new file mode 100644 index 00000000000000..a36f748ded8129 --- /dev/null +++ b/ci/azure-macos-35.yml @@ -0,0 +1,28 @@ +name: pandas +channels: + - defaults +dependencies: + - beautifulsoup4 + - bottleneck + - cython>=0.28.2 + - html5lib + - jinja2 + - lxml + - matplotlib + - nomkl + - numexpr + - numpy=1.10.4 + - openpyxl=2.5.5 + - pytables + - python=3.5* + - pytz + - xarray + - xlrd + - xlsxwriter + - xlwt + # universal + - pytest + - pytest-xdist + - pip: + - python-dateutil==2.5.3 + - hypothesis>=3.58.0 diff --git a/ci/azure-windows-27.yaml b/ci/azure-windows-27.yaml new file mode 100644 index 00000000000000..bcd9ddee1715ed --- /dev/null +++ b/ci/azure-windows-27.yaml @@ -0,0 +1,31 @@ +name: pandas +channels: + - defaults + - conda-forge +dependencies: + - beautifulsoup4 + - bottleneck + - dateutil + - gcsfs + - html5lib + - jinja2=2.8 + - lxml + - matplotlib + - numexpr + - numpy=1.12* + - openpyxl=2.5.5 + - pytables + - python=2.7.* + - pytz + - s3fs + - scipy + - sqlalchemy + - xlrd + - xlsxwriter + - xlwt + # universal + - cython>=0.28.2 + - pytest + - pytest-xdist + - moto + - hypothesis>=3.58.0 diff --git a/ci/azure-windows-36.yaml b/ci/azure-windows-36.yaml new file mode 100644 index 00000000000000..6230e9b6a1885f --- /dev/null +++ b/ci/azure-windows-36.yaml @@ -0,0 +1,28 @@ +name: pandas +channels: + - defaults + - conda-forge +dependencies: + - blosc + - bottleneck + - fastparquet + - feather-format + - matplotlib + - numexpr + - numpy=1.14* + - openpyxl=2.5.5 + - pyarrow + - pytables + - python-dateutil + - python=3.6.* + - pytz + - scipy + - thrift=0.10* + - xlrd + - xlsxwriter + - xlwt + # universal + - cython>=0.28.2 + - pytest + - pytest-xdist + - hypothesis>=3.58.0 diff --git a/ci/azure/macos.yml b/ci/azure/macos.yml new file mode 100644 index 00000000000000..25b66615dac7e5 --- /dev/null +++ b/ci/azure/macos.yml @@ -0,0 +1,39 @@ +parameters: + name: '' + vmImage: '' + +jobs: +- job: ${{ parameters.name }} + pool: + vmImage: ${{ parameters.vmImage }} + strategy: + maxParallel: 11 + matrix: + py35_np_110: + ENV_FILE: ci/azure-macos-35.yml + CONDA_PY: "35" + CONDA_ENV: pandas + TEST_ARGS: "--skip-slow --skip-network" + + steps: + - script: | + if [ "$(uname)" == "Linux" ]; then sudo apt-get install -y libc6-dev-i386; fi + echo "Installing Miniconda" + ci/incremental/install_miniconda.sh + export PATH=$HOME/miniconda3/bin:$PATH + echo "Setting up Conda environment" + ci/incremental/setup_conda_environment.sh + displayName: 'Before Install' + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + ci/incremental/build.sh + displayName: 'Build' + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + ci/script_single.sh + ci/script_multi.sh + echo "[Test done]" + displayName: 'Test' + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd diff --git a/ci/azure/windows-py27.yml b/ci/azure/windows-py27.yml new file mode 100644 index 00000000000000..e60844896b71cf --- /dev/null +++ b/ci/azure/windows-py27.yml @@ -0,0 +1,41 @@ +parameters: + name: '' + vmImage: '' + +jobs: +- job: ${{ parameters.name }} + pool: + vmImage: ${{ parameters.vmImage }} + strategy: + maxParallel: 11 + matrix: + py36_np14: + ENV_FILE: ci/azure-windows-27.yml + CONDA_PY: "27" + CONDA_ENV: pandas + + steps: + - task: CondaEnvironment@1 + inputs: + updateConda: no + packageSpecs: '' + + # Need to install VC 9.0 only for Python 2.7 + # Once we understand how to do tasks conditional on build matrix variables + # we could merge this into azure-windows.yml + - powershell: | + $wc = New-Object net.webclient + $wc.Downloadfile("https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi", "VCForPython27.msi") + Start-Process "VCForPython27.msi" /qn -Wait + displayName: 'Install VC 9.0' + + - script: | + ci\\incremental\\setup_conda_environment.cmd + displayName: 'Before Install' + - script: | + ci\\incremental\\build.cmd + displayName: 'Build' + - script: | + call activate %CONDA_ENV% + pytest --skip-slow --skip-network pandas -n 2 -r sxX --strict %* + displayName: 'Test' diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml new file mode 100644 index 00000000000000..6090139fb4f3e3 --- /dev/null +++ b/ci/azure/windows.yml @@ -0,0 +1,32 @@ +parameters: + name: '' + vmImage: '' + +jobs: +- job: ${{ parameters.name }} + pool: + vmImage: ${{ parameters.vmImage }} + strategy: + maxParallel: 11 + matrix: + py36_np14: + ENV_FILE: ci/azure-windows-36.yml + CONDA_PY: "36" + CONDA_ENV: pandas + + steps: + - task: CondaEnvironment@1 + inputs: + updateConda: no + packageSpecs: '' + + - script: | + ci\\incremental\\setup_conda_environment.cmd + displayName: 'Before Install' + - script: | + ci\\incremental\\build.cmd + displayName: 'Build' + - script: | + call activate %CONDA_ENV% + pytest --skip-slow --skip-network pandas -n 2 -r sxX --strict %* + displayName: 'Test' diff --git a/ci/incremental/build.cmd b/ci/incremental/build.cmd new file mode 100644 index 00000000000000..d2fd06d7d9e508 --- /dev/null +++ b/ci/incremental/build.cmd @@ -0,0 +1,10 @@ +@rem https://github.com/numba/numba/blob/master/buildscripts/incremental/build.cmd +call activate %CONDA_ENV% + +@rem Build numba extensions without silencing compile errors +python setup.py build_ext -q --inplace + +@rem Install pandas locally +python -m pip install -e . + +if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/ci/incremental/build.sh b/ci/incremental/build.sh new file mode 100755 index 00000000000000..8f2301a3b7ef50 --- /dev/null +++ b/ci/incremental/build.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +source activate $CONDA_ENV + +# Make sure any error below is reported as such +set -v -e + +echo "[building extensions]" +python setup.py build_ext -q --inplace +python -m pip install -e . + +echo +echo "[show environment]" +conda list + +echo +echo "[done]" +exit 0 diff --git a/ci/incremental/install_miniconda.sh b/ci/incremental/install_miniconda.sh new file mode 100755 index 00000000000000..a47dfdb324b348 --- /dev/null +++ b/ci/incremental/install_miniconda.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -v -e + +# Install Miniconda +unamestr=`uname` +if [[ "$unamestr" == 'Linux' ]]; then + if [[ "$BITS32" == "yes" ]]; then + wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86.sh -O miniconda.sh + else + wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + fi +elif [[ "$unamestr" == 'Darwin' ]]; then + wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh +else + echo Error +fi +chmod +x miniconda.sh +./miniconda.sh -b diff --git a/ci/incremental/setup_conda_environment.cmd b/ci/incremental/setup_conda_environment.cmd new file mode 100644 index 00000000000000..b4446c49fabd3a --- /dev/null +++ b/ci/incremental/setup_conda_environment.cmd @@ -0,0 +1,21 @@ +@rem https://github.com/numba/numba/blob/master/buildscripts/incremental/setup_conda_environment.cmd +@rem The cmd /C hack circumvents a regression where conda installs a conda.bat +@rem script in non-root environments. +set CONDA_INSTALL=cmd /C conda install -q -y +set PIP_INSTALL=pip install -q + +@echo on + +@rem Deactivate any environment +call deactivate +@rem Display root environment (for debugging) +conda list +@rem Clean up any left-over from a previous build +conda remove --all -q -y -n %CONDA_ENV% +@rem Scipy, CFFI, jinja2 and IPython are optional dependencies, but exercised in the test suite +conda env create -n %CONDA_ENV% --file=ci\azure-windows-%CONDA_PY%.yaml + +call activate %CONDA_ENV% +conda list + +if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh new file mode 100755 index 00000000000000..c716a391386448 --- /dev/null +++ b/ci/incremental/setup_conda_environment.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +set -v -e + +CONDA_INSTALL="conda install -q -y" +PIP_INSTALL="pip install -q" + +# Deactivate any environment +source deactivate +# Display root environment (for debugging) +conda list +# Clean up any left-over from a previous build +# (note workaround for https://github.com/conda/conda/issues/2679: +# `conda env remove` issue) +conda remove --all -q -y -n $CONDA_ENV + +echo +echo "[create env]" +time conda env create -q -n "${CONDA_ENV}" --file="${ENV_FILE}" || exit 1 + +# Activate first +set +v +source activate $CONDA_ENV +set -v + +# remove any installed pandas package +# w/o removing anything else +echo +echo "[removing installed pandas]" +conda remove pandas -y --force +pip uninstall -y pandas + +echo +echo "[no installed pandas]" +conda list pandas + +# # Install the compiler toolchain +# if [[ $(uname) == Linux ]]; then +# if [[ "$CONDA_SUBDIR" == "linux-32" || "$BITS32" == "yes" ]] ; then +# $CONDA_INSTALL gcc_linux-32 gxx_linux-32 +# else +# $CONDA_INSTALL gcc_linux-64 gxx_linux-64 +# fi +# elif [[ $(uname) == Darwin ]]; then +# $CONDA_INSTALL clang_osx-64 clangxx_osx-64 +# # Install llvm-openmp and intel-openmp on OSX too +# $CONDA_INSTALL llvm-openmp intel-openmp +# fi