From 6085f5f4c6d9f4412c6bef9b9f6bd76814d2945e Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 26 Oct 2018 07:07:43 -0500 Subject: [PATCH] CI: Migrate some CircleCI jobs to Azure (#22992) --- .circleci/config.yml | 111 +----------------- .travis.yml | 8 +- azure-pipelines.yml | 8 +- ...le-27-compat.yaml => azure-27-compat.yaml} | 0 ...le_slow.yaml => azure-36-locale_slow.yaml} | 0 ci/azure-37-locale.yaml | 35 ++++++ ci/azure/linux.yml | 56 +++++++++ ci/azure/macos.yml | 2 +- ci/circle-35-ascii.yaml | 15 --- ci/{ => circle}/install_circle.sh | 0 ci/{ => circle}/run_circle.sh | 0 ci/{ => circle}/show_circle.sh | 0 ci/incremental/setup_conda_environment.sh | 8 +- ci/script_multi.sh | 12 +- ci/script_single.sh | 12 +- doc/source/contributing.rst | 2 +- 16 files changed, 122 insertions(+), 147 deletions(-) rename ci/{circle-27-compat.yaml => azure-27-compat.yaml} (100%) rename ci/{circle-36-locale_slow.yaml => azure-36-locale_slow.yaml} (100%) create mode 100644 ci/azure-37-locale.yaml create mode 100644 ci/azure/linux.yml delete mode 100644 ci/circle-35-ascii.yaml rename ci/{ => circle}/install_circle.sh (100%) rename ci/{ => circle}/run_circle.sh (100%) rename ci/{ => circle}/show_circle.sh (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index e947f30d285cdb..5b100368189014 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,39 +1,6 @@ version: 2 jobs: - # -------------------------------------------------------------------------- - # 0. py27_compat - # -------------------------------------------------------------------------- - py27_compat: - docker: - - image: continuumio/miniconda:latest - # databases configuration - - image: circleci/postgres:9.6.5-alpine-ram - environment: - POSTGRES_USER: postgres - POSTGRES_DB: pandas_nosetest - - image: circleci/mysql:8-ram - environment: - MYSQL_USER: "root" - MYSQL_HOST: "localhost" - MYSQL_ALLOW_EMPTY_PASSWORD: "true" - MYSQL_DATABASE: "pandas_nosetest" - environment: - JOB: "2.7_COMPAT" - ENV_FILE: "ci/circle-27-compat.yaml" - LOCALE_OVERRIDE: "it_IT.UTF-8" - MINICONDA_DIR: /home/ubuntu/miniconda3 - steps: - - checkout - - run: - name: build - command: | - ./ci/install_circle.sh - ./ci/show_circle.sh - - run: - name: test - command: ./ci/run_circle.sh --skip-slow --skip-network - # -------------------------------------------------------------------------- # 1. py36_locale # -------------------------------------------------------------------------- @@ -62,86 +29,14 @@ jobs: - run: name: build command: | - ./ci/install_circle.sh - ./ci/show_circle.sh + ./ci/circle/install_circle.sh + ./ci/circle/show_circle.sh - run: name: test - command: ./ci/run_circle.sh --skip-slow --skip-network - - # -------------------------------------------------------------------------- - # 2. py36_locale_slow - # -------------------------------------------------------------------------- - py36_locale_slow: - docker: - - image: continuumio/miniconda:latest - # databases configuration - - image: circleci/postgres:9.6.5-alpine-ram - environment: - POSTGRES_USER: postgres - POSTGRES_DB: pandas_nosetest - - image: circleci/mysql:8-ram - environment: - MYSQL_USER: "root" - MYSQL_HOST: "localhost" - MYSQL_ALLOW_EMPTY_PASSWORD: "true" - MYSQL_DATABASE: "pandas_nosetest" - - environment: - JOB: "3.6_LOCALE_SLOW" - ENV_FILE: "ci/circle-36-locale_slow.yaml" - LOCALE_OVERRIDE: "zh_CN.UTF-8" - MINICONDA_DIR: /home/ubuntu/miniconda3 - steps: - - checkout - - run: - name: build - command: | - ./ci/install_circle.sh - ./ci/show_circle.sh - - run: - name: test - command: ./ci/run_circle.sh --only-slow --skip-network - - # -------------------------------------------------------------------------- - # 3. py35_ascii - # -------------------------------------------------------------------------- - py35_ascii: - docker: - - image: continuumio/miniconda:latest - # databases configuration - - image: circleci/postgres:9.6.5-alpine-ram - environment: - POSTGRES_USER: postgres - POSTGRES_DB: pandas_nosetest - - image: circleci/mysql:8-ram - environment: - MYSQL_USER: "root" - MYSQL_HOST: "localhost" - MYSQL_ALLOW_EMPTY_PASSWORD: "true" - MYSQL_DATABASE: "pandas_nosetest" - - environment: - JOB: "3.5_ASCII" - ENV_FILE: "ci/circle-35-ascii.yaml" - LOCALE_OVERRIDE: "C" - MINICONDA_DIR: /home/ubuntu/miniconda3 - steps: - - checkout - - run: - name: build - command: | - ./ci/install_circle.sh - ./ci/show_circle.sh - - run: - name: test - command: ./ci/run_circle.sh --skip-slow --skip-network - + command: ./ci/circle/run_circle.sh --skip-slow --skip-network workflows: version: 2 build_and_test: jobs: - - py27_compat - py36_locale - - py36_locale_slow - - py35_ascii diff --git a/.travis.yml b/.travis.yml index e8f7f3465bfd50..8ac4d827b0820b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -116,10 +116,10 @@ after_success: after_script: - echo "after_script start" - source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - - if [ -e /tmp/single.xml ]; then - ci/print_skipped.py /tmp/single.xml; + - if [ -e test-data-single.xml ]; then + ci/print_skipped.py test-data-single.xml; fi - - if [ -e /tmp/multiple.xml ]; then - ci/print_skipped.py /tmp/multiple.xml; + - if [ -e test-data-multiple.xml ]; then + ci/print_skipped.py test-data-multiple.xml; fi - echo "after_script done" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5d473bfc5a38c7..373c22fdf8e629 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,10 +7,10 @@ jobs: parameters: name: macOS vmImage: xcode9-macos10.13 -# - template: ci/azure/linux.yml -# parameters: -# name: Linux -# vmImage: ubuntu-16.04 +- template: ci/azure/linux.yml + parameters: + name: Linux + vmImage: ubuntu-16.04 # Windows Python 2.7 needs VC 9.0 installed, and not sure # how to make that a conditional task, so for now these are diff --git a/ci/circle-27-compat.yaml b/ci/azure-27-compat.yaml similarity index 100% rename from ci/circle-27-compat.yaml rename to ci/azure-27-compat.yaml diff --git a/ci/circle-36-locale_slow.yaml b/ci/azure-36-locale_slow.yaml similarity index 100% rename from ci/circle-36-locale_slow.yaml rename to ci/azure-36-locale_slow.yaml diff --git a/ci/azure-37-locale.yaml b/ci/azure-37-locale.yaml new file mode 100644 index 00000000000000..ef97b85406709e --- /dev/null +++ b/ci/azure-37-locale.yaml @@ -0,0 +1,35 @@ +name: pandas +channels: + - defaults + - conda-forge +dependencies: + - beautifulsoup4 + - cython>=0.28.2 + - html5lib + - ipython + - jinja2 + - lxml + - matplotlib + - nomkl + - numexpr + - numpy + - openpyxl=2.5.5 + - psycopg2 + - pymysql + - pytables + - python-dateutil + - python=3.6* + - pytz + - s3fs + - scipy + - sqlalchemy + - xarray + - xlrd + - xlsxwriter + - xlwt + # universal + - pytest + - pytest-xdist + - moto + - pip: + - hypothesis>=3.58.0 diff --git a/ci/azure/linux.yml b/ci/azure/linux.yml new file mode 100644 index 00000000000000..f34cba69a6195e --- /dev/null +++ b/ci/azure/linux.yml @@ -0,0 +1,56 @@ +parameters: + name: '' + vmImage: '' + +jobs: +- job: ${{ parameters.name }} + pool: + vmImage: ${{ parameters.vmImage }} + strategy: + maxParallel: 11 + matrix: + py27_np_19: + ENV_FILE: ci/azure-27-compat.yaml + CONDA_PY: "27" + CONDA_ENV: pandas + TEST_ARGS: "--skip-slow --skip-network" + + py36_locale: + ENV_FILE: ci/azure-37-locale.yaml + CONDA_PY: "37" + CONDA_ENV: pandas + TEST_ARGS: "--skip-slow --skip-network" + LOCALE_OVERRIDE: "zh_CN.UTF-8" + + py36_locale_slow: + ENV_FILE: ci/azure-36-locale_slow.yaml + CONDA_PY: "36" + CONDA_ENV: pandas + TEST_ARGS: "--only-slow --skip-network" + + steps: + - script: | + if [ "$(uname)" == "Linux" ]; then sudo apt-get install -y libc6-dev-i386; fi + echo "Installing Miniconda"{ + ci/incremental/install_miniconda.sh + export PATH=$HOME/miniconda3/bin:$PATH + echo "Setting up Conda environment" + ci/incremental/setup_conda_environment.sh + displayName: 'Before Install' + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + ci/incremental/build.sh + displayName: 'Build' + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + ci/script_single.sh + ci/script_multi.sh + echo "[Test done]" + displayName: 'Test' + - script: | + export PATH=$HOME/miniconda3/bin:$PATH + source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd + - task: PublishTestResults@2 + inputs: + testResultsFiles: 'test-data-*.xml' + testRunTitle: 'Linux' \ No newline at end of file diff --git a/ci/azure/macos.yml b/ci/azure/macos.yml index fb10d89731f26e..53ce51c76683c4 100644 --- a/ci/azure/macos.yml +++ b/ci/azure/macos.yml @@ -39,5 +39,5 @@ jobs: source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - task: PublishTestResults@2 inputs: - testResultsFiles: '/tmp/*.xml' + testResultsFiles: 'test-data-*.xml' testRunTitle: 'MacOS-35' diff --git a/ci/circle-35-ascii.yaml b/ci/circle-35-ascii.yaml deleted file mode 100644 index 281ed59e2deff0..00000000000000 --- a/ci/circle-35-ascii.yaml +++ /dev/null @@ -1,15 +0,0 @@ -name: pandas -channels: - - defaults -dependencies: - - cython>=0.28.2 - - nomkl - - numpy - - python-dateutil - - python=3.5* - - pytz - # universal - - pytest - - pytest-xdist - - pip: - - hypothesis>=3.58.0 diff --git a/ci/install_circle.sh b/ci/circle/install_circle.sh similarity index 100% rename from ci/install_circle.sh rename to ci/circle/install_circle.sh diff --git a/ci/run_circle.sh b/ci/circle/run_circle.sh similarity index 100% rename from ci/run_circle.sh rename to ci/circle/run_circle.sh diff --git a/ci/show_circle.sh b/ci/circle/show_circle.sh similarity index 100% rename from ci/show_circle.sh rename to ci/circle/show_circle.sh diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh index c716a391386448..f3ac99d5e7c5a6 100755 --- a/ci/incremental/setup_conda_environment.sh +++ b/ci/incremental/setup_conda_environment.sh @@ -27,13 +27,17 @@ set -v # w/o removing anything else echo echo "[removing installed pandas]" -conda remove pandas -y --force -pip uninstall -y pandas +conda remove pandas -y --force || true +pip uninstall -y pandas || true echo echo "[no installed pandas]" conda list pandas +if [ -n "$LOCALE_OVERRIDE" ]; then + sudo locale-gen "$LOCALE_OVERRIDE" +fi + # # Install the compiler toolchain # if [[ $(uname) == Linux ]]; then # if [[ "$CONDA_SUBDIR" == "linux-32" || "$BITS32" == "yes" ]] ; then diff --git a/ci/script_multi.sh b/ci/script_multi.sh index dcc5a14d7b3b4d..e076558e8fff3b 100755 --- a/ci/script_multi.sh +++ b/ci/script_multi.sh @@ -27,17 +27,17 @@ if [ "$DOC" ]; then echo "We are not running pytest as this is a doc-build" elif [ "$COVERAGE" ]; then - echo pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas - pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas + echo pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas + pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas elif [ "$SLOW" ]; then TEST_ARGS="--only-slow --skip-network" - echo pytest -m "not single and slow" -v --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas - pytest -m "not single and slow" -v --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas + echo pytest -m "not single and slow" -v --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas + pytest -m "not single and slow" -v --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas else - echo pytest -n 2 -m "not single" --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas - pytest -n 2 -m "not single" --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas # TODO: doctest + echo pytest -n 2 -m "not single" --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas + pytest -n 2 -m "not single" --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas # TODO: doctest fi diff --git a/ci/script_single.sh b/ci/script_single.sh index 09e7446a2d8763..42d326e0965ee3 100755 --- a/ci/script_single.sh +++ b/ci/script_single.sh @@ -5,8 +5,9 @@ echo "[script_single]" source activate pandas if [ -n "$LOCALE_OVERRIDE" ]; then + echo "Setting LC_ALL and LANG to $LOCALE_OVERRIDE" export LC_ALL="$LOCALE_OVERRIDE"; - echo "Setting LC_ALL to $LOCALE_OVERRIDE" + export LANG="$LOCALE_OVERRIDE"; pycmd='import pandas; print("pandas detected console encoding: %s" % pandas.get_option("display.encoding"))' python -c "$pycmd" @@ -25,14 +26,13 @@ if [ "$DOC" ]; then echo "We are not running pytest as this is a doc-build" elif [ "$COVERAGE" ]; then - echo pytest -s -m "single" --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas - pytest -s -m "single" --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas - + echo pytest -s -m "single" --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=test-data-single.xml $TEST_ARGS pandas + pytest -s -m "single" --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=test-data-single.xml $TEST_ARGS pandas echo pytest -s --strict scripts pytest -s --strict scripts else - echo pytest -m "single" --junitxml=/tmp/single.xml --strict $TEST_ARGS pandas - pytest -m "single" --junitxml=/tmp/single.xml --strict $TEST_ARGS pandas # TODO: doctest + echo pytest -m "single" --junitxml=test-data-single.xml --strict $TEST_ARGS pandas + pytest -m "single" --junitxml=test-data-single.xml --strict $TEST_ARGS pandas fi diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst index 67b8d287d5d1ab..66d545a0de6e9e 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributing.rst @@ -684,7 +684,7 @@ Test-driven development/code writing ------------------------------------ *pandas* is serious about testing and strongly encourages contributors to embrace -`test-driven development (TDD) `_. +`test-driven development (TDD) `_. This development process "relies on the repetition of a very short development cycle: first the developer writes an (initially failing) automated test case that defines a desired improvement or new function, then produces the minimum amount of code to pass that test."