diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 4ec54f6d7f2e5..a110c47b37c5a 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -656,7 +656,7 @@ test_linux_wheels() { } test_macos_wheels() { - local py_arches="3.5m 3.6m 3.7m 3.8" + local py_arches="3.5m 3.6m 3.7m 3.8 3.9" for py_arch in ${py_arches}; do local env=_verify_wheel-${py_arch} diff --git a/dev/tasks/python-wheels/github.osx.yml b/dev/tasks/python-wheels/github.osx.yml new file mode 100644 index 0000000000000..5e9214f0a9c34 --- /dev/null +++ b/dev/tasks/python-wheels/github.osx.yml @@ -0,0 +1,142 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE: must set "Crossbow" as name to have the badge links working in the +# github comment reports! +name: Crossbow + +on: + push: + branches: + - "*-github-*" + +jobs: + build: + name: Build wheel for OS X + runs-on: macos-latest + steps: + - name: Checkout Arrow + run: | + set -ex + + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + + - name: Build wheel + run: | + set -ex + + git clone https://github.com/matthew-brett/multibuild + git -C multibuild checkout 8882150df6529658700b66bec124dfb77eefca26 + + # source utilities required for wheel builds + export CONFIG_PATH=`pwd`/arrow/dev/tasks/python-wheels/osx-build.sh + source multibuild/common_utils.sh + source multibuild/travis_osx_steps.sh + + # provided by travis_osx_steps + before_install + + brew install \ + automake \ + bison \ + boost \ + ccache \ + cmake \ + flatbuffers \ + flex \ + git \ + openblas \ + openssl@1.1 \ + protobuf \ + python@3.8 \ + thrift \ + wget + + # overridden by osx-build.sh + build_wheel arrow + + # test the built wheels, move Homebrew directory to ensure things + # are properly statically-linked + CELLAR=$(brew --cellar) + sudo mv $CELLAR{,.bak} + + # make a new Cellar, allowing numpy to dynamically link to openblas + sudo mkdir $CELLAR + sudo ln -s ../Cellar.bak/openblas $CELLAR/openblas + # and brew's openblas will need gcc + sudo ln -s ../Cellar.bak/gcc $CELLAR/gcc + + # install the built wheel and test dependencies (osx-build.sh) + install_wheel arrow + + # run unit tests before removing the system libraries (osx-build.sh) + ulimit -S -n 4096 && run_unit_tests arrow + + # run the import tests (osx-build.sh) + run_import_tests + + # restore the original Cellar + sudo rm $CELLAR/{gcc,openblas} + sudo rmdir $CELLAR + sudo mv $CELLAR{.bak,} + env: + # pyarrow config + ARROW_S3: {{ arrow_s3 }} + MACOSX_DEPLOYMENT_TARGET: {{ macos_deployment_target }} + PYARROW_BUILD_VERBOSE: 1 + PYARROW_VERSION: {{ arrow.no_rc_version }} + + # multibuild config + MB_PYTHON_VERSION: {{ python_version }} + PLAT: x86_64 + + - name: Setup Crossbow + run: | + pip3 install \ + click \ + github3.py \ + jinja2 \ + jira \ + pygit2 \ + ruamel.yaml \ + setuptools_scm \ + toolz + + - name: Upload artifacts + run: | + python3 arrow/dev/tasks/crossbow.py \ + --queue-path $(pwd) \ + --queue-remote {{ queue_remote_url }} \ + upload-artifacts \ + --sha {{ task.branch }} \ + --tag {{ task.tag }} \ + --pattern "arrow/python/dist/*.whl" + env: + CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN }}' }} + + {% if arrow.branch == 'master' %} + - name: Upload package to Gemfury + run: | + # upload to gemfury pypi repository, there should be a single wheel + path=$(ls arrow/python/dist/*.whl) + curl -F "package=@${path}" https://${CROSSBOW_GEMFURY_TOKEN}@push.fury.io/${CROSSBOW_GEMFURY_ORG}/ + env: + CROSSBOW_GEMFURY_TOKEN: {{ '${{ secrets.CROSSBOW_GEMFURY_TOKEN }}' }} + CROSSBOW_GEMFURY_ORG: {{ '${{ secrets.CROSSBOW_GEMFURY_ORG }}' }} + {% endif %} diff --git a/dev/tasks/python-wheels/osx-build.sh b/dev/tasks/python-wheels/osx-build.sh index e029932ecf6d8..e6bd6159de90d 100755 --- a/dev/tasks/python-wheels/osx-build.sh +++ b/dev/tasks/python-wheels/osx-build.sh @@ -50,7 +50,10 @@ function build_wheel { export ARROW_HOME=`pwd`/arrow-dist export PARQUET_HOME=`pwd`/arrow-dist - pip install $(pip_opts) -r python/requirements-wheel-build.txt + # If NumPy builds from the source make sure it is built against OpenBLAS + # See: https://github.com/numpy/numpy/issues/15947#issuecomment-686159427 + OPENBLAS=$(brew --prefix openblas) pip install \ + $(pip_opts) -r python/requirements-wheel-build.txt git submodule update --init export ARROW_TEST_DATA=`pwd`/testing/data diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml deleted file mode 100644 index 99c667a64507e..0000000000000 --- a/dev/tasks/python-wheels/travis.osx.yml +++ /dev/null @@ -1,99 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -os: osx -osx_image: xcode11.3 -language: generic - -addons: - homebrew: - packages: - - automake - - bison - - boost - - ccache - - cmake - - flatbuffers - - flex - - git - - openssl@1.1 - - protobuf - - python@3.8 - - thrift - - wget - update: false - -# don't build twice -if: tag IS blank - -env: - global: - - PLAT=x86_64 - - TRAVIS_TAG={{ task.tag }} - - PYARROW_VERSION={{ arrow.no_rc_version }} - - PYARROW_BUILD_VERBOSE=1 - - MB_PYTHON_VERSION={{ python_version }} - - MACOSX_DEPLOYMENT_TARGET={{ macos_deployment_target }} - - ARROW_S3={{ arrow_s3 }} - -before_install: - - git clone https://github.com/matthew-brett/multibuild - - git -C multibuild checkout 8882150df6529658700b66bec124dfb77eefca26 - - - git clone --no-checkout {{ arrow.remote }} arrow - - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - - git -C arrow checkout FETCH_HEAD - - git -C arrow submodule update --init - - # source utilities required for wheel builds - - export CONFIG_PATH=`pwd`/arrow/dev/tasks/python-wheels/osx-build.sh - - source multibuild/common_utils.sh - - source multibuild/travis_osx_steps.sh - - before_install - -install: - # the following functions are defined in osx-build.sh - - build_wheel arrow - - # test the built wheels, move Homebrew directory to ensure - # things are properly statically-linked - - sudo mv $(brew --cellar){,.bak} - # install the built wheel and test dependencies - - install_wheel arrow - # run unit tests before removing the system libraries - - ulimit -S -n 4096 && run_unit_tests arrow - # run the import tests - - run_import_tests - - - sudo mv $(brew --cellar){.bak,} - # before_install activates a virtualenv but we need the system python3 - - deactivate - - # crossbow dependencies for deployment - - /usr/local/opt/python@3.8/libexec/bin/pip install click ruamel.yaml setuptools_scm github3.py toolz requests[security] - - /usr/local/opt/python@3.8/libexec/bin/python arrow/dev/tasks/crossbow.py - --queue-path $(pwd) - --queue-remote {{ queue_remote_url }} - upload-artifacts - --sha {{ task.branch }} - --tag {{ task.tag }} - --pattern "arrow/python/dist/*.whl" - - {% if arrow.branch == 'master' %} - # upload to gemfury pypi repository, there should be a single wheel - - path=$(ls arrow/python/dist/*.whl) - - curl -F "package=@${path}" https://${CROSSBOW_GEMFURY_TOKEN}@push.fury.io/${CROSSBOW_GEMFURY_ORG}/ - {% endif %} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 9325948520e50..569e59f80ddf4 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -290,6 +290,15 @@ tasks: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + conda-osx-clang-py39: + ci: azure + template: conda-recipes/azure.osx.yml + params: + config: osx_python3.9.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + ############################## Conda Windows ################################ conda-win-vs2017-py36-r36: @@ -494,8 +503,8 @@ tasks: ############################## Wheel OSX #################################### wheel-osx-mavericks-cp36m: - ci: travis - template: python-wheels/travis.osx.yml + ci: github + template: python-wheels/github.osx.yml params: python_version: 3.6 macos_deployment_target: 10.9 @@ -504,8 +513,8 @@ tasks: - pyarrow-{no_rc_version}-cp36-cp36m-macosx_10_9_x86_64.whl wheel-osx-mavericks-cp37m: - ci: travis - template: python-wheels/travis.osx.yml + ci: github + template: python-wheels/github.osx.yml params: python_version: 3.7 macos_deployment_target: 10.9 @@ -514,8 +523,8 @@ tasks: - pyarrow-{no_rc_version}-cp37-cp37m-macosx_10_9_x86_64.whl wheel-osx-mavericks-cp38: - ci: travis - template: python-wheels/travis.osx.yml + ci: github + template: python-wheels/github.osx.yml params: python_version: 3.8 macos_deployment_target: 10.9 @@ -523,11 +532,21 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp38-cp38-macosx_10_9_x86_64.whl + wheel-osx-mavericks-cp39: + ci: github + template: python-wheels/github.osx.yml + params: + python_version: 3.9 + macos_deployment_target: 10.9 + arrow_s3: "OFF" + artifacts: + - pyarrow-{no_rc_version}-cp39-cp39-macosx_10_9_x86_64.whl + # enable S3 support from macOS 10.13 so we don't need to bundle curl, crypt and ssl wheel-osx-high-sierra-cp36m: - ci: travis - template: python-wheels/travis.osx.yml + ci: github + template: python-wheels/github.osx.yml params: python_version: 3.6 macos_deployment_target: 10.13 @@ -536,8 +555,8 @@ tasks: - pyarrow-{no_rc_version}-cp36-cp36m-macosx_10_13_x86_64.whl wheel-osx-high-sierra-cp37m: - ci: travis - template: python-wheels/travis.osx.yml + ci: github + template: python-wheels/github.osx.yml params: python_version: 3.7 macos_deployment_target: 10.13 @@ -546,8 +565,8 @@ tasks: - pyarrow-{no_rc_version}-cp37-cp37m-macosx_10_13_x86_64.whl wheel-osx-high-sierra-cp38: - ci: travis - template: python-wheels/travis.osx.yml + ci: github + template: python-wheels/github.osx.yml params: python_version: 3.8 macos_deployment_target: 10.13 @@ -555,6 +574,16 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp38-cp38-macosx_10_13_x86_64.whl + wheel-osx-high-sierra-cp39: + ci: github + template: python-wheels/github.osx.yml + params: + python_version: 3.9 + macos_deployment_target: 10.13 + arrow_s3: "ON" + artifacts: + - pyarrow-{no_rc_version}-cp39-cp39-macosx_10_13_x86_64.whl + ############################## Wheel Windows ################################ wheel-win-cp36m: