From c50bac36052cb751d7d459f075747bdd1b2dc027 Mon Sep 17 00:00:00 2001 From: Geoffrey Martin-Noble Date: Mon, 29 Aug 2022 09:49:10 -0700 Subject: [PATCH] Build Linux releases on big managed runners (#10126) This speed the linux builds up a bit, bringing the time for the longest job down from ~5 hours to ~20 minutes. Note that this is *only* the Linux jobs. The mac ones still take about 4 hours. This should still help when iterating on the release though and for faster failure indicators (it was indeed helpful when I was iterating here). I ran into issues when testing because I was using a package suffix in the workflow dispatch, which evidently had never actually been tested because it was totally broken. This gave me a lot of wonderful opportunity to bash my head against bash and I reworked a lot of the `build_linux_package.sh` script. In retrospect, I wish I'd just removed the `package_suffix` feature. Test run: https://github.com/iree-org/iree/actions/runs/2923210349 skip-ci --- .github/workflows/build_package.yml | 79 +++++++++++-------- build_tools/github_actions/build_dist.py | 2 +- .../python_deploy/build_linux_packages.sh | 78 ++++++++++-------- runtime/setup.py | 11 +-- 4 files changed, 95 insertions(+), 75 deletions(-) diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.yml index 609fae29877e..ba881a00c3e5 100644 --- a/.github/workflows/build_package.yml +++ b/.github/workflows/build_package.yml @@ -27,39 +27,39 @@ on: jobs: build_core: - name: "${{ matrix.os }} :: Build ${{ matrix.build_package }} Package" - runs-on: ${{ matrix.os }} + name: "${{ matrix.build-family }} :: Build ${{ matrix.build-package }} Package" + runs-on: ${{ matrix.runs-on }} continue-on-error: ${{ matrix.experimental }} strategy: fail-fast: false matrix: include: # Ubuntu packages. - - os: ubuntu-20.04 - build_family: linux - build_package: main-dist-linux + - runs-on: [managed-releaser, os-family=Linux, runner-group=releaser] + build-family: linux + build-package: main-dist-linux experimental: false - - os: ubuntu-20.04 - build_family: linux - build_package: py-compiler-pkg + - runs-on: [managed-releaser, os-family=Linux, runner-group=releaser] + build-family: linux + build-package: py-compiler-pkg experimental: false - - os: ubuntu-20.04 - build_family: linux - build_package: py-runtime-pkg + - runs-on: [managed-releaser, os-family=Linux, runner-group=releaser] + build-family: linux + build-package: py-runtime-pkg experimental: false - - os: ubuntu-20.04 - build_family: linux - build_package: py-tf-compiler-tools-pkg + - runs-on: [managed-releaser, os-family=Linux, runner-group=releaser] + build-family: linux + build-package: py-tf-compiler-tools-pkg experimental: false # Macos packages. - - os: macos-latest - build_family: macos - build_package: py-compiler-pkg + - runs-on: macos-11 + build-family: macos + build-package: py-compiler-pkg experimental: true - - os: macos-latest - build_family: macos - build_package: py-runtime-pkg + - runs-on: macos-11 + build-family: macos + build-package: py-runtime-pkg experimental: true env: MANYLINUX_X86_64_IMAGE: gcr.io/iree-oss/manylinux2014_x86_64-release@sha256:b09c10868f846308bad2eab253a77d0a3f097816c40342bc289d8e62509bc5f9 @@ -75,7 +75,7 @@ jobs: # OS specific setup ########################################################################## - name: Install MacOS Deps - if: "matrix.build_family == 'macos'" + if: "matrix.build-family == 'macos'" shell: bash run: | sudo ./main_checkout/build_tools/python_deploy/install_macos_deps.sh @@ -109,7 +109,7 @@ jobs: # console scripts (like cmake, ninja) will not be on the path. ########################################################################## - name: Main distribution (Linux) - if: "matrix.build_package == 'main-dist-linux'" + if: "matrix.build-package == 'main-dist-linux'" shell: bash run: | docker run --rm -w=/work \ @@ -124,19 +124,23 @@ jobs: ########################################################################## - name: Build runtime wheels (Linux) - if: "matrix.build_package == 'py-runtime-pkg' && matrix.build_family == 'linux'" + if: "matrix.build-package == 'py-runtime-pkg' && matrix.build-family == 'linux'" shell: bash + env: + package_suffix: ${{ github.event.inputs.package_suffix }} + packages: "iree-runtime iree-runtime-instrumented" + output_dir: "${{ github.workspace }}/bindist" run: | - packages="iree-runtime iree-runtime-instrumented" \ - output_dir="$PWD/bindist" \ ./main_checkout/build_tools/python_deploy/build_linux_packages.sh - name: Build runtime wheels (MacOS) - if: "matrix.build_package == 'py-runtime-pkg' && matrix.build_family == 'macos'" + if: "matrix.build-package == 'py-runtime-pkg' && matrix.build-family == 'macos'" shell: bash + env: + package_suffix: ${{ github.event.inputs.package_suffix }} + packages: "iree-runtime iree-runtime-instrumented" + output_dir: "${{ github.workspace }}/bindist" run: | - packages="iree-runtime iree-runtime-instrumented" \ - output_dir="$PWD/bindist" \ ./main_checkout/build_tools/python_deploy/build_macos_packages.sh ########################################################################## @@ -145,24 +149,28 @@ jobs: # One step per OS. ########################################################################## - name: Build compiler wheels (Linux) - if: "matrix.build_package == 'py-compiler-pkg' && matrix.build_family == 'linux'" + if: "matrix.build-package == 'py-compiler-pkg' && matrix.build-family == 'linux'" shell: bash + env: + package_suffix: ${{ github.event.inputs.package_suffix }} + packages: "iree-compiler" + output_dir: "${{ github.workspace }}/bindist" run: | - packages="iree-compiler" \ - output_dir="$PWD/bindist" \ ./main_checkout/build_tools/python_deploy/build_linux_packages.sh - name: Build compiler wheels (MacOS) - if: "matrix.build_package == 'py-compiler-pkg' && matrix.build_family == 'macos'" + if: "matrix.build-package == 'py-compiler-pkg' && matrix.build-family == 'macos'" shell: bash + env: + package_suffix: ${{ github.event.inputs.package_suffix }} + packages: "iree-compiler" + output_dir: "${{ github.workspace }}/bindist" run: | # libzstd on GitHub Action bots is not compatible with MacOS universal. # https://github.com/iree-org/iree/issues/9955 sudo rm -rf /usr/local/lib/libzstd.*.dylib sudo rm -rf /usr/local/lib/cmake/zstd/* - packages="iree-compiler" \ - output_dir="$PWD/bindist" \ ./main_checkout/build_tools/python_deploy/build_macos_packages.sh ########################################################################## @@ -173,7 +181,7 @@ jobs: ########################################################################## - name: Build TF Compiler Tools wheels - if: "matrix.build_package == 'py-tf-compiler-tools-pkg'" + if: "matrix.build-package == 'py-tf-compiler-tools-pkg'" shell: bash run: | docker run --rm -w=/work \ @@ -203,6 +211,7 @@ jobs: validate_and_publish: name: "Invoke workflow to validate and publish release" needs: build_core + if: github.event.inputs.release_id != '' runs-on: ubuntu-20.04 steps: - name: "Invoke workflow :: Validate and Publish Release" diff --git a/build_tools/github_actions/build_dist.py b/build_tools/github_actions/build_dist.py index 6b1c65d8185e..081da82cd692 100644 --- a/build_tools/github_actions/build_dist.py +++ b/build_tools/github_actions/build_dist.py @@ -84,7 +84,7 @@ def load_version_info(): try: version_info = load_version_info() except FileNotFoundError: - print("version_info.json found. Using defaults") + print("version_info.json not found. Using defaults") version_info = { "package-version": "0.1dev1", "package-suffix": "-dev", diff --git a/build_tools/python_deploy/build_linux_packages.sh b/build_tools/python_deploy/build_linux_packages.sh index 5c91d3f113fe..23bdfc474c22 100755 --- a/build_tools/python_deploy/build_linux_packages.sh +++ b/build_tools/python_deploy/build_linux_packages.sh @@ -37,75 +37,79 @@ # # It can be run on a workstation but recommend using a git worktree dedicated # to packaging to avoid stomping on development artifacts. -set -eu -o errtrace +set -xeu -o errtrace this_dir="$(cd $(dirname $0) && pwd)" script_name="$(basename $0)" -repo_root="$(cd $this_dir/../../ && pwd)" -script_name="$(basename $0)" +repo_root="$(cd "${this_dir}" && git rev-parse --show-toplevel)" manylinux_docker_image="${manylinux_docker_image:-gcr.io/iree-oss/manylinux2014_x86_64-release@sha256:b09c10868f846308bad2eab253a77d0a3f097816c40342bc289d8e62509bc5f9}" python_versions="${override_python_versions:-cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310}" output_dir="${output_dir:-${this_dir}/wheelhouse}" packages="${packages:-iree-runtime iree-runtime-instrumented iree-compiler}" +package_suffix="${package_suffix:-}" function run_on_host() { echo "Running on host" echo "Launching docker image ${manylinux_docker_image}" # Canonicalize paths. - mkdir -p "$output_dir" - output_dir="$(cd $output_dir && pwd)" + mkdir -p "${output_dir}" + output_dir="$(cd "${output_dir}" && pwd)" echo "Outputting to ${output_dir}" mkdir -p "${output_dir}" docker run --rm \ - -v "${repo_root}:/main_checkout/iree" \ - -v "${output_dir}:/wheelhouse" \ + -v "${repo_root}:${repo_root}" \ + -v "${output_dir}:${output_dir}" \ -e __MANYLINUX_BUILD_WHEELS_IN_DOCKER=1 \ -e "override_python_versions=${python_versions}" \ -e "packages=${packages}" \ - ${manylinux_docker_image} \ - -- bash /main_checkout/iree/build_tools/python_deploy/build_linux_packages.sh + -e "package_suffix=${package_suffix}" \ + -e "output_dir=${output_dir}" \ + "${manylinux_docker_image}" \ + -- ${this_dir}/${script_name} echo "******************** BUILD COMPLETE ********************" echo "Generated binaries:" - ls -l $output_dir + ls -l "${output_dir}" } function run_in_docker() { echo "Running in docker" echo "Using python versions: ${python_versions}" - local orig_path="$PATH" + local orig_path="${PATH}" # Build phase. - for package in $packages; do + for package in ${packages}; do echo "******************** BUILDING PACKAGE ${package} ********************" - for python_version in $python_versions; do - python_dir="/opt/python/$python_version" - if ! [ -x "$python_dir/bin/python" ]; then - echo "ERROR: Could not find python: $python_dir (skipping)" + for python_version in ${python_versions}; do + python_dir="/opt/python/${python_version}" + if ! [ -x "${python_dir}/bin/python" ]; then + echo "ERROR: Could not find python: ${python_dir} (skipping)" continue fi - export PATH=$python_dir/bin:$orig_path + export PATH="${python_dir}/bin:${orig_path}" echo ":::: Python version $(python --version)" - case "$package" in + # replace dashes with underscores + package_suffix="${package_suffix//-/_}" + case "${package}" in iree-runtime) - clean_wheels iree_runtime $python_version + clean_wheels "iree_runtime${package_suffix}" "${python_version}" build_iree_runtime - run_audit_wheel iree_runtime $python_version + run_audit_wheel "iree_runtime${package_suffix}" "${python_version}" ;; iree-runtime-instrumented) - clean_wheels iree_runtime_instrumented $python_version + clean_wheels "iree_runtime_instrumented${package_suffix}" "${python_version}" build_iree_runtime_instrumented - run_audit_wheel iree_runtime_instrumented $python_version + run_audit_wheel "iree_runtime_instrumented${package_suffix}" "${python_version}" ;; iree-compiler) - clean_wheels iree_compiler $python_version + clean_wheels "iree_compiler${package_suffix}" "${python_version}" build_iree_compiler - run_audit_wheel iree_compiler $python_version + run_audit_wheel "iree_compiler${package_suffix}" "${python_version}" ;; *) - echo "Unrecognized package '$package'" + echo "Unrecognized package '${package}'" exit 1 ;; esac @@ -113,36 +117,42 @@ function run_in_docker() { done } +function build_wheel() { + python -m pip wheel --disable-pip-version-check -v -w "${output_dir}" "${repo_root}/$@" +} + function build_iree_runtime() { IREE_HAL_DRIVER_CUDA=ON \ - python -m pip wheel -v -w /wheelhouse /main_checkout/iree/runtime/ + build_wheel runtime/ } function build_iree_runtime_instrumented() { IREE_HAL_DRIVER_CUDA=ON IREE_BUILD_TRACY=ON IREE_ENABLE_RUNTIME_TRACING=ON \ IREE_RUNTIME_CUSTOM_PACKAGE_SUFFIX="-instrumented" \ - python -m pip wheel -v -w /wheelhouse /main_checkout/iree/runtime/ + build_wheel runtime/ } function build_iree_compiler() { IREE_TARGET_BACKEND_CUDA=ON \ - python -m pip wheel -v -w /wheelhouse /main_checkout/iree/compiler/ + build_wheel compiler/ } function run_audit_wheel() { local wheel_basename="$1" local python_version="$2" - generic_wheel="/wheelhouse/${wheel_basename}-*-${python_version}-linux_x86_64.whl" - echo ":::: Auditwheel $generic_wheel" - auditwheel repair -w /wheelhouse $generic_wheel - rm -v $generic_wheel + # Force wildcard expansion here + generic_wheel="$(echo "${output_dir}/${wheel_basename}-"*"-${python_version}-linux_x86_64.whl")" + ls "${generic_wheel}" + echo ":::: Auditwheel ${generic_wheel}" + auditwheel repair -w "${output_dir}" "${generic_wheel}" + rm -v "${generic_wheel}" } function clean_wheels() { local wheel_basename="$1" local python_version="$2" - echo ":::: Clean wheels $wheel_basename $python_version" - rm -f -v /wheelhouse/${wheel_basename}-*-${python_version}-*.whl + echo ":::: Clean wheels ${wheel_basename} ${python_version}" + rm -f -v "${output_dir}/${wheel_basename}-"*"-${python_version}-"*".whl" } # Trampoline to the docker container if running on the host. diff --git a/runtime/setup.py b/runtime/setup.py index 34116576b180..7d1879fd3ad2 100644 --- a/runtime/setup.py +++ b/runtime/setup.py @@ -27,9 +27,7 @@ # IREE_BUILD_TRACY # IREE_ENABLE_CPUINFO -from gettext import install import json -from multiprocessing.spawn import prepare import os import platform import re @@ -37,15 +35,18 @@ import subprocess import sys import sysconfig - from distutils.command.build import build as _build -from setuptools import find_namespace_packages, setup, Extension +from gettext import install +from multiprocessing.spawn import prepare + +from setuptools import Extension, find_namespace_packages, setup from setuptools.command.build_ext import build_ext as _build_ext from setuptools.command.build_py import build_py as _build_py def check_pip_version(): from packaging import version + # Pip versions < 22.0.3 default to out of tree builds, which is quite # incompatible with what we do (and has other issues). Pip >= 22.0.4 # removed this option entirely and are only in-tree builds. Since the @@ -360,7 +361,7 @@ def generate_version_py(): custom_package_suffix = "" setup( - name=f"iree-runtime{PACKAGE_SUFFIX}{custom_package_suffix}", + name=f"iree-runtime{custom_package_suffix}{PACKAGE_SUFFIX}", version=f"{PACKAGE_VERSION}", author="IREE Authors", author_email="iree-discuss@googlegroups.com",