Build Linux releases on big managed runners (iree-org#10126)

This speed the linux builds up a bit, bringing the time for the longest job down from ~5 hours to ~20 minutes. Note that this is *only* the Linux jobs. The mac ones still take about 4 hours. This should still help when iterating on the release though and for faster failure indicators (it was indeed helpful when I was iterating here). I ran into issues when testing because I was using a package suffix in the workflow dispatch, which evidently had never actually been tested because it was totally broken. This gave me a lot of wonderful opportunity to bash my head against bash and I reworked a lot of the `build_linux_package.sh` script. In retrospect, I wish I'd just removed the `package_suffix` feature. Test run: https://github.com/iree-org/iree/actions/runs/2923210349 skip-ci
nod-ai · Aug 29, 2022 · c50bac3 · c50bac3
1 parent c338ae9
commit c50bac3
Show file tree

Hide file tree

Showing 4 changed files with 95 additions and 75 deletions.
diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.yml
@@ -27,39 +27,39 @@ on:
 
 jobs:
   build_core:
-    name: "${{ matrix.os }} :: Build ${{ matrix.build_package }} Package"
-    runs-on: ${{ matrix.os }}
+    name: "${{ matrix.build-family }} :: Build ${{ matrix.build-package }} Package"
+    runs-on: ${{ matrix.runs-on }}
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       fail-fast: false
       matrix:
         include:
           # Ubuntu packages.
-          - os: ubuntu-20.04
-            build_family: linux
-            build_package: main-dist-linux
+          - runs-on: [managed-releaser, os-family=Linux, runner-group=releaser]
+            build-family: linux
+            build-package: main-dist-linux
             experimental: false
-          - os: ubuntu-20.04
-            build_family: linux
-            build_package: py-compiler-pkg
+          - runs-on: [managed-releaser, os-family=Linux, runner-group=releaser]
+            build-family: linux
+            build-package: py-compiler-pkg
             experimental: false
-          - os: ubuntu-20.04
-            build_family: linux
-            build_package: py-runtime-pkg
+          - runs-on: [managed-releaser, os-family=Linux, runner-group=releaser]
+            build-family: linux
+            build-package: py-runtime-pkg
             experimental: false
-          - os: ubuntu-20.04
-            build_family: linux
-            build_package: py-tf-compiler-tools-pkg
+          - runs-on: [managed-releaser, os-family=Linux, runner-group=releaser]
+            build-family: linux
+            build-package: py-tf-compiler-tools-pkg
             experimental: false
 
           # Macos packages.
-          - os: macos-latest
-            build_family: macos
-            build_package: py-compiler-pkg
+          - runs-on: macos-11
+            build-family: macos
+            build-package: py-compiler-pkg
             experimental: true
-          - os: macos-latest
-            build_family: macos
-            build_package: py-runtime-pkg
+          - runs-on: macos-11
+            build-family: macos
+            build-package: py-runtime-pkg
             experimental: true
     env:
       MANYLINUX_X86_64_IMAGE: gcr.io/iree-oss/manylinux2014_x86_64-release@sha256:b09c10868f846308bad2eab253a77d0a3f097816c40342bc289d8e62509bc5f9
@@ -75,7 +75,7 @@ jobs:
       # OS specific setup
       ##########################################################################
       - name: Install MacOS Deps
-        if: "matrix.build_family == 'macos'"
+        if: "matrix.build-family == 'macos'"
         shell: bash
         run: |
           sudo ./main_checkout/build_tools/python_deploy/install_macos_deps.sh
@@ -109,7 +109,7 @@ jobs:
       # console scripts (like cmake, ninja) will not be on the path.
       ##########################################################################
       - name: Main distribution (Linux)
-        if: "matrix.build_package == 'main-dist-linux'"
+        if: "matrix.build-package == 'main-dist-linux'"
         shell: bash
         run: |
           docker run --rm -w=/work \
@@ -124,19 +124,23 @@ jobs:
       ##########################################################################
 
       - name: Build runtime wheels (Linux)
-        if: "matrix.build_package == 'py-runtime-pkg' && matrix.build_family == 'linux'"
+        if: "matrix.build-package == 'py-runtime-pkg' && matrix.build-family == 'linux'"
         shell: bash
+        env:
+          package_suffix: ${{ github.event.inputs.package_suffix }}
+          packages: "iree-runtime iree-runtime-instrumented"
+          output_dir: "${{ github.workspace }}/bindist"
         run: |
-          packages="iree-runtime iree-runtime-instrumented" \
-          output_dir="$PWD/bindist" \
           ./main_checkout/build_tools/python_deploy/build_linux_packages.sh
 
       - name: Build runtime wheels (MacOS)
-        if: "matrix.build_package == 'py-runtime-pkg' && matrix.build_family == 'macos'"
+        if: "matrix.build-package == 'py-runtime-pkg' && matrix.build-family == 'macos'"
         shell: bash
+        env:
+          package_suffix: ${{ github.event.inputs.package_suffix }}
+          packages: "iree-runtime iree-runtime-instrumented"
+          output_dir: "${{ github.workspace }}/bindist"
         run: |
-          packages="iree-runtime iree-runtime-instrumented" \
-          output_dir="$PWD/bindist" \
           ./main_checkout/build_tools/python_deploy/build_macos_packages.sh
 
       ##########################################################################
@@ -145,24 +149,28 @@ jobs:
       # One step per OS.
       ##########################################################################
       - name: Build compiler wheels (Linux)
-        if: "matrix.build_package == 'py-compiler-pkg' && matrix.build_family == 'linux'"
+        if: "matrix.build-package == 'py-compiler-pkg' && matrix.build-family == 'linux'"
         shell: bash
+        env:
+          package_suffix: ${{ github.event.inputs.package_suffix }}
+          packages: "iree-compiler"
+          output_dir: "${{ github.workspace }}/bindist"
         run: |
-          packages="iree-compiler" \
-          output_dir="$PWD/bindist" \
           ./main_checkout/build_tools/python_deploy/build_linux_packages.sh
 
       - name: Build compiler wheels (MacOS)
-        if: "matrix.build_package == 'py-compiler-pkg' && matrix.build_family == 'macos'"
+        if: "matrix.build-package == 'py-compiler-pkg' && matrix.build-family == 'macos'"
         shell: bash
+        env:
+          package_suffix: ${{ github.event.inputs.package_suffix }}
+          packages: "iree-compiler"
+          output_dir: "${{ github.workspace }}/bindist"
         run: |
           # libzstd on GitHub Action bots is not compatible with MacOS universal.
           # https://github.com/iree-org/iree/issues/9955
           sudo rm -rf /usr/local/lib/libzstd.*.dylib
           sudo rm -rf /usr/local/lib/cmake/zstd/*
 
-          packages="iree-compiler" \
-          output_dir="$PWD/bindist" \
           ./main_checkout/build_tools/python_deploy/build_macos_packages.sh
 
       ##########################################################################
@@ -173,7 +181,7 @@ jobs:
       ##########################################################################
 
       - name: Build TF Compiler Tools wheels
-        if: "matrix.build_package == 'py-tf-compiler-tools-pkg'"
+        if: "matrix.build-package == 'py-tf-compiler-tools-pkg'"
         shell: bash
         run: |
           docker run --rm -w=/work \
@@ -203,6 +211,7 @@ jobs:
   validate_and_publish:
     name: "Invoke workflow to validate and publish release"
     needs: build_core
+    if: github.event.inputs.release_id != ''
     runs-on: ubuntu-20.04
     steps:
       - name: "Invoke workflow :: Validate and Publish Release"

diff --git a/build_tools/github_actions/build_dist.py b/build_tools/github_actions/build_dist.py
@@ -84,7 +84,7 @@ def load_version_info():
 try:
   version_info = load_version_info()
 except FileNotFoundError:
-  print("version_info.json found. Using defaults")
+  print("version_info.json not found. Using defaults")
   version_info = {
       "package-version": "0.1dev1",
       "package-suffix": "-dev",

diff --git a/build_tools/python_deploy/build_linux_packages.sh b/build_tools/python_deploy/build_linux_packages.sh
@@ -37,112 +37,122 @@
 #
 # It can be run on a workstation but recommend using a git worktree dedicated
 # to packaging to avoid stomping on development artifacts.
-set -eu -o errtrace
+set -xeu -o errtrace
 
 this_dir="$(cd $(dirname $0) && pwd)"
 script_name="$(basename $0)"
-repo_root="$(cd $this_dir/../../ && pwd)"
-script_name="$(basename $0)"
+repo_root="$(cd "${this_dir}" && git rev-parse --show-toplevel)"
 manylinux_docker_image="${manylinux_docker_image:-gcr.io/iree-oss/manylinux2014_x86_64-release@sha256:b09c10868f846308bad2eab253a77d0a3f097816c40342bc289d8e62509bc5f9}"
 python_versions="${override_python_versions:-cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310}"
 output_dir="${output_dir:-${this_dir}/wheelhouse}"
 packages="${packages:-iree-runtime iree-runtime-instrumented iree-compiler}"
+package_suffix="${package_suffix:-}"
 
 function run_on_host() {
   echo "Running on host"
   echo "Launching docker image ${manylinux_docker_image}"
 
   # Canonicalize paths.
-  mkdir -p "$output_dir"
-  output_dir="$(cd $output_dir && pwd)"
+  mkdir -p "${output_dir}"
+  output_dir="$(cd "${output_dir}" && pwd)"
   echo "Outputting to ${output_dir}"
   mkdir -p "${output_dir}"
   docker run --rm \
-    -v "${repo_root}:/main_checkout/iree" \
-    -v "${output_dir}:/wheelhouse" \
+    -v "${repo_root}:${repo_root}" \
+    -v "${output_dir}:${output_dir}" \
     -e __MANYLINUX_BUILD_WHEELS_IN_DOCKER=1 \
     -e "override_python_versions=${python_versions}" \
     -e "packages=${packages}" \
-    ${manylinux_docker_image} \
-    -- bash /main_checkout/iree/build_tools/python_deploy/build_linux_packages.sh
+    -e "package_suffix=${package_suffix}" \
+    -e "output_dir=${output_dir}" \
+    "${manylinux_docker_image}" \
+    -- ${this_dir}/${script_name}
 
   echo "******************** BUILD COMPLETE ********************"
   echo "Generated binaries:"
-  ls -l $output_dir
+  ls -l "${output_dir}"
 }
 
 function run_in_docker() {
   echo "Running in docker"
   echo "Using python versions: ${python_versions}"
 
-  local orig_path="$PATH"
+  local orig_path="${PATH}"
 
   # Build phase.
-  for package in $packages; do
+  for package in ${packages}; do
     echo "******************** BUILDING PACKAGE ${package} ********************"
-    for python_version in $python_versions; do
-      python_dir="/opt/python/$python_version"
-      if ! [ -x "$python_dir/bin/python" ]; then
-        echo "ERROR: Could not find python: $python_dir (skipping)"
+    for python_version in ${python_versions}; do
+      python_dir="/opt/python/${python_version}"
+      if ! [ -x "${python_dir}/bin/python" ]; then
+        echo "ERROR: Could not find python: ${python_dir} (skipping)"
         continue
       fi
-      export PATH=$python_dir/bin:$orig_path
+      export PATH="${python_dir}/bin:${orig_path}"
       echo ":::: Python version $(python --version)"
-      case "$package" in
+      # replace dashes with underscores
+      package_suffix="${package_suffix//-/_}"
+      case "${package}" in
         iree-runtime)
-          clean_wheels iree_runtime $python_version
+          clean_wheels "iree_runtime${package_suffix}" "${python_version}"
           build_iree_runtime
-          run_audit_wheel iree_runtime $python_version
+          run_audit_wheel "iree_runtime${package_suffix}" "${python_version}"
           ;;
         iree-runtime-instrumented)
-          clean_wheels iree_runtime_instrumented $python_version
+          clean_wheels "iree_runtime_instrumented${package_suffix}" "${python_version}"
           build_iree_runtime_instrumented
-          run_audit_wheel iree_runtime_instrumented $python_version
+          run_audit_wheel "iree_runtime_instrumented${package_suffix}" "${python_version}"
           ;;
         iree-compiler)
-          clean_wheels iree_compiler $python_version
+          clean_wheels "iree_compiler${package_suffix}" "${python_version}"
           build_iree_compiler
-          run_audit_wheel iree_compiler $python_version
+          run_audit_wheel "iree_compiler${package_suffix}" "${python_version}"
           ;;
         *)
-          echo "Unrecognized package '$package'"
+          echo "Unrecognized package '${package}'"
           exit 1
           ;;
       esac
     done
   done
 }
 
+function build_wheel() {
+  python -m pip wheel --disable-pip-version-check -v -w "${output_dir}" "${repo_root}/$@"
+}
+
 function build_iree_runtime() {
   IREE_HAL_DRIVER_CUDA=ON \
-  python -m pip wheel -v -w /wheelhouse /main_checkout/iree/runtime/
+  build_wheel runtime/
 }
 
 function build_iree_runtime_instrumented() {
   IREE_HAL_DRIVER_CUDA=ON IREE_BUILD_TRACY=ON IREE_ENABLE_RUNTIME_TRACING=ON \
   IREE_RUNTIME_CUSTOM_PACKAGE_SUFFIX="-instrumented" \
-  python -m pip wheel -v -w /wheelhouse /main_checkout/iree/runtime/
+  build_wheel runtime/
 }
 
 function build_iree_compiler() {
   IREE_TARGET_BACKEND_CUDA=ON \
-  python -m pip wheel -v -w /wheelhouse /main_checkout/iree/compiler/
+  build_wheel compiler/
 }
 
 function run_audit_wheel() {
   local wheel_basename="$1"
   local python_version="$2"
-  generic_wheel="/wheelhouse/${wheel_basename}-*-${python_version}-linux_x86_64.whl"
-  echo ":::: Auditwheel $generic_wheel"
-  auditwheel repair -w /wheelhouse $generic_wheel
-  rm -v $generic_wheel
+  # Force wildcard expansion here
+  generic_wheel="$(echo "${output_dir}/${wheel_basename}-"*"-${python_version}-linux_x86_64.whl")"
+  ls "${generic_wheel}"
+  echo ":::: Auditwheel ${generic_wheel}"
+  auditwheel repair -w "${output_dir}" "${generic_wheel}"
+  rm -v "${generic_wheel}"
 }
 
 function clean_wheels() {
   local wheel_basename="$1"
   local python_version="$2"
-  echo ":::: Clean wheels $wheel_basename $python_version"
-  rm -f -v /wheelhouse/${wheel_basename}-*-${python_version}-*.whl
+  echo ":::: Clean wheels ${wheel_basename} ${python_version}"
+  rm -f -v "${output_dir}/${wheel_basename}-"*"-${python_version}-"*".whl"
 }
 
 # Trampoline to the docker container if running on the host.

diff --git a/runtime/setup.py b/runtime/setup.py
@@ -27,25 +27,26 @@
 #   IREE_BUILD_TRACY
 #   IREE_ENABLE_CPUINFO
 
-from gettext import install
 import json
-from multiprocessing.spawn import prepare
 import os
 import platform
 import re
 import shutil
 import subprocess
 import sys
 import sysconfig
-
 from distutils.command.build import build as _build
-from setuptools import find_namespace_packages, setup, Extension
+from gettext import install
+from multiprocessing.spawn import prepare
+
+from setuptools import Extension, find_namespace_packages, setup
 from setuptools.command.build_ext import build_ext as _build_ext
 from setuptools.command.build_py import build_py as _build_py
 
 
 def check_pip_version():
   from packaging import version
+
   # Pip versions < 22.0.3 default to out of tree builds, which is quite
   # incompatible with what we do (and has other issues). Pip >= 22.0.4
   # removed this option entirely and are only in-tree builds. Since the
@@ -360,7 +361,7 @@ def generate_version_py():
   custom_package_suffix = ""
 
 setup(
-    name=f"iree-runtime{PACKAGE_SUFFIX}{custom_package_suffix}",
+    name=f"iree-runtime{custom_package_suffix}{PACKAGE_SUFFIX}",
     version=f"{PACKAGE_VERSION}",
     author="IREE Authors",
     author_email="iree-discuss@googlegroups.com",