Add job for building and testing with Bazel (#9980)

Part of #9855
iree-org · Aug 2, 2022 · dea0c88 · dea0c88
1 parent 67ee19f
commit dea0c88
Show file tree

Hide file tree

Showing 4 changed files with 102 additions and 32 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -27,7 +27,11 @@ name: CI
 #
 # - We have to do a weird hack to get a pseudo-ternary operator. See
 # https://github.com/actions/runner/issues/409, hence patterns like:
-# runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
+# `github.event_name == 'pull_request' && 'presubmit' || 'postsubmit'`
+# to mean `'presubmit' if github.event_name == 'pull_request' else 'postsubmit'`
+# Note that this only works if the true branch value is truthy. If it is falsey
+# then we would always get the false branch condition (`p && false` is always
+# false).
 
 on:
   workflow_dispatch:
@@ -96,7 +100,7 @@ jobs:
 
   build_all:
     runs-on:
-      # Hacks, and order matters. See the comment at the top of the file.
+      # Pseudo-ternary hack and order matters. See comment at top of file.
       - self-hosted
       - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
       - cpu
@@ -145,6 +149,32 @@ jobs:
           gcloud alpha storage cp "${BUILD_DIR_ARCHIVE}" "${GCS_ARTIFACT}"
           echo "::set-output name=gcs-artifact::${GCS_ARTIFACT}"
 
+  build_test_all_bazel:
+    runs-on:
+      # Pseudo-ternary hack and order matters. See comment at top of file.
+      - self-hosted
+      - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
+      - cpu
+      - os-family=Linux
+    steps:
+      - name: "Checking out repository"
+        uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
+        with:
+          submodules: true
+      - name: "Building with Bazel"
+        env:
+          # Psuedo-ternary hack. Don't swap the condition! See comment at top of file.
+          IREE_BAZEL_WRITE_REMOTE_CACHE: ${{ github.event_name != 'pull_request' && 1 || 0 }}
+        # This doesn't really need everything in the frontends image, but we
+        # want the cache to be shared with the integrations build (no point
+        # building LLVM twice) and the cache key is the docker container it's
+        # run in (to ensure correct cache hits).
+        run: |
+          ./build_tools/github_actions/docker_run.sh \
+            gcr.io/iree-oss/frontends-swiftshader@sha256:41e516b8c1b432e3c02896c4bf4b7f06df6a67371aa167b88767b8d4d2018ea6 \
+            ./build_tools/bazel/build_core.sh
+
+
   host_tools_assertions:
     uses: ./.github/workflows/host_tools.yml
     with:
@@ -153,7 +183,7 @@ jobs:
 
   build_tf_integrations:
     runs-on:
-      # Hacks, and order matters. See the comment at the top of the file.
+      # Pseudo-ternary hack and order matters. See comment at top of file.
       - self-hosted
       - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
       - cpu
@@ -171,6 +201,8 @@ jobs:
         id: build
         env:
           IREE_TF_BINARIES_OUTPUT_DIR: iree-tf-binaries
+          # Psuedo-ternary hack. Don't swap the condition! See comment at top of file.
+          IREE_BAZEL_WRITE_REMOTE_CACHE: ${{ github.event_name != 'pull_request' && 1 || 0 }}
         run: |
           ./build_tools/github_actions/docker_run.sh \
             --env "IREE_TF_BINARIES_OUTPUT_DIR=$(realpath "${IREE_TF_BINARIES_OUTPUT_DIR}")" \
@@ -197,7 +229,7 @@ jobs:
   test_all:
     needs: build_all
     runs-on:
-      # Hacks, and order matters. See the comment at the top of the file.
+      # Pseudo-ternary hack and order matters. See comment at top of file.
       - self-hosted
       - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
       - cpu
@@ -227,7 +259,7 @@ jobs:
     if: github.event_name != 'pull_request'
     needs: build_all
     runs-on:
-      # Hacks, and order matters. See the comment at the top of the file.
+      # Pseudo-ternary hack and order matters. See comment at top of file.
       - self-hosted
       - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
       - gpu
@@ -266,7 +298,7 @@ jobs:
   test_tf_integrations:
     needs: [build_tf_integrations, build_all]
     runs-on:
-      # Hacks, and order matters. See the comment at the top of the file.
+      # Pseudo-ternary hack and order matters. See comment at top of file.
       - self-hosted
       - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
       - cpu
@@ -305,7 +337,7 @@ jobs:
     needs: [build_tf_integrations, build_all]
     if: github.event_name != 'pull_request'
     runs-on:
-      # Hacks, and order matters. See the comment at the top of the file.
+      # Pseudo-ternary hack and order matters. See comment at top of file.
       - self-hosted
       - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
       - gpu
@@ -347,7 +379,7 @@ jobs:
 
   asan:
     runs-on:
-      # Hacks, and order matters. See the comment at the top of the file.
+      # Pseudo-ternary hack and order matters. See comment at top of file.
       - self-hosted
       - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
       - cpu
@@ -365,7 +397,7 @@ jobs:
 
   tsan:
     runs-on:
-      # Hacks, and order matters. See the comment at the top of the file.
+      # Pseudo-ternary hack and order matters. See comment at top of file.
       - self-hosted
       - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
       - cpu
@@ -384,7 +416,7 @@ jobs:
   riscv32:
     needs: host_tools_assertions
     runs-on:
-      # Hacks, and order matters. See the comment at the top of the file.
+      # Pseudo-ternary hack and order matters. See comment at top of file.
       - self-hosted
       - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
       - cpu
@@ -419,7 +451,7 @@ jobs:
   riscv64:
     needs: [build_all, host_tools_assertions, build_tf_integrations]
     runs-on:
-      # Hacks, and order matters. See the comment at the top of the file.
+      # Pseudo-ternary hack and order matters. See comment at top of file.
       - self-hosted
       - runner-group=${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}
       - cpu
@@ -477,6 +509,7 @@ jobs:
       - build_runtime
       - test_runtime
       - build_all
+      - build_test_all_bazel
       - test_all
       - test_gpu
       - host_tools_assertions

diff --git a/build_tools/bazel/build_core.sh b/build_tools/bazel/build_core.sh
@@ -20,7 +20,16 @@
 #   See https://docs.bazel.build/versions/master/command-line-reference.html#flag--test_tag_filters)
 #   Default: If IREE_VULKAN_DISABLE=1, "-nokokoro,-driver=vulkan". Else "-nokokoro".
 
-set -exuo pipefail
+set -xeuo pipefail
+
+
+IREE_BAZEL_READ_REMOTE_CACHE="${IREE_BAZEL_READ_REMOTE_CACHE:-1}"
+IREE_BAZEL_WRITE_REMOTE_CACHE="${IREE_BAZEL_WRITE_REMOTE_CACHE:-0}"
+BAZEL_BIN="${BAZEL_BIN:-$(which bazel)}"
+
+if (( ${IREE_BAZEL_WRITE_REMOTE_CACHE} == 1 && ${IREE_BAZEL_READ_REMOTE_CACHE} != 1 )); then
+  echo "Can't have 'IREE_BAZEL_WRITE_REMOTE_CACHE' (${IREE_BAZEL_WRITE_REMOTE_CACHE}) set without 'IREE_BAZEL_READ_REMOTE_CACHE' (${IREE_BAZEL_READ_REMOTE_CACHE})"
+fi
 
 # Use user-environment variables if set, otherwise use CI-friendly defaults.
 if ! [[ -v IREE_VULKAN_DISABLE ]]; then
@@ -64,8 +73,7 @@ if ! [[ -v TEST_TAG_FILTERS ]]; then
   TEST_TAG_FILTERS="$(IFS="," ; echo "${default_test_tag_filters[*]?}")"
 fi
 
-# Build and test everything in supported directories not excluded by the tag
-# filters.
+# Build and test everything in the main workspace (not integrations).
 # Note that somewhat contrary to its name `bazel test` will also build
 # any non-test targets specified.
 # We use `bazel query //...` piped to `bazel test` rather than the simpler
@@ -77,21 +85,41 @@ fi
 # xargs is set to high arg limits to avoid multiple Bazel invocations and will
 # hard fail if the limits are exceeded.
 # See https://github.com/bazelbuild/bazel/issues/12479
-bazel \
-  --noworkspace_rc \
-  --bazelrc=build_tools/bazel/iree.bazelrc \
-  query //... | \
-      xargs --max-args 1000000 --max-chars 1000000 --exit \
-        bazel \
-          --noworkspace_rc \
-          --bazelrc=build_tools/bazel/iree.bazelrc \
-            test \
-              --color=yes \
-              ${test_env_args[@]} \
-              --config=generic_clang \
-              --build_tag_filters="${BUILD_TAG_FILTERS?}" \
-              --test_tag_filters="${TEST_TAG_FILTERS?}" \
-              --keep_going \
-              --test_output=errors \
-              --config=rs \
-              --config=remote_cache_bazel_ci
+
+declare -a BAZEL_STARTUP_CMD=(
+  "${BAZEL_BIN}"
+  --noworkspace_rc
+  --bazelrc=build_tools/bazel/iree.bazelrc
+)
+
+declare -a BAZEL_TEST_CMD=(
+  "${BAZEL_STARTUP_CMD[@]}"
+  test
+)
+
+if [[ ! -z "${SANDBOX_BASE}" ]]; then
+  BAZEL_TEST_CMD+=(--sandbox_base="${SANDBOX_BASE}")
+fi
+
+if (( IREE_BAZEL_READ_REMOTE_CACHE == 1 )); then
+  BAZEL_TEST_CMD+=(--config=remote_cache_bazel_ci)
+fi
+
+if (( IREE_BAZEL_WRITE_REMOTE_CACHE != 1 )); then
+  BAZEL_TEST_CMD+=(--noremote_upload_local_results)
+fi
+
+BAZEL_TEST_CMD+=(
+  --color=yes
+  "${test_env_args[@]}"
+  --build_tag_filters="${BUILD_TAG_FILTERS?}"
+  --test_tag_filters="${TEST_TAG_FILTERS?}"
+  --keep_going
+  --test_output=errors
+  --config=rs
+  --config=generic_clang
+)
+
+"${BAZEL_STARTUP_CMD[@]}" query //... | \
+  xargs --max-args 1000000 --max-chars 1000000 --exit \
+    "${BAZEL_TEST_CMD[@]}"
diff --git a/build_tools/docker/docker_run.sh b/build_tools/docker/docker_run.sh
@@ -86,6 +86,13 @@ function docker_run() {
       --volume="${HOME?}/.config/gcloud:${HOME?}/.config/gcloud:ro"
     )
 
+    # Give the container a ramdisk and set the Bazel sandbox base to point to
+    # it. This helps a lot with Bazel getting IO bound.
+    DOCKER_RUN_ARGS+=(
+      --tmpfs /dev/shm
+      --env SANDBOX_BASE=/dev/shm
+    )
+
     docker run "${DOCKER_RUN_ARGS[@]}" "$@"
 }
 

diff --git a/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-swiftshader/core/build.sh b/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-swiftshader/core/build.sh
@@ -24,5 +24,7 @@ bazel --version
 echo "Initializing submodules"
 git submodule update --init --jobs 8 --depth 1
 
+export IREE_BAZEL_WRITE_REMOTE_CACHE=1
+
 echo "Building and testing with bazel"
 ./build_tools/bazel/build_core.sh