Include most Vulkan tests in ASan build. (#8489)

Progress on #5715 and #5716 Leaks in the Vulkan-related libraries we use were hidden behind incomplete handling of shared library loading/unloading in ASan. By disabling calls to `dlclose()` in both `iree/base/internal/dynamic_library_posix.c` and the Vulkan Loader (`libvulkan.so.1`) so those libraries remained open for ASan to reference, I was able to get useful leak reports. Those reports showed that my NVIDIA system Vulkan ICD (`libnvidia-glcore.so`) was leaking and an up to date SwiftShader (`libvk_swiftshader.so`) was _not_ leaking. This PR updates SwiftShader to a commit that doesn't leak (with our usage, anyways) and enables most of the Vulkan tests that were previously excluded from running under ASan. --- A few tests are still failing with crashes in ASan, with logs like this: ``` Tracer caught signal 11: addr=0x0 pc=0x50c558 sp=0x7fb28fdffd10 ==50923==LeakSanitizer has encountered a fatal error. ==50923==HINT: For debugging, try setting environment variable LSAN_OPTIONS=verbosity=1:log_threads=1 ==50923==HINT: LeakSanitizer does not work under ptrace (strace, gdb, etc) ``` ([full logs here](https://source.cloud.google.com/results/invocations/a37ab871-4cab-4591-a5d6-8ad849f196e3/targets/iree%2Fgcp_ubuntu%2Fcmake%2Flinux%2Fx86-swiftshader-asan%2Fpresubmit/log)), so I'm keeping those disabled explicitly.
iree-org · Mar 15, 2022 · bcb6199 · bcb6199
2 parents 590a852 + e35d003
commit bcb6199
Show file tree

Hide file tree

Showing 13 changed files with 25 additions and 33 deletions.
diff --git a/build_tools/bazel/iree.bazelrc b/build_tools/bazel/iree.bazelrc
@@ -264,7 +264,7 @@ build:remote_cache_bazel_ci --config=_remote_cache_base
 # specific docker container the CI Bazel builds are run in. The image URL is
 # included for clarity and so that this reference is automatically updated by
 # manage_images.py
-build:remote_cache_bazel_ci --host_platform_remote_properties_override='properties:{name:"cache-silo-key" value:"gcr.io/iree-oss/frontends-swiftshader@sha256:c73aef3cb6ac80fa23583bed49e68edaf148ff7c2a40a700b60c3ccb4c5584b9"}'
+build:remote_cache_bazel_ci --host_platform_remote_properties_override='properties:{name:"cache-silo-key" value:"gcr.io/iree-oss/frontends-swiftshader@sha256:ba1dd452360a8942316b6f095abd555e4214ed5c3cdc892038d38eaf4bb8d162"}'
 
 ###############################################################################
 # Configuration for uploading build results to Result Store UI

diff --git a/build_tools/buildkite/samples.yml b/build_tools/buildkite/samples.yml
@@ -7,15 +7,15 @@
 steps:
   - label: "Test Colab notebooks"
     commands:
-      - "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/samples@sha256:4b382911b10c72729563da7b1017cd2ac309edc8d7630b3f4d38911b51aa8813 python3 ./colab/test_notebooks.py"
+      - "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/samples@sha256:cc54bc69193197be33115615ebe7eea7691812f0e87efb54bf8fd7061a19c6ae python3 ./colab/test_notebooks.py"
     env:
       IREE_DOCKER_WORKDIR: "/usr/src/github/iree"
     agents:
       - "queue=build"
 
   - label: "Test Samples"
     commands:
-      - "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/samples@sha256:4b382911b10c72729563da7b1017cd2ac309edc8d7630b3f4d38911b51aa8813 ./build_tools/testing/test_samples.sh"
+      - "docker run --user=$(id -u):$(id -g) --volume=\\$PWD:\\$IREE_DOCKER_WORKDIR --workdir=\\$IREE_DOCKER_WORKDIR --rm gcr.io/iree-oss/samples@sha256:cc54bc69193197be33115615ebe7eea7691812f0e87efb54bf8fd7061a19c6ae ./build_tools/testing/test_samples.sh"
     env:
       IREE_DOCKER_WORKDIR: "/usr/src/github/iree"
     agents:

diff --git a/build_tools/cmake/test.sh b/build_tools/cmake/test.sh
@@ -18,8 +18,7 @@ fi
 # Respect the user setting, but default to as many jobs as we have cores.
 export CTEST_PARALLEL_LEVEL=${CTEST_PARALLEL_LEVEL:-$(nproc)}
 
-# Respect the user setting, but default to turning off the vulkan tests
-# and turning on the llvmaot ones.
+# Respect the user setting, but default to turning on vulkan and llvmaot.
 export IREE_VULKAN_DISABLE=${IREE_VULKAN_DISABLE:-0}
 export IREE_LLVMAOT_DISABLE=${IREE_LLVMAOT_DISABLE:-0}
 # CUDA is off by default.

diff --git a/build_tools/docker/frontends-swiftshader/Dockerfile b/build_tools/docker/frontends-swiftshader/Dockerfile
@@ -5,7 +5,7 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 FROM gcr.io/iree-oss/frontends@sha256:3eab65512589e7dabeced8aeb2c392e82f5bf1caafad7639e1b679be908ceb33
-COPY --from=gcr.io/iree-oss/swiftshader@sha256:7c030be534f1dfd105a82d7ba31cf0500c80fb60f34ca1cf683cc08cb0e7e46c \
+COPY --from=gcr.io/iree-oss/swiftshader@sha256:cbb7bd12f095df6d2eabbd0317fca9d44c4edd130d7a6c83e8363142c3e7468c \
   /swiftshader /swiftshader
 
 # Set VK_ICD_FILENAMES so Vulkan loader can find the SwiftShader ICD.

diff --git a/build_tools/docker/prod_digests.txt b/build_tools/docker/prod_digests.txt
@@ -1,9 +1,9 @@
 gcr.io/iree-oss/base@sha256:1e551dda4cfdc8f92dc13c4c2c8e5f5b79d3103a4534bcc002ef5cc02ee9dd4f
-gcr.io/iree-oss/swiftshader@sha256:7c030be534f1dfd105a82d7ba31cf0500c80fb60f34ca1cf683cc08cb0e7e46c
-gcr.io/iree-oss/samples@sha256:4b382911b10c72729563da7b1017cd2ac309edc8d7630b3f4d38911b51aa8813
+gcr.io/iree-oss/swiftshader@sha256:cbb7bd12f095df6d2eabbd0317fca9d44c4edd130d7a6c83e8363142c3e7468c
+gcr.io/iree-oss/samples@sha256:cc54bc69193197be33115615ebe7eea7691812f0e87efb54bf8fd7061a19c6ae
 gcr.io/iree-oss/frontends@sha256:3eab65512589e7dabeced8aeb2c392e82f5bf1caafad7639e1b679be908ceb33
 gcr.io/iree-oss/frontends-nvidia@sha256:901a83b3f16e0d31fe239cc6a79db0230d80086d36821a839eaa382edb10dabd
-gcr.io/iree-oss/frontends-swiftshader@sha256:c73aef3cb6ac80fa23583bed49e68edaf148ff7c2a40a700b60c3ccb4c5584b9
+gcr.io/iree-oss/frontends-swiftshader@sha256:ba1dd452360a8942316b6f095abd555e4214ed5c3cdc892038d38eaf4bb8d162
 gcr.io/iree-oss/gradle-android@sha256:bd29fa61f5e274ca86b80ee5cf8e3db7b36177fbc7c9f4121c486fc71100d878
 gcr.io/iree-oss/riscv@sha256:0d2bc912b0f32e3e38fcacaf1175cafbb60871bb93996ab5518d95adfa760873
 gcr.io/iree-oss/nvidia@sha256:7d015fd46d02b13d5da03085e1e77d27f36d466abc573f28e135c3a1533fb530

diff --git a/build_tools/docker/samples/Dockerfile b/build_tools/docker/samples/Dockerfile
@@ -9,7 +9,7 @@
 #   * Vulkan (using SwiftShader)
 #   * Python (including `venv` and common pip packages needed for Colab)
 
-FROM gcr.io/iree-oss/swiftshader@sha256:7c030be534f1dfd105a82d7ba31cf0500c80fb60f34ca1cf683cc08cb0e7e46c
+FROM gcr.io/iree-oss/swiftshader@sha256:cbb7bd12f095df6d2eabbd0317fca9d44c4edd130d7a6c83e8363142c3e7468c
 
 # Update setuptools per https://github.com/pypa/setuptools/issues/1694#issuecomment-466010982
 RUN apt-get update \

diff --git a/build_tools/docker/swiftshader/Dockerfile b/build_tools/docker/swiftshader/Dockerfile
@@ -9,20 +9,16 @@ WORKDIR /install-swiftshader
 
 RUN apt-get update && apt-get install -y git
 
-ARG SWIFTSHADER_COMMIT=755b78dc66b2362621a78b6964a9df3af94e960c
+ARG SWIFTSHADER_COMMIT=c27e99245d423648fe5f0eafa73d49ce7b53d201
+
+# zlib and xcb/shm.h are needed for compiling SwiftShader.
+RUN apt-get update && apt-get install -y zlib1g-dev libxcb-shm0-dev
 
-# zlib is needed for compiling SwiftShader.
-RUN apt-get update && apt-get install -y zlib1g-dev
 RUN git clone https://github.com/google/swiftshader
 RUN cd swiftshader && git checkout "${SWIFTSHADER_COMMIT?}" && cd ..
 # Only build SwiftShader Vulkan ICD.
 RUN cmake -S swiftshader/ -B build-swiftshader/ \
            -GNinja \
-           -DSWIFTSHADER_BUILD_VULKAN=ON \
-           -DSWIFTSHADER_BUILD_EGL=OFF \
-           -DSWIFTSHADER_BUILD_GLESv2=OFF \
-           -DSWIFTSHADER_BUILD_GLES_CM=OFF \
-           -DSWIFTSHADER_BUILD_PVR=OFF \
            -DSWIFTSHADER_BUILD_TESTS=OFF
 RUN cmake --build build-swiftshader/ \
            --config Release \

diff --git a/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-swiftshader/core/build_kokoro.sh b/build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-swiftshader/core/build_kokoro.sh
@@ -28,7 +28,7 @@ docker_setup
 # and the cache key is the docker container it's run in (to ensure correct cache
 # hits).
 docker run "${DOCKER_RUN_ARGS[@]?}" \
-  gcr.io/iree-oss/frontends-swiftshader@sha256:c73aef3cb6ac80fa23583bed49e68edaf148ff7c2a40a700b60c3ccb4c5584b9 \
+  gcr.io/iree-oss/frontends-swiftshader@sha256:ba1dd452360a8942316b6f095abd555e4214ed5c3cdc892038d38eaf4bb8d162 \
   build_tools/kokoro/gcp_ubuntu/bazel/linux/x86-swiftshader/core/build.sh
 
 # Kokoro will rsync this entire directory back to the executor orchestrating the

diff --git a/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build_kokoro.sh b/build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build_kokoro.sh
@@ -24,7 +24,7 @@ source "${KOKORO_ARTIFACTS_DIR?}/github/iree/build_tools/kokoro/gcp_ubuntu/docke
 docker_setup
 
 docker run "${DOCKER_RUN_ARGS[@]?}" \
-  gcr.io/iree-oss/frontends-swiftshader@sha256:c73aef3cb6ac80fa23583bed49e68edaf148ff7c2a40a700b60c3ccb4c5584b9 \
+  gcr.io/iree-oss/frontends-swiftshader@sha256:ba1dd452360a8942316b6f095abd555e4214ed5c3cdc892038d38eaf4bb8d162 \
   build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build.sh
 
 # Kokoro will rsync this entire directory back to the executor orchestrating the

diff --git a/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader-asan/build.sh b/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader-asan/build.sh
@@ -52,10 +52,8 @@ echo "------------------"
 # Respect the user setting, but default to as many jobs as we have cores.
 export CTEST_PARALLEL_LEVEL=${CTEST_PARALLEL_LEVEL:-$(nproc)}
 
-# Respect the user setting, but default to turning off the vulkan tests
-# and turning on the llvmaot ones.
-# TODO(#5716): Fix and enable Vulkan tests.
-export IREE_VULKAN_DISABLE=${IREE_VULKAN_DISABLE:-1}
+# Respect the user setting, but default to turning on vulkan and llvmaot.
+export IREE_VULKAN_DISABLE=${IREE_VULKAN_DISABLE:-0}
 export IREE_LLVMAOT_DISABLE=${IREE_LLVMAOT_DISABLE:-0}
 # CUDA is off by default.
 export IREE_CUDA_DISABLE=${IREE_CUDA_DISABLE:-1}
@@ -105,7 +103,11 @@ label_exclude_regex="($(IFS="|" ; echo "${label_exclude_args[*]?}"))"
 # These tests currently have asan failures
 # TODO(#5715): Fix these
 declare -a excluded_tests=(
+  # Mysterious "LeakSanitizer has encountered a fatal error." crashes
   "iree/samples/simple_embedding/simple_embedding_vulkan_test"
+  "iree/tools/test/iree-benchmark-module.mlir.test"
+  "iree/tools/test/iree-run-module.mlir.test"
+  "iree/tools/test/multiple_exported_functions.mlir.test"
 )
 
 # Prefix with `^` anchor
@@ -120,5 +122,5 @@ cd ${CMAKE_BUILD_DIR?}
 
 echo "Testing with ctest"
 ctest --timeout 900 --output-on-failure \
-  --label-exclude "^driver=cuda$|^driver=vulkan$" \
+  --label-exclude "${label_exclude_regex}" \
   --exclude-regex "${excluded_tests_regex?}"
diff --git a/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader-asan/build_kokoro.sh b/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader-asan/build_kokoro.sh
@@ -24,7 +24,7 @@ source "${KOKORO_ARTIFACTS_DIR?}/github/iree/build_tools/kokoro/gcp_ubuntu/docke
 docker_setup
 
 docker run "${DOCKER_RUN_ARGS[@]?}" \
-  gcr.io/iree-oss/swiftshader@sha256:7c030be534f1dfd105a82d7ba31cf0500c80fb60f34ca1cf683cc08cb0e7e46c \
+  gcr.io/iree-oss/swiftshader@sha256:cbb7bd12f095df6d2eabbd0317fca9d44c4edd130d7a6c83e8363142c3e7468c \
   build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader-asan/build.sh
 
 # Kokoro will rsync this entire directory back to the executor orchestrating the

diff --git a/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader/build_kokoro.sh b/build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader/build_kokoro.sh
@@ -24,7 +24,7 @@ source "${KOKORO_ARTIFACTS_DIR?}/github/iree/build_tools/kokoro/gcp_ubuntu/docke
 docker_setup
 
 docker run "${DOCKER_RUN_ARGS[@]?}" \
-  gcr.io/iree-oss/swiftshader@sha256:7c030be534f1dfd105a82d7ba31cf0500c80fb60f34ca1cf683cc08cb0e7e46c \
+  gcr.io/iree-oss/swiftshader@sha256:cbb7bd12f095df6d2eabbd0317fca9d44c4edd130d7a6c83e8363142c3e7468c \
   build_tools/kokoro/gcp_ubuntu/cmake/linux/x86-swiftshader/build.sh
 
 # Kokoro will rsync this entire directory back to the executor orchestrating the

diff --git a/build_tools/third_party/swiftshader/build_vk_swiftshader.sh b/build_tools/third_party/swiftshader/build_vk_swiftshader.sh
@@ -54,7 +54,7 @@ else
   SWIFTSHADER_INSTALL_DIR="${BASE_DIR?}"'\.swiftshader'
 fi
 
-SWIFTSHADER_COMMIT=755b78dc66b2362621a78b6964a9df3af94e960c
+SWIFTSHADER_COMMIT=c27e99245d423648fe5f0eafa73d49ce7b53d201
 SWIFTSHADER_DIR="$(mktemp --directory --tmpdir swiftshader_XXXXXX)"
 
 #  Clone swiftshader and checkout the appropriate commit.
@@ -77,11 +77,6 @@ fi
 
 cmake -B "${SWIFTSHADER_INSTALL_DIR?}" \
     -GNinja \
-    -DSWIFTSHADER_BUILD_VULKAN=ON \
-    -DSWIFTSHADER_BUILD_EGL=OFF \
-    -DSWIFTSHADER_BUILD_GLESv2=OFF \
-    -DSWIFTSHADER_BUILD_GLES_CM=OFF \
-    -DSWIFTSHADER_BUILD_PVR=OFF \
     -DSWIFTSHADER_BUILD_TESTS=OFF \
     "${SWIFTSHADER_DIR?}"