Add a CI check for testing toolchains as-distributed (#449)

* Add a CI check for testing toolchains as-distributed

One aspect of testing lost in the CMake-based migration is the ability
to test the toolchains as distributed in release artifacts. Tests use
`--sysroot` and `-resource-dir` (soon) to customize how the host
compiler runs but this means that it would be possible to regress the
default sysroot theoretically. To rectify this situation this commit
adds a new CI test which uses the release artifacts of previous steps to
build a `wasi-sdk-*.tar.gz` tarball which is then extracted and tested
as-is. A new flag was added to the cmake configuration to avoid
depending on fresh sysroot libraries for tests and instead test the host
toolchain.

* Fix version.py script running

* Fix artifact download

* Add ninja

* Update submodules in new test job

* Only add extra options for libcxx build

Otherwise the test directory seems like it inherits these options which
isn't desired when testing the host toolchain.

.github/workflows/main.yml

@@ -211,3 +211,39 @@ jobs:
         run: gh release create --draft --prerelease --generate-notes ${{ github.ref_name }} ./dist/*
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  # Test the final artifacts as-is without passing `--sysroot` or
+  # `-resource-dir` or any extra flags. This exercises running the compiler
+  # as-is from the distribution tarballs and ensuring that it can build and pass
+  # all tests.
+  test-standalone:
+    name: Test standalone toolchain
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - run: git fetch --tags --force
+        name: Force-fetch tags to work around actions/checkout#290
+      - run: git submodule update --init --depth 32 --jobs 3
+      - name: Setup `wasmtime` for tests
+        uses: bytecodealliance/actions/wasmtime/setup@v1
+        with:
+          version: "18.0.2"
+      - name: Install ninja
+        run: sudo apt-get install -y ninja-build
+        if: runner.os == 'Linux'
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist-x86_64-linux
+          path: dist-x86_64-linux
+      - run: ./ci/merge-artifacts.sh
+      - run: tar xf dist/wasi-sdk-*.tar.gz
+      - run: |
+          cmake -G Ninja -B build -S . \
+            -DWASI_SDK_INCLUDE_TESTS=ON \
+            -DWASI_SDK_TEST_HOST_TOOLCHAIN=ON \
+            -DCMAKE_TOOLCHAIN_FILE=$(ls ./wasi-sdk-*/share/cmake/wasi-sdk.cmake)
+      - run: ninja -C build build-tests
+      - run: ctest --output-on-failure --parallel 10 --test-dir build/tests

README.md

@@ -85,6 +85,8 @@ in compiling WebAssembly code. Supported CMake flags are:
 * `-DWASI_SDK_DEBUG_PREFIX_MAKE=OFF` - disable `-fdebug-prefix-map` when
   building C/C++ code to use full host paths instead.
 * `-DWASI_SDK_INCLUDE_TESTS=ON` - used for building tests.
+* `-DWASI_SDK_TEST_HOST_TOOLCHAIN=ON` - test the host toolchain's wasi-libc and
+  sysroot libraries, don't build or use fresh libraries for tests.
 * `-DWASI_SDK_TARGETS=..` - a list of targets to build, by default all WASI
   targets are compiled.
 

cmake/wasi-sdk-sysroot.cmake

@@ -14,11 +14,6 @@ set(wasi_tmp_install ${CMAKE_CURRENT_BINARY_DIR}/install)
 set(wasi_sysroot ${wasi_tmp_install}/share/wasi-sysroot)
 set(wasi_resource_dir ${wasi_tmp_install}/lib/clang/${clang_version})
 
-# Force usage of the custom-built resource-dir and sysroot for the rest of the
-# wasi compiles.
-add_compile_options(-resource-dir ${wasi_resource_dir})
-add_compile_options(--sysroot ${wasi_sysroot})
-
 if(WASI_SDK_DEBUG_PREFIX_MAP)
   add_compile_options(
     -fdebug-prefix-map=${CMAKE_CURRENT_SOURCE_DIR}=wasisdk://v${wasi_sdk_version})
@@ -167,9 +162,17 @@ function(define_libcxx target)
 
   get_property(dir_compile_opts DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY COMPILE_OPTIONS)
   get_property(dir_link_opts DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY LINK_OPTIONS)
-  set(extra_cflags_list ${CMAKE_C_FLAGS} ${target_flags} --target=${target} ${dir_compile_opts} ${dir_link_opts})
+  set(extra_flags
+    ${target_flags}
+    --target=${target}
+    ${dir_compile_opts}
+    ${dir_link_opts}
+    --sysroot ${wasi_sysroot}
+    -resource-dir ${wasi_resource_dir})
+
+  set(extra_cflags_list ${CMAKE_C_FLAGS} ${extra_flags})
   list(JOIN extra_cflags_list " " extra_cflags)
-  set(extra_cxxflags_list ${CMAKE_CXX_FLAGS} ${target_flags} --target=${target} ${dir_compile_opts} ${dir_link_opts})
+  set(extra_cxxflags_list ${CMAKE_CXX_FLAGS} ${extra_flags})
   list(JOIN extra_cxxflags_list " " extra_cxxflags)
 
   ExternalProject_Add(libcxx-${target}-build

tests/CMakeLists.txt

@@ -5,8 +5,12 @@ include(CTest)
 enable_testing()
 set(CMAKE_EXECUTABLE_SUFFIX ".wasm")
 
-add_compile_options(--sysroot=${wasi_sysroot} -resource-dir ${wasi_resource_dir})
-add_link_options(--sysroot=${wasi_sysroot} -resource-dir ${wasi_resource_dir})
+option(WASI_SDK_TEST_HOST_TOOLCHAIN "Test against the host toolchain, not a fresh sysroot" OFF)
+
+if(NOT WASI_SDK_TEST_HOST_TOOLCHAIN)
+  add_compile_options(--sysroot=${wasi_sysroot} -resource-dir ${wasi_resource_dir})
+  add_link_options(--sysroot=${wasi_sysroot} -resource-dir ${wasi_resource_dir})
+endif()
 
 # Sanity check setup
 if (NOT ${CMAKE_SYSTEM_NAME} STREQUAL WASI)
@@ -22,6 +26,8 @@ set(WASI_SDK_RUNWASI "wasmtime" CACHE STRING "Runner for tests")
 # Test everything at O0, O2, and O2+LTO
 set(opt_flags -O0 -O2 "-O2 -flto")
 
+add_custom_target(build-tests)
+
 # Executes a single `test` specified.
 #
 # This will compile `test` for all the various targets and with various
@@ -35,6 +41,7 @@ function(add_testcase runwasi test)
 
       # Add a new test executable based on `test`
       add_executable(${target_name} ${test})
+      add_dependencies(build-tests ${target_name})
 
       # Configure all the compile options necessary. For example `--target` here
       # if the target doesn't look like it's already in the name of the compiler
@@ -60,9 +67,13 @@ function(add_testcase runwasi test)
       # Apply language-specific options and dependencies.
       if(test MATCHES "cc$")
         target_compile_options(${target_name} PRIVATE -fno-exceptions)
-        add_dependencies(${target_name} libcxx-${target})
+        if(NOT WASI_SDK_TEST_HOST_TOOLCHAIN)
+          add_dependencies(${target_name} libcxx-${target})
+        endif()
       else()
-        add_dependencies(${target_name} wasi-libc-${target})
+        if(NOT WASI_SDK_TEST_HOST_TOOLCHAIN)
+          add_dependencies(${target_name} wasi-libc-${target})
+        endif()
       endif()
 
       # Apply target-specific options.