From 3ccc5dc20488c66f2d6290648e539da780aa0175 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Thu, 11 Jan 2024 11:12:17 +0100 Subject: [PATCH 1/2] Submit CTest results for compile-only tests to elastic --- .gitlab/docker/Dockerfile.spack_build | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.gitlab/docker/Dockerfile.spack_build b/.gitlab/docker/Dockerfile.spack_build index 208a09a5d..2651607ac 100644 --- a/.gitlab/docker/Dockerfile.spack_build +++ b/.gitlab/docker/Dockerfile.spack_build @@ -18,7 +18,17 @@ RUN spack spec -lI $spack_spec RUN spack build-env $spack_spec -- bash -c "cmake -B${BUILD_DIR} ${SOURCE_DIR} \ $CMAKE_COMMON_FLAGS $CMAKE_FLAGS && cmake --build ${BUILD_DIR} --target all tests examples" -# Run compile only tests and tests.unit.build -RUN spack build-env $spack_spec -- bash -c "ctest --label-regex COMPILE_ONLY \ - --test-dir ${BUILD_DIR} -j${NUM_PROCS} --timeout 30 --output-on-failure --no-compress-output \ - -R tests --no-tests=error" +# Run compile only tests and tests.unit.build, submit ctest metrics to elastic +RUN CTEST_XML=$PWD/ctest.xml; \ + trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT; \ + spack build-env $spack_spec -- \ + bash -c "ctest \ + --output-junit ${CTEST_XML} \ + --label-regex COMPILE_ONLY \ + --test-dir ${BUILD_DIR} \ + -j${NUM_PROCS} \ + --timeout 30 \ + --output-on-failure \ + --no-compress-output \ + -R tests \ + -no-tests=error" From ee540669ec1f421e9fcf7ab3d100fb5304629446 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Thu, 11 Jan 2024 12:28:57 +0100 Subject: [PATCH 2/2] Pass CI and other environment variables to build stage in CSCS CI --- .gitlab/docker/Dockerfile.spack_build | 14 ++++++++++++++ .gitlab/includes/clang11_pipeline.yml | 4 +--- .gitlab/includes/clang12_pipeline.yml | 4 +--- .gitlab/includes/clang13_pipeline.yml | 4 +--- .gitlab/includes/clang14_cuda11_pipeline.yml | 4 +--- .gitlab/includes/clang15_pipeline.yml | 4 +--- .gitlab/includes/clang16_pipeline.yml | 4 +--- .gitlab/includes/common_pipeline.yml | 4 ++++ .gitlab/includes/gcc11_pipeline.yml | 4 +--- .gitlab/includes/gcc12_cuda12_pipeline.yml | 4 +--- .gitlab/includes/gcc12_pipeline.yml | 4 +--- .gitlab/includes/gcc13_pipeline.yml | 4 +--- .gitlab/includes/gcc9_cuda11_pipeline.yml | 4 +--- .gitlab/includes/gcc9_pipeline.yml | 4 +--- .gitlab/includes/hip_pipeline.yml | 4 +--- 15 files changed, 31 insertions(+), 39 deletions(-) diff --git a/.gitlab/docker/Dockerfile.spack_build b/.gitlab/docker/Dockerfile.spack_build index 2651607ac..55ed0b102 100644 --- a/.gitlab/docker/Dockerfile.spack_build +++ b/.gitlab/docker/Dockerfile.spack_build @@ -19,6 +19,20 @@ RUN spack build-env $spack_spec -- bash -c "cmake -B${BUILD_DIR} ${SOURCE_DIR} \ $CMAKE_COMMON_FLAGS $CMAKE_FLAGS && cmake --build ${BUILD_DIR} --target all tests examples" # Run compile only tests and tests.unit.build, submit ctest metrics to elastic +ARG ARCH +ARG BUILD_TYPE +ARG CI_COMMIT_AUTHOR +ARG CI_COMMIT_BRANCH +ARG CI_COMMIT_DESCRIPTION +ARG CI_COMMIT_MESSAGE +ARG CI_COMMIT_SHA +ARG CI_COMMIT_SHORT_SHA +ARG CI_COMMIT_TIMESTAMP +ARG CI_COMMIT_TITLE +ARG COMPILER +ARG CSCS_LOGSTASH_URL +ARG SPACK_COMMIT +ARG SPACK_SPEC RUN CTEST_XML=$PWD/ctest.xml; \ trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT; \ spack build-env $spack_spec -- \ diff --git a/.gitlab/includes/clang11_pipeline.yml b/.gitlab/includes/clang11_pipeline.yml index 5c38a0e4e..c1577792a 100644 --- a/.gitlab/includes/clang11_pipeline.yml +++ b/.gitlab/includes/clang11_pipeline.yml @@ -45,6 +45,7 @@ clang11_debug_build: extends: - .container-builder - .variables_clang11_config + - .build_variables_common - .cmake_variables_common needs: - clang11_spack_image @@ -56,9 +57,6 @@ clang11_debug_build: - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env - variables: - DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: dotenv: build.env diff --git a/.gitlab/includes/clang12_pipeline.yml b/.gitlab/includes/clang12_pipeline.yml index b39e330ce..02e0bd79c 100644 --- a/.gitlab/includes/clang12_pipeline.yml +++ b/.gitlab/includes/clang12_pipeline.yml @@ -49,6 +49,7 @@ clang12_debug_build: extends: - .container-builder - .variables_clang12_config + - .build_variables_common - .cmake_variables_common needs: - clang12_spack_image @@ -60,9 +61,6 @@ clang12_debug_build: - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env - variables: - DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: dotenv: build.env diff --git a/.gitlab/includes/clang13_pipeline.yml b/.gitlab/includes/clang13_pipeline.yml index 51179864d..fd3f0b192 100644 --- a/.gitlab/includes/clang13_pipeline.yml +++ b/.gitlab/includes/clang13_pipeline.yml @@ -46,6 +46,7 @@ clang13_debug_build: extends: - .container-builder - .variables_clang13_config + - .build_variables_common - .cmake_variables_common needs: - clang13_spack_image @@ -57,9 +58,6 @@ clang13_debug_build: - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env - variables: - DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: dotenv: build.env diff --git a/.gitlab/includes/clang14_cuda11_pipeline.yml b/.gitlab/includes/clang14_cuda11_pipeline.yml index dc0def9a8..8de003c35 100644 --- a/.gitlab/includes/clang14_cuda11_pipeline.yml +++ b/.gitlab/includes/clang14_cuda11_pipeline.yml @@ -48,6 +48,7 @@ clang14_cuda11_debug_build: extends: - .container-builder - .variables_clang14_cuda11_config + - .build_variables_common - .cmake_variables_common needs: - clang14_cuda11_spack_image @@ -59,9 +60,6 @@ clang14_cuda11_debug_build: - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env - variables: - DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: dotenv: build.env diff --git a/.gitlab/includes/clang15_pipeline.yml b/.gitlab/includes/clang15_pipeline.yml index 85e936114..71e530bce 100644 --- a/.gitlab/includes/clang15_pipeline.yml +++ b/.gitlab/includes/clang15_pipeline.yml @@ -44,6 +44,7 @@ clang15_debug_build: extends: - .container-builder - .variables_clang15_config + - .build_variables_common - .cmake_variables_common needs: - clang15_spack_image @@ -55,9 +56,6 @@ clang15_debug_build: - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env - variables: - DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: dotenv: build.env diff --git a/.gitlab/includes/clang16_pipeline.yml b/.gitlab/includes/clang16_pipeline.yml index bf568c738..a00b62be5 100644 --- a/.gitlab/includes/clang16_pipeline.yml +++ b/.gitlab/includes/clang16_pipeline.yml @@ -44,6 +44,7 @@ clang16_debug_build: extends: - .container-builder - .variables_clang16_config + - .build_variables_common - .cmake_variables_common needs: - clang16_spack_image @@ -55,9 +56,6 @@ clang16_debug_build: - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env - variables: - DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: dotenv: build.env diff --git a/.gitlab/includes/common_pipeline.yml b/.gitlab/includes/common_pipeline.yml index f52ff572c..b1f3c2676 100644 --- a/.gitlab/includes/common_pipeline.yml +++ b/.gitlab/includes/common_pipeline.yml @@ -24,6 +24,10 @@ variables: -DPIKA_WITH_CHECK_MODULE_DEPENDENCIES=ON \ -DPIKA_WITH_EXAMPLES=ON -DPIKA_WITH_TESTS=ON \ -DPIKA_WITH_PARALLEL_TESTS_BIND_NONE=ON" +.build_variables_common: + variables: + DOCKERFILE: .gitlab/docker/Dockerfile.spack_build + DOCKER_BUILD_ARGS: '["ARCH","BASE_IMAGE","BUILD_TYPE","CI_COMMIT_AUTHOR","CI_COMMIT_BRANCH","CI_COMMIT_DESCRIPTION","CI_COMMIT_MESSAGE","CI_COMMIT_SHA","CI_COMMIT_SHORT_SHA","CI_COMMIT_TIMESTAMP","CI_COMMIT_TITLE","COMPILER","CSCS_LOGSTASH_URL","SPACK_COMMIT","SPACK_SPEC","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' .test_common: variables: diff --git a/.gitlab/includes/gcc11_pipeline.yml b/.gitlab/includes/gcc11_pipeline.yml index 2b7549843..a5060d801 100644 --- a/.gitlab/includes/gcc11_pipeline.yml +++ b/.gitlab/includes/gcc11_pipeline.yml @@ -45,6 +45,7 @@ gcc11_debug_build: extends: - .container-builder - .variables_gcc11_config + - .build_variables_common - .cmake_variables_common needs: - gcc11_spack_image @@ -56,9 +57,6 @@ gcc11_debug_build: - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env - variables: - DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: dotenv: build.env diff --git a/.gitlab/includes/gcc12_cuda12_pipeline.yml b/.gitlab/includes/gcc12_cuda12_pipeline.yml index f9dd131bb..23f7b62d4 100644 --- a/.gitlab/includes/gcc12_cuda12_pipeline.yml +++ b/.gitlab/includes/gcc12_cuda12_pipeline.yml @@ -48,6 +48,7 @@ gcc12_cuda12_debug_build: extends: - .container-builder - .variables_gcc12_cuda12_config + - .build_variables_common - .cmake_variables_common needs: - gcc12_cuda12_spack_image @@ -59,9 +60,6 @@ gcc12_cuda12_debug_build: - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env - variables: - DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: dotenv: build.env diff --git a/.gitlab/includes/gcc12_pipeline.yml b/.gitlab/includes/gcc12_pipeline.yml index b6276325e..22b8d335d 100644 --- a/.gitlab/includes/gcc12_pipeline.yml +++ b/.gitlab/includes/gcc12_pipeline.yml @@ -44,6 +44,7 @@ gcc12_debug_build: extends: - .container-builder - .variables_gcc12_config + - .build_variables_common - .cmake_variables_common needs: - gcc12_spack_image @@ -55,9 +56,6 @@ gcc12_debug_build: - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env - variables: - DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: dotenv: build.env diff --git a/.gitlab/includes/gcc13_pipeline.yml b/.gitlab/includes/gcc13_pipeline.yml index f45816257..f85d2a458 100644 --- a/.gitlab/includes/gcc13_pipeline.yml +++ b/.gitlab/includes/gcc13_pipeline.yml @@ -46,6 +46,7 @@ gcc13_debug_build: extends: - .container-builder - .variables_gcc13_config + - .build_variables_common - .cmake_variables_common needs: - gcc13_spack_image @@ -57,9 +58,6 @@ gcc13_debug_build: - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env - variables: - DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: dotenv: build.env diff --git a/.gitlab/includes/gcc9_cuda11_pipeline.yml b/.gitlab/includes/gcc9_cuda11_pipeline.yml index fd154aa10..7f1eb3190 100644 --- a/.gitlab/includes/gcc9_cuda11_pipeline.yml +++ b/.gitlab/includes/gcc9_cuda11_pipeline.yml @@ -45,6 +45,7 @@ gcc9_cuda11_debug_build: extends: - .container-builder - .variables_gcc9_cuda11_config + - .build_variables_common - .cmake_variables_common needs: - gcc9_cuda11_spack_image @@ -56,9 +57,6 @@ gcc9_cuda11_debug_build: - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env - variables: - DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: dotenv: build.env diff --git a/.gitlab/includes/gcc9_pipeline.yml b/.gitlab/includes/gcc9_pipeline.yml index a84a169dd..acff69ca9 100644 --- a/.gitlab/includes/gcc9_pipeline.yml +++ b/.gitlab/includes/gcc9_pipeline.yml @@ -45,6 +45,7 @@ gcc9_debug_build: extends: - .container-builder - .variables_gcc9_config + - .build_variables_common - .cmake_variables_common needs: - gcc9_spack_image @@ -56,9 +57,6 @@ gcc9_debug_build: - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env - variables: - DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: dotenv: build.env diff --git a/.gitlab/includes/hip_pipeline.yml b/.gitlab/includes/hip_pipeline.yml index 09169a188..8a9595e86 100644 --- a/.gitlab/includes/hip_pipeline.yml +++ b/.gitlab/includes/hip_pipeline.yml @@ -48,6 +48,7 @@ gcc12_hip5_debug_build: extends: - .container-builder - .variables_gcc12_hip5_config + - .build_variables_common - .cmake_variables_common needs: - gcc12_hip5_spack_image @@ -59,9 +60,6 @@ gcc12_hip5_debug_build: - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16` - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env - variables: - DOCKERFILE: .gitlab/docker/Dockerfile.spack_build - DOCKER_BUILD_ARGS: '["BASE_IMAGE","SOURCE_DIR","BUILD_DIR","CMAKE_COMMON_FLAGS","CMAKE_FLAGS"]' artifacts: reports: dotenv: build.env