Refactor Clang CSCS CI configurations

pika-org · Jan 22, 2024 · ca3a650 · ca3a650
1 parent cf3b038
commit ca3a650
Show file tree

Hide file tree

Showing 7 changed files with 49 additions and 401 deletions.
diff --git a/.gitlab/includes/clang11_pipeline.yml b/.gitlab/includes/clang11_pipeline.yml
@@ -21,75 +21,25 @@ include:
                   -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON"
 
 clang11_spack_compiler_image:
-  stage: spack_compiler
-  needs: [base_spack_image]
   extends:
-    - .container-builder
+    - .compiler_image_template
     - .variables_clang11_config
-  before_script:
-    - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16`
-    - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16`
-    - compiler=${COMPILER/@/-}
-    - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG
-    - echo -e "compiler=$compiler" >> compiler.env
-    - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env
-  variables:
-    DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler
-    DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]'
-  artifacts:
-    reports:
-      dotenv: compiler.env
 
 clang11_spack_image:
-  stage: spack_dependencies
   needs: [clang11_spack_compiler_image]
   extends:
-    - .container-builder
+    - .dependencies_image_template
     - .variables_clang11_config
-  before_script:
-    - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16`
-    - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16`
-    - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG
-    - echo -e "compiler=$compiler" >> dependencies.env
-    - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env
-  variables:
-    DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies
-    DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]'
-  artifacts:
-    reports:
-      dotenv: dependencies.env
 
 clang11_debug_build:
-  stage: build
+  needs: [clang11_spack_image]
   extends:
-    - .container-builder
+    - .build_template
     - .variables_clang11_config
-    - .build_variables_common
-    - .cmake_variables_common
-  needs:
-    - clang11_spack_image
-  before_script:
-    - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16`
-    - build_type=`echo $BUILD_TYPE | tr '[:upper:]' '[:lower:]'`
-    - configuration=$DOCKERFILE_SHA-$BASE_IMAGE-$BUILD_DIR-$CMAKE_COMMON_FLAGS-$CMAKE_FLAGS-$CI_COMMIT_SHORT_SHA-$SOURCE_DIR
-    - configuration=${configuration//-D/}
-    - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16`
-    - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG
-    - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env
-  artifacts:
-    reports:
-      dotenv: build.env
 
 clang11_debug_test:
+  needs: [clang11_debug_build]
   extends:
-    - .variables_clang11_config
+    - .test_template
     - .test_common_mc
-    - .cmake_variables_common
-  needs:
-    - clang11_debug_build
-  script:
-    - spack arch
-    - export CTEST_XML=$PWD/ctest.xml
-    - trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT
-    - spack -e pika_ci build-env $spack_spec -- bash -c "ctest --output-junit ${CTEST_XML} --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error"
-  image: $PERSIST_IMAGE_NAME
+    - .variables_clang11_config
diff --git a/.gitlab/includes/clang12_pipeline.yml b/.gitlab/includes/clang12_pipeline.yml
@@ -25,76 +25,25 @@ include:
                   -DPIKA_WITH_THREAD_STEALING_COUNTS=ON"
 
 clang12_spack_compiler_image:
-  stage: spack_compiler
-  needs: [base_spack_image]
   extends:
-    - .container-builder
+    - .compiler_image_template
     - .variables_clang12_config
-  before_script:
-    - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16`
-    - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16`
-    - compiler=${COMPILER/@/-}
-    - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG
-    - echo -e "compiler=$compiler" >> compiler.env
-    - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env
-  variables:
-    DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler
-    DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]'
-  artifacts:
-    reports:
-      dotenv: compiler.env
 
 clang12_spack_image:
-  stage: spack_dependencies
   needs: [clang12_spack_compiler_image]
   extends:
-    - .container-builder
+    - .dependencies_image_template
     - .variables_clang12_config
-  before_script:
-    - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16`
-    - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16`
-    - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG
-    - echo -e "compiler=$compiler" >> dependencies.env
-    - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env
-  variables:
-    DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies
-    DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]'
-  artifacts:
-    reports:
-      dotenv: dependencies.env
-
 
 clang12_debug_build:
-  stage: build
+  needs: [clang12_spack_image]
   extends:
-    - .container-builder
+    - .build_template
     - .variables_clang12_config
-    - .build_variables_common
-    - .cmake_variables_common
-  needs:
-    - clang12_spack_image
-  before_script:
-    - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16`
-    - build_type=`echo $BUILD_TYPE | tr '[:upper:]' '[:lower:]'`
-    - configuration=$DOCKERFILE_SHA-$BASE_IMAGE-$BUILD_DIR-$CMAKE_COMMON_FLAGS-$CMAKE_FLAGS-$CI_COMMIT_SHORT_SHA-$SOURCE_DIR
-    - configuration=${configuration//-D/}
-    - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16`
-    - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG
-    - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env
-  artifacts:
-    reports:
-      dotenv: build.env
 
 clang12_debug_test:
+  needs: [clang12_debug_build]
   extends:
-    - .variables_clang12_config
+    - .test_template
     - .test_common_mc
-    - .cmake_variables_common
-  needs:
-    - clang12_debug_build
-  script:
-    - spack arch
-    - export CTEST_XML=$PWD/ctest.xml
-    - trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT
-    - spack -e pika_ci build-env $spack_spec -- bash -c "ctest --output-junit ${CTEST_XML} --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error"
-  image: $PERSIST_IMAGE_NAME
+    - .variables_clang12_config
diff --git a/.gitlab/includes/clang13_pipeline.yml b/.gitlab/includes/clang13_pipeline.yml
@@ -22,75 +22,25 @@ include:
                   -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON -DPIKA_WITH_STDEXEC=ON"
 
 clang13_spack_compiler_image:
-  stage: spack_compiler
-  needs: [base_spack_image]
   extends:
-    - .container-builder
+    - .compiler_image_template
     - .variables_clang13_config
-  before_script:
-    - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16`
-    - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16`
-    - compiler=${COMPILER/@/-}
-    - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG
-    - echo -e "compiler=$compiler" >> compiler.env
-    - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env
-  variables:
-    DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler
-    DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]'
-  artifacts:
-    reports:
-      dotenv: compiler.env
 
 clang13_spack_image:
-  stage: spack_dependencies
   needs: [clang13_spack_compiler_image]
   extends:
-    - .container-builder
+    - .dependencies_image_template
     - .variables_clang13_config
-  before_script:
-    - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16`
-    - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16`
-    - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG
-    - echo -e "compiler=$compiler" >> dependencies.env
-    - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env
-  variables:
-    DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies
-    DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]'
-  artifacts:
-    reports:
-      dotenv: dependencies.env
 
 clang13_debug_build:
-  stage: build
+  needs: [clang13_spack_image]
   extends:
-    - .container-builder
+    - .build_template
     - .variables_clang13_config
-    - .build_variables_common
-    - .cmake_variables_common
-  needs:
-    - clang13_spack_image
-  before_script:
-    - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16`
-    - build_type=`echo $BUILD_TYPE | tr '[:upper:]' '[:lower:]'`
-    - configuration=$DOCKERFILE_SHA-$BASE_IMAGE-$BUILD_DIR-$CMAKE_COMMON_FLAGS-$CMAKE_FLAGS-$CI_COMMIT_SHORT_SHA-$SOURCE_DIR
-    - configuration=${configuration//-D/}
-    - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16`
-    - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG
-    - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env
-  artifacts:
-    reports:
-      dotenv: build.env
 
 clang13_debug_test:
+  needs: [clang13_debug_build]
   extends:
-    - .variables_clang13_config
+    - .test_template
     - .test_common_mc
-    - .cmake_variables_common
-  needs:
-    - clang13_debug_build
-  script:
-    - spack arch
-    - export CTEST_XML=$PWD/ctest.xml
-    - trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT
-    - spack -e pika_ci build-env $spack_spec -- bash -c "ctest --output-junit ${CTEST_XML} --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error"
-  image: $PERSIST_IMAGE_NAME
+    - .variables_clang13_config
diff --git a/.gitlab/includes/clang14_cuda11_pipeline.yml b/.gitlab/includes/clang14_cuda11_pipeline.yml
@@ -23,76 +23,25 @@ include:
                   -DCMAKE_CUDA_ARCHITECTURES=80 -DPIKA_WITH_CXX_LAMBDA_CAPTURE_DECLTYPE=OFF"
 
 clang14_cuda11_spack_compiler_image:
-  stage: spack_compiler
-  timeout: 2 hours
-  needs: [base_spack_image]
   extends:
-    - .container-builder
+    - .compiler_image_template
     - .variables_clang14_cuda11_config
-  before_script:
-    - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16`
-    - CONFIG_TAG=`echo $DOCKERFILE_SHA-$ARCH-$BASE_IMAGE-$COMPILER | sha256sum - | head -c 16`
-    - compiler=${COMPILER/@/-}
-    - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler:$CONFIG_TAG
-    - echo -e "compiler=$compiler" >> compiler.env
-    - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> compiler.env
-  variables:
-    DOCKERFILE: .gitlab/docker/Dockerfile.spack_compiler
-    DOCKER_BUILD_ARGS: '["BASE_IMAGE","ARCH","COMPILER"]'
-  artifacts:
-    reports:
-      dotenv: compiler.env
 
 clang14_cuda11_spack_image:
-  stage: spack_dependencies
   needs: [clang14_cuda11_spack_compiler_image]
   extends:
-    - .container-builder
+    - .dependencies_image_template
     - .variables_clang14_cuda11_config
-  before_script:
-    - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16`
-    - CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SPEC | sha256sum - | head -c 16`
-    - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/base/pika-$compiler-dependencies:$CONFIG_TAG
-    - echo -e "compiler=$compiler" >> dependencies.env
-    - echo -e "BASE_IMAGE=$PERSIST_IMAGE_NAME" >> dependencies.env
-  variables:
-    DOCKERFILE: .gitlab/docker/Dockerfile.spack_dependencies
-    DOCKER_BUILD_ARGS: '["BASE_IMAGE","SPACK_SPEC"]'
-  artifacts:
-    reports:
-      dotenv: dependencies.env
 
 clang14_cuda11_debug_build:
-  stage: build
+  needs: [clang14_cuda11_spack_image]
   extends:
-    - .container-builder
+    - .build_template
     - .variables_clang14_cuda11_config
-    - .build_variables_common
-    - .cmake_variables_common
-  needs:
-    - clang14_cuda11_spack_image
-  before_script:
-    - export DOCKERFILE_SHA=`sha256sum $DOCKERFILE | head -c 16`
-    - build_type=`echo $BUILD_TYPE | tr '[:upper:]' '[:lower:]'`
-    - configuration=$DOCKERFILE_SHA-$BASE_IMAGE-$BUILD_DIR-$CMAKE_COMMON_FLAGS-$CMAKE_FLAGS-$CI_COMMIT_SHORT_SHA-$SOURCE_DIR
-    - configuration=${configuration//-D/}
-    - CONFIG_TAG=`echo $configuration | sha256sum - | head -c 16`
-    - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/pika-$compiler-$build_type-build:$CONFIG_TAG
-    - echo -e "PERSIST_IMAGE_NAME=$PERSIST_IMAGE_NAME" >> build.env
-  artifacts:
-    reports:
-      dotenv: build.env
 
 clang14_cuda11_debug_test:
+  needs: [clang14_cuda11_debug_build]
   extends:
-    - .variables_clang14_cuda11_config
+    - .test_template
     - .test_common_gpu_clariden_cuda
-    - .cmake_variables_common
-  needs:
-    - clang14_cuda11_debug_build
-  script:
-    - spack arch
-    - export CTEST_XML=$PWD/ctest.xml
-    - trap "${SOURCE_DIR}/.gitlab/scripts/collect_ctest_metrics.sh ${CTEST_XML}" EXIT
-    - spack -e pika_ci build-env $spack_spec -- bash -c "ctest --output-junit ${CTEST_XML} --label-exclude COMPILE_ONLY --test-dir ${BUILD_DIR} -j$(($(nproc)/2)) --timeout 120 --output-on-failure --no-compress-output --no-tests=error"
-  image: $PERSIST_IMAGE_NAME
+    - .variables_clang14_cuda11_config