Merge pull request #56 from ecmwf-ifs/nabr-nvidia-ci

Switch to CTest for CI and add NVHPC 23.5 to CI testing

.github/scripts/bootstrap-nvhpc.sh

@@ -2,7 +2,8 @@
 set -euo pipefail
 set -x
 
-nvhpc_version=21.9
+# Set nvhpc version to default value if unset
+: "${nvhpc_version:=21.9}"
 
 # Use Atlas' nvhpc installation script
 wget https://raw.githubusercontent.com/ecmwf/atlas/develop/tools/install-nvhpc.sh

.github/scripts/verify-targets.sh

@@ -15,25 +15,25 @@ then
   targets+=(dwarf-cloudsc-c)
 fi
 
-if [[ "$gpu_flag" == "--with-gpu" ]]
+if [[ "$build_flags" == *"--with-gpu"* ]]
 then
-  targets+=(dwarf-cloudsc-gpu-scc dwarf-cloudsc-gpu-scc-hoist dwarf-cloudsc-gpu-scc-k-caching) 
+  targets+=(dwarf-cloudsc-gpu-scc dwarf-cloudsc-gpu-scc-hoist dwarf-cloudsc-gpu-scc-k-caching)
   targets+=(dwarf-cloudsc-gpu-omp-scc-hoist)
-  if [[ "$claw_flag" == "--with-claw" ]]
+  if [[ "$build_flags" == *"--with-claw"* ]]
   then
     targets+=(dwarf-cloudsc-gpu-claw)
   fi
-  if [[ "$cuda_flag" == "--with-cuda" ]]
+  if [[ "$build_flags" == *"--with-cuda"* ]]
   then
     targets+=(dwarf-cloudsc-gpu-scc-cuf dwarf-cloudsc-gpu-scc-cuf-k-caching)
-  fi
-  if [[ "$cuda_flag" == "--with-cuda" && "$io_library_flag" == "--with-serialbox" ]]
-  then
-      targets+=(dwarf-cloudsc-cuda dwarf-cloudsc-cuda-hoist dwarf-cloudsc-cuda-k-caching)
+    if [[ "$io_library_flag" == "--with-serialbox" ]]
+    then
+        targets+=(dwarf-cloudsc-cuda dwarf-cloudsc-cuda-hoist dwarf-cloudsc-cuda-k-caching)
+    fi
   fi
 fi
 
-if [[ "$loki_flag" == "--with-loki" ]]
+if [[ "$build_flags" == *"--with-loki"* ]]
 then
   targets+=(dwarf-cloudsc-loki-idem dwarf-cloudsc-loki-sca)
   targets+=(dwarf-cloudsc-loki-scc dwarf-cloudsc-loki-scc-hoist)
@@ -42,22 +42,22 @@ then
   then
     targets+=(dwarf-cloudsc-loki-c)
   fi
-  if [[ "$claw_flag" == "--with-claw" ]]
+  if [[ "$build_flags" == *"--with-claw"* ]]
   then
     targets+=(dwarf-cloudsc-loki-claw-cpu dwarf-cloudsc-loki-claw-gpu)
   fi
-  if [[ "$cuda_flag" == "--with-cuda" ]]
+  if [[ "$build_flags" == *"--with-cuda"* ]]
   then
     targets+=(dwarf-cloudsc-loki-scc-cuf-hoist dwarf-cloudsc-loki-scc-cuf-parametrise)
   fi
 fi
 
-if [[ "$pyiface_flag" == "--cloudsc-fortran-pyiface=ON" ]]
+if [[ "$build_flags" == *"--cloudsc-fortran-pyiface=ON"* ]]
 then
   targets+=(cloudsc_pyiface.py)
 fi
 
-if [[ "$python_f2py_flag" == "--cloudsc-python-f2py=ON" ]]
+if [[ "$build_flags" == *"--cloudsc-python-f2py=ON"* ]]
 then
   targets+=(cloudsc_f2py.py)
 fi

.github/workflows/build.yml

@@ -17,7 +17,7 @@ on:
 jobs:
   # This workflow contains a single job called "build"
   build:
-    name: Test on ${{ matrix.arch }} ${{ matrix.io_library_flag }} ${{ matrix.mpi_flag }} ${{ matrix.prec_flag }} ${{ matrix.gpu_flag }} ${{ matrix.loki_flag }} ${{ matrix.claw_flag }} ${{ matrix.pyiface_flag }} ${{ matrix.python_f2py_flag }}
+    name: Test on ${{ matrix.arch }} ${{ matrix.io_library_flag }} ${{ matrix.prec_flag }} ${{ matrix.build_flags }}
 
     # The type of runner that the job will run on
     runs-on: ubuntu-20.04
@@ -27,57 +27,54 @@ jobs:
 
       matrix:
 
-        arch:
-          - github/ubuntu/gnu/9.4.0
+        arch: ['gnu/9.4.0']  # Default arch on Github is GNU 9.4.0 for now
 
         io_library_flag: ['', '--with-serialbox']  # Switch between Serialbox and HDF5
 
-        mpi_flag: ['', '--with-mpi']  # Enable MPI-parallel build
-
         prec_flag: ['', '--single-precision']  # Switch single/double precision
 
-        gpu_flag: ['', '--with-gpu']  # GPU-variants enabled
-
-        cuda_flag: ['']  # Enable CUDA variants
+        build_flags:
+          - ''  # Plain build without any options
+          - '--with-gpu --with-loki' # Enable Loki and GPU variants
+          - '--with-gpu --with-loki --with-mpi' # Enable Loki and GPU variants with MPI
 
-        loki_flag: ['', '--with-loki']  # Loki source-to-source translation enabled
+        pyiface_flag: ['']  # Enable the pyiface variant
 
-        claw_flag: ['']  # Flag to enable CLAW-generated variants
+        python_f2py_flag: ['']  # Enable the f2py variant
 
-        pyiface_flag: ['']  # Flag to enable Python-interface variant
+        claw_flag: ['']
 
-        python_f2py_flag: ['']  # Flag to enable Python variants
+        ctest_exclude_pattern: ['-scc-hoist-']  # Regex to disable CTest tests
 
         include:
-          # Add nvhpc build configurations with serialbox and HDF5
-          - arch: github/ubuntu/nvhpc/21.9
+          # Add pyiface build configuration for double precision, non-MPI, HDF5 only
+          - arch: gnu/9.4.0
             io_library_flag: ''
-            mpi_flag: ''
             prec_flag: ''
-            gpu_flag: '--with-gpu'
-            cuda_flag: '--with-cuda'
-            loki_flag: '--with-loki'
-            pyiface_flag: ''
-            python_f2py_flag: ''
-          - arch: github/ubuntu/nvhpc/21.9
+            build_flags: '--cloudsc-fortran-pyiface=ON --cloudsc-python-f2py=ON'
+
+          # Add nvhpc build configurations with serialbox and HDF5
+          - arch: nvhpc/21.9
+            nvhpc_version: 21.9
+            io_library_flag: ''
+            build_flags: '--with-gpu --with-loki --with-cuda'
+            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda-'  # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE
+          - arch: nvhpc/21.9
+            nvhpc_version: 21.9
             io_library_flag: '--with-serialbox'
-            mpi_flag: ''
-            prec_flag: ''
-            gpu_flag: '--with-gpu'
-            cuda_flag: '--with-cuda'
-            loki_flag: '--with-loki'
-            pyiface_flag: ''
-            python_f2py_flag: ''
-          # Add pyiface build configuration for HDF5 only
-          - arch: github/ubuntu/gnu/9.4.0
+            build_flags: '--with-gpu --with-loki --with-cuda'
+            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-cuda-'  # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE
+
+          - arch: nvhpc/23.5
+            nvhpc_version: 23.5
             io_library_flag: ''
-            mpi_flag: ''
-            prec_flag: ''
-            gpu_flag: ''
-            cuda_flag: ''
-            loki_flag: ''
-            pyiface_flag: '--cloudsc-fortran-pyiface=ON'
-            python_f2py_flag: '--cloudsc-python-f2py=ON'
+            build_flags: '--with-gpu --with-loki --with-cuda'
+            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda-'  # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE
+          - arch: nvhpc/23.5
+            nvhpc_version: 23.5
+            io_library_flag: '--with-serialbox'
+            build_flags: '--with-gpu --with-loki --with-cuda'
+            ctest_exclude_pattern: '-gpu-|-scc-|-loki-c|-loki-sca|-cuda-'  # GPU variants don't work on CPU runners, loki-c variant causes SIGFPE
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
@@ -91,12 +88,32 @@ jobs:
 
       # Install MPI
       - name: Install MPI via Apt
-        if: contains( matrix.mpi_flag, 'with-mpi' )
+        if: contains( matrix.build_flags, 'with-mpi' )
         run: sudo apt-get install libopenmpi-dev
 
+      # Free up disk space for nvhpc
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        if: contains( matrix.arch, 'nvhpc' )
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: false
+
+          # all of these default to true, but feel free to set to
+          # "false" if necessary for your workflow
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: true
+          swap-storage: true
+
       # Install Compiler
       - name: Install nvhpc
         if: contains( matrix.arch, 'nvhpc' )
+        env:
+          nvhpc_version: ${{ matrix.nvhpc_version }}
         run: .github/scripts/bootstrap-nvhpc.sh
 
       # Install HDF5
@@ -106,7 +123,9 @@ jobs:
 
       - name: Install HDF5 from source
         if: contains( matrix.arch, 'nvhpc' ) && ! contains( matrix.io_library_flag, 'with-serialbox' )
-        run: source arch/${{ matrix.arch }}/env.sh && .github/scripts/install-hdf5.sh
+        run: |
+          source arch/github/ubuntu/${{ matrix.arch }}/env.sh
+          FC=pgf90 .github/scripts/install-hdf5.sh
 
       # Install Boost
       - name: Install Boost libraries
@@ -121,29 +140,29 @@ jobs:
       - name: Bundle build
         run: |
           ./cloudsc-bundle build --retry-verbose \
-          --arch=arch/${{ matrix.arch }} ${{ matrix.prec_flag }} \
-          ${{ matrix.mpi_flag }} ${{ matrix.io_library_flag }} ${{ matrix.gpu_flag }} \
-          ${{ matrix.claw_flag}} ${{ matrix.loki_flag }} ${{ matrix.cuda_flag }} \
-          ${{ matrix.pyiface_flag }} ${{ matrix.python_f2py_flag }}
+          --arch=arch/github/ubuntu/${{ matrix.arch }} \
+          ${{ matrix.prec_flag }} ${{ matrix.io_library_flag }} ${{ matrix.build_flags }}
 
       # Verify targets exist
       - name: Verify targets
         env:
           io_library_flag: ${{ matrix.io_library_flag }}
           prec_flag: ${{ matrix.prec_flag }}
-          gpu_flag: ${{ matrix.gpu_flag }}
-          cuda_flag: ${{ matrix.cuda_flag }}
-          loki_flag: ${{ matrix.loki_flag }}
-          claw_flag: ${{ matrix.claw_flag }}
-          pyiface_flag: ${{ matrix.pyiface_flag }}
-          python_f2py_flag: ${{ matrix.python_f2py_flag }}
+          build_flags: ${{ matrix.build_flags }}
         run: .github/scripts/verify-targets.sh
 
-      # Run double-precision targets
-      # (Mind the exclusions inside the script!)
-      - name: Run targets
-        env:
-          mpi_flag: ${{ matrix.mpi_flag }}
-          arch: ${{ matrix.arch }}
+      # Run ctest
+      - name: Run CTest
+        if: ${{ matrix.prec_flag == '' }}
+        working-directory: ./build
+        run: |
+          source env.sh
+          ctest -O ctest.log --output-on-failure -E "${{ matrix.ctest_exclude_pattern }}"
+
+      # Upload test output
+      - name: Archive CTest output
+        uses: actions/upload-artifact@v3
         if: ${{ matrix.prec_flag == '' }}
-        run: .github/scripts/run-targets.sh
+        with:
+          name: ctest-log
+          path: build/ctest.log

CMakeLists.txt

@@ -9,9 +9,6 @@
 # define minimum version of cmake required
 cmake_minimum_required( VERSION 3.17 FATAL_ERROR )
 
-# Disable warnings about setting `ENABLE_ACC` variable for ecbuild_add_option
-cmake_policy( SET CMP0077 NEW )
-
 find_package( ecbuild REQUIRED )
 
 # define the project
@@ -31,7 +28,7 @@ endif()
 include( cloudsc_compile_options )
 
 ### OpenACC
-if( NOT DEFINED ENABLE_ACC OR ENABLE_ACC )
+if( ${CMAKE_VERSION} VERSION_LESS "3.25" AND (NOT DEFINED ENABLE_ACC OR ENABLE_ACC ) )
     # Incredibly inconvenient: FindOpenACC does _not_ set OpenACC_FOUND, only
     # the language-specific components OpenACC_Fortran_FOUND and OpenACC_C_FOUND.
     # This means, even internally CMake considers OpenACC as not found.
@@ -40,6 +37,7 @@ if( NOT DEFINED ENABLE_ACC OR ENABLE_ACC )
     # the result, and then, trigger a second find_package via ecbuild_add_option.
     # This then conveniently takes the previously set OpenACC_FOUND into account
     # and rectifies CMake's internal bookkeeping in the process.
+    # This has been fixed in CMake 3.25
     find_package( OpenACC )
     if( OpenACC_Fortran_FOUND AND OpenACC_C_FOUND )
         set( OpenACC_FOUND ON )

arch/github/ubuntu/nvhpc/21.9/env.sh

@@ -33,3 +33,6 @@ export CXX=pgc++
 export FC=pgf90
 
 export ECBUILD_TOOLCHAIN="./toolchain.cmake"
+
+# Increase stack size to maximum
+ulimit -S -s unlimited

arch/github/ubuntu/nvhpc/23.5/env.sh

@@ -0,0 +1,36 @@
+# (C) Copyright 1988- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+# Source me to get the correct configure/build/run environment
+
+### Variables
+export NVHPC_INSTALL_DIR=${GITHUB_WORKSPACE}/nvhpc-install
+export NVHPC_VERSION=23.5
+export NVHPC_DIR=${NVHPC_INSTALL_DIR}/Linux_x86_64/${NVHPC_VERSION}
+
+### Compilers
+export PATH=${NVHPC_DIR}/compilers/bin:${PATH}
+export NVHPC_LIBRARY_PATH=${NVHPC_DIR}/compilers/lib
+export LD_LIBRARY_PATH=${NVHPC_LIBRARY_PATH}
+
+### MPI
+export MPI_HOME=${NVHPC_DIR}/comm_libs/mpi
+export PATH=${MPI_HOME}/bin:${PATH}
+
+### HDF5
+export HDF5_DIR=${GITHUB_WORKSPACE}/hdf5-install
+export LD_LIBRARY_PATH=${HDF5_DIR}/lib:${LD_LIBRARY_PATH}
+export PATH=${HDF5_DIR}/bin:${PATH}
+
+### Compiler variables
+export CC=nvc
+export CXX=nvc++
+export FC=nvfortran
+
+# Increase stack size to maximum
+ulimit -S -s unlimited