From 370d724364474f28e678fa4d8b9d57376f6fefb7 Mon Sep 17 00:00:00 2001 From: Edward Hartnett <38856240+edwardhartnett@users.noreply.github.com> Date: Wed, 14 Dec 2022 05:50:27 -0700 Subject: [PATCH] Add TEST_FILE_DIR option to CMake build, where test data files can be found instead of using FTP. (#732) Add handling for the TEST_FILE_DIR option in CMake. This allows the programmer to specify a local directory to check first, before downloading test data files from the FTP site. Use this capability, and a cache, to cache the data for all CI runs as well. This prevents the CI system from downloading these test files hundreds of times per day. Part of #719. Fixes #728 --- .../workflows/debug-docs-test_coverage.yml | 9 ++++++- .github/workflows/developer.yml | 26 ++++++++++++++++--- .github/workflows/intel.yml | 24 +++++++++++++++-- .github/workflows/linux-mac-nceplibs-mpi.yml | 22 +++++++++++++++- .github/workflows/netcdf-versions.yml | 23 +++++++++++++++- CMakeLists.txt | 2 ++ tests/CMakeLists.txt | 24 ++++++++++++----- 7 files changed, 114 insertions(+), 16 deletions(-) diff --git a/.github/workflows/debug-docs-test_coverage.yml b/.github/workflows/debug-docs-test_coverage.yml index be1371f69..65dc9eaed 100644 --- a/.github/workflows/debug-docs-test_coverage.yml +++ b/.github/workflows/debug-docs-test_coverage.yml @@ -100,6 +100,13 @@ jobs: path: ufs_utils submodules: recursive + - name: cache-data + id: cache-data + uses: actions/cache@v2 + with: + path: ~/data + key: data-1 + - name: build run: | export ESMFMKFILE=~/esmf/lib/esmf.mk @@ -109,7 +116,7 @@ jobs: export CXX=mpicxx export FC=mpifort export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:~/jasper/lib;~/jasper/lib64" - cmake .. -DCMAKE_PREFIX_PATH='~/jasper;~/nceplibs' -DCMAKE_BUILD_TYPE=Debug -DENABLE_DOCS=On -DCMAKE_Fortran_FLAGS="-g -fprofile-arcs -ftest-coverage -O0" + cmake -DTEST_FILE_DIR=/home/runner/data -DCMAKE_PREFIX_PATH='~/jasper;~/nceplibs' -DCMAKE_BUILD_TYPE=Debug -DENABLE_DOCS=On .. make -j2 - name: test diff --git a/.github/workflows/developer.yml b/.github/workflows/developer.yml index 4401171f6..2b7ad929e 100644 --- a/.github/workflows/developer.yml +++ b/.github/workflows/developer.yml @@ -1,7 +1,7 @@ # UFS_UTILS test workflow. # -# This workflow tests all developer options - address santizer, -# warning check, documentation check, and test code coverage. +# This workflow tests all developer options including +# documentation check, and test code coverage. # # Ed Hartnett 12/11/22 name: developer @@ -326,15 +326,21 @@ jobs: path: ufs_utils submodules: true + - name: cache-data + id: cache-data + uses: actions/cache@v2 + with: + path: ~/data + key: data-1 + - name: build run: | - set -x cd ufs_utils mkdir build doxygen --version export ESMFMKFILE=~/esmf/lib/esmf.mk cd build - cmake .. -DFTP_TEST_FILES=ON -DTEST_FILE_DIR=/home/runner/data -DENABLE_DOCS=On -DJasper_ROOT=~/jasper -DCMAKE_PREFIX_PATH="~/g2c;~/bacio;~/g2;~/w3nco;~/sfcio;~/sigio;~/nemsio;~/sp;~/ip" -DCMAKE_Fortran_FLAGS="-g -fprofile-arcs -ftest-coverage -O0" -DCMAKE_C_FLAGS="-g -fprofile-arcs -ftest-coverage -O0" -DCMAKE_BUILD_TYPE=Debug + cmake -DTEST_FILE_DIR=/home/runner/data -DENABLE_DOCS=On -DCMAKE_PREFIX_PATH="~/jasper;~/g2c;~/bacio;~/g2;~/w3nco;~/sfcio;~/sigio;~/nemsio;~/sp;~/ip" -DCMAKE_Fortran_FLAGS="-g -fprofile-arcs -ftest-coverage -O0" -DCMAKE_C_FLAGS="-g -fprofile-arcs -ftest-coverage -O0" -DCMAKE_BUILD_TYPE=Debug .. make -j2 VERBOSE=1 - name: test @@ -343,6 +349,18 @@ jobs: ctest --verbose --rerun-failed --output-on-failure gcovr --root .. -v --html-details --exclude ../tests --exclude CMakeFiles --print-summary -o test-coverage.html &> /dev/null + - name: cache-data + if: steps.cache-data.outputs.cache-hit != 'true' + run: | + mkdir ~/data + cp ufs_utils/build/tests/chgres_cube/data/* ~/data + cp ufs_utils/build/tests/sfc_climo_gen/data/* ~/data + cp ufs_utils/build/tests/cpld_gridgen/data/* ~/data + cp ufs_utils/tests/filter_topo/data/* ~/data + cp ufs_utils/tests/emcsfc_snow2mdl/data/* ~/data + cp ufs_utils/tests/chgres_cube/data/* ~/data + ls -l ~/data + - name: upload-test-coverage uses: actions/upload-artifact@v2 with: diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index 200a91f03..e7f7c4388 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -12,7 +12,7 @@ on: - README.md # Use custom shell with -l so .bash_profile is sourced which loads intel/oneapi/setvars.sh -# without having to do it in manually every step +# without having to do it in manually every step. defaults: run: shell: bash -leo pipefail {0} @@ -166,16 +166,36 @@ jobs: path: ufs_utils submodules: recursive + - name: cache-data + id: cache-data + uses: actions/cache@v2 + with: + path: ~/data + key: data-1 + - name: build run: | export ESMFMKFILE=~/esmf/lib/esmf.mk cd ufs_utils mkdir build && cd build export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:~/jasper/lib;~/jasper/lib64" - cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH='~;~/jasper;~/nceplibs;~/netcdf' + cmake -DTEST_FILE_DIR=/home/runner/data -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH='~;~/jasper;~/nceplibs;~/netcdf' .. make -j2 - name: test run: | cd ufs_utils/build ctest --rerun-failed --output-on-failure + + - name: cache-data + if: steps.cache-data.outputs.cache-hit != 'true' + run: | + mkdir ~/data + cp ufs_utils/build/tests/chgres_cube/data/* ~/data + cp ufs_utils/build/tests/sfc_climo_gen/data/* ~/data + cp ufs_utils/build/tests/cpld_gridgen/data/* ~/data + cp ufs_utils/tests/filter_topo/data/* ~/data + cp ufs_utils/tests/emcsfc_snow2mdl/data/* ~/data + cp ufs_utils/tests/chgres_cube/data/* ~/data + ls -l ~/data + diff --git a/.github/workflows/linux-mac-nceplibs-mpi.yml b/.github/workflows/linux-mac-nceplibs-mpi.yml index 8b7aab007..bb01b29ee 100644 --- a/.github/workflows/linux-mac-nceplibs-mpi.yml +++ b/.github/workflows/linux-mac-nceplibs-mpi.yml @@ -217,6 +217,13 @@ jobs: path: ufs_utils submodules: recursive + - name: cache-data + id: cache-data + uses: actions/cache@v2 + with: + path: ~/data + key: data-2 + - name: build run: | export ESMFMKFILE=~/esmf/lib/esmf.mk @@ -227,7 +234,7 @@ jobs: export FC=mpifort export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:~/jasper/lib;~/jasper/lib64" export DYLD_LIBRARY_PATH="${LD_LIBRARY_PATH}:~/jasper/lib;~/jasper/lib64" - cmake .. -DCMAKE_PREFIX_PATH='~/jasper;~/nceplibs;~/netcdf' + cmake -DTEST_FILE_DIR=/home/runner/data -DCMAKE_PREFIX_PATH='~/jasper;~/nceplibs;~/netcdf' .. make -j2 - name: test @@ -236,3 +243,16 @@ jobs: # Oversubscribe for OpenMPI to run more processes than CPUs export OMPI_MCA_rmaps_base_oversubscribe=1 ctest --rerun-failed --output-on-failure + + - name: cache-data + if: steps.cache-data.outputs.cache-hit != 'true' + run: | + mkdir ~/data + cp ufs_utils/build/tests/chgres_cube/data/* ~/data + cp ufs_utils/build/tests/sfc_climo_gen/data/* ~/data + cp ufs_utils/build/tests/cpld_gridgen/data/* ~/data + cp ufs_utils/tests/filter_topo/data/* ~/data + cp ufs_utils/tests/emcsfc_snow2mdl/data/* ~/data + cp ufs_utils/tests/chgres_cube/data/* ~/data + ls -l ~/data + diff --git a/.github/workflows/netcdf-versions.yml b/.github/workflows/netcdf-versions.yml index 424bff3ca..ca02e5e26 100644 --- a/.github/workflows/netcdf-versions.yml +++ b/.github/workflows/netcdf-versions.yml @@ -151,6 +151,13 @@ jobs: path: ufs_utils submodules: recursive + - name: cache-data + id: cache-data + uses: actions/cache@v2 + with: + path: ~/data + key: data-1 + - name: build run: | export ESMFMKFILE=~/esmf/lib/esmf.mk @@ -160,7 +167,7 @@ jobs: export CXX=mpicxx export FC=mpifort export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:~/jasper/lib;~/jasper/lib64" - cmake .. -DCMAKE_PREFIX_PATH='~/jasper;~/nceplibs;~/netcdf' -DCMAKE_BUILD_TYPE=Debug + cmake -DTEST_FILE_DIR=/home/runner/work/UFS_UTILS/UFS_UTILS/data -DCMAKE_PREFIX_PATH='~/jasper;~/nceplibs;~/netcdf' -DCMAKE_BUILD_TYPE=Debug .. make -j2 - name: test @@ -168,3 +175,17 @@ jobs: cd ufs_utils/build # export LSAN_OPTIONS=suppressions=LSanSuppress.supp ctest --rerun-failed --output-on-failure + + - name: cache-data + if: steps.cache-data.outputs.cache-hit != 'true' + run: | + mkdir ~/data + cp ufs_utils/build/tests/chgres_cube/data/* ~/data + cp ufs_utils/build/tests/sfc_climo_gen/data/* ~/data + cp ufs_utils/build/tests/cpld_gridgen/data/* ~/data + cp ufs_utils/tests/filter_topo/data/* ~/data + cp ufs_utils/tests/emcsfc_snow2mdl/data/* ~/data + cp ufs_utils/tests/chgres_cube/data/* ~/data + ls -l ~/data + + diff --git a/CMakeLists.txt b/CMakeLists.txt index 76e92900e..f8e33a472 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,9 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") # User options. option(OPENMP "use OpenMP threading" ON) option(ENABLE_DOCS "Enable generation of doxygen-based documentation." OFF) +SET(TEST_FILE_DIR "." CACHE STRING "Check this directory for test files before using FTP.") +# Set the build type. if(NOT CMAKE_BUILD_TYPE MATCHES "^(Debug|Release|RelWithDebInfo|MinSizeRel)$") message(STATUS "Setting build type to 'Release' as none was specified.") set(CMAKE_BUILD_TYPE diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 54550d864..b76bc1477 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -3,19 +3,29 @@ # # Ed Hartnett 2/11/21 -# This function is used to download unit test data. -# It takes two arguments, the URL and the file to -# be downloaded. - +# Some test files are large and are kept on the NOAA EMC FTP +# site. This function is used to download such test data. It takes two +# arguments, the URL and the file to be downloaded. function(PULL_DATA THE_URL THE_FILE) + # If the TEST_FILE_DIR was specified, look for our test data files + # there before FTPing them. Developers can keep all test files on + # their machines, and save the time of downloading them every time. + message(STATUS "TEST_FILE_DIR ${TEST_FILE_DIR} THE_FILE ${THE_FILE}") + if(NOT ${TEST_FILE_DIR} STREQUAL ".") + if (EXISTS ${TEST_FILE_DIR}/${THE_FILE}) + message(STATUS "Copying file ${TEST_FILE_DIR}/${THE_FILE} to test data directory.") + FILE(COPY ${TEST_FILE_DIR}/${THE_FILE} + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/data) + endif() + endif() if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/data/${THE_FILE}") + message(STATUS "Downloading file ${CMAKE_CURRENT_BINARY_DIR}/data/${THE_FILE}.") file(DOWNLOAD ${THE_URL}/${THE_FILE} ${CMAKE_CURRENT_BINARY_DIR}/data/${THE_FILE} - SHOW_PROGRESS STATUS status - INACTIVITY_TIMEOUT 120 - ) + INACTIVITY_TIMEOUT 30 + ) list(GET status 0 status_num) if(NOT status_num EQUAL 0 OR NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/data/${THE_FILE}") message(FATAL_ERROR "Could not download ${THE_FILE}")