Skip to content

Commit

Permalink
Merge branch 'develop' into feature/cycle_frac2
Browse files Browse the repository at this point in the history
  • Loading branch information
GeorgeGayno-NOAA committed Jan 3, 2024
2 parents ed8c24d + 9bba725 commit ee79c49
Show file tree
Hide file tree
Showing 52 changed files with 4,669 additions and 341 deletions.
2 changes: 1 addition & 1 deletion .github/PULL_REQUEST_TEMPLATE
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ One or more paragraphs describing the problem, solution, and required changes.
## TESTS CONDUCTED:
If there are changes to the build or source code, the tests below must be conducted. Contact a repository manager if you need assistance.

- [ ] Compile branch on all Tier 1 machines using Intel (Orion, Jet, Hera and WCOSS2).
- [ ] Compile branch on all Tier 1 machines using Intel (Orion, Jet, Hera, Hercules and WCOSS2).
- [ ] Compile branch on Hera using GNU.
- [ ] Compile branch in 'Debug' mode on WCOSS2.
- [ ] Run unit tests locally on any Tier 1 machine.
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/ubuntu_intel.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,14 @@ jobs:
sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
sudo apt-get update
sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-mpi-devel intel-oneapi-openmp intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic
sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-mpi-devel intel-oneapi-openmp intel-oneapi-compiler-fortran-2023.2.1 intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.1
echo "source /opt/intel/oneapi/setvars.sh" >> ~/.bash_profile
# Install dependencies using Spack
- name: install-dependencies-with-spack
if: steps.cache-env.outputs.cache-hit != 'true'
run: |
git clone -c feature.manyFiles=true https://github.com/NOAA-EMC/spack.git
git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git
source spack/share/spack/setup-env.sh
sed "s/\[intel, gcc@10:10, apple-clang@14\]/\[intel\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
spack env create ufs_utils-env spack_ci.yaml
Expand Down
2 changes: 2 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ option(FVCOMTOOLS "Enable building fvcom_tools.fd" ON)
option(GBLEVENTS "Enable building gblevents.fd" ON)
option(CPLD_GRIDGEN "Enable building cpld_gridgen.fd" ON)
option(WEIGHT_GEN "Enable building weight_gen.fd" ON)
option(OCEAN_MERGE "Enable building ocean_merge.fd" ON)

# Option to build application specific utilities
option(GFS "Enable building GFS-only utilities" OFF)
Expand All @@ -46,6 +47,7 @@ if(GFS)
set(GBLEVENTS OFF CACHE BOOL "Disable building gblevents.fd" FORCE)
set(CPLD_GRIDGEN OFF CACHE BOOL "Disable building cpld_gridgen.fd" FORCE)
set(WEIGHT_GEN OFF CACHE BOOL "Disable building weight_gen.fd" FORCE)
set(OCEAN_MERGE OFF CACHE BOOL "Disable building ocean_merge.fd" FORCE)
endif()

SET(TEST_FILE_DIR "." CACHE STRING "Check this directory for test files before using FTP.")
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ It also uses the following repositories:

## Installing

On Orion, Jet, Hera and WCOSS2 do the following:
On Orion, Hercules, Jet, Hera and WCOSS2 do the following:

1) Set the 'fixed' directories using the `link_fixdirs.sh`
script in `./fix`. Usage: `./link_fixdirs.sh $RUN_ENVIR $machine`,
Expand Down
4 changes: 2 additions & 2 deletions ci/spack.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ spack:
- g2@3.4.5
- ip@4.4.0 precision=d
- nemsio@2.5.4
- sp@2.3.3
- sp@2.5.0
- w3emc@2.10.0
- sfcio@1.4.1
- sigio@2.3.2
Expand All @@ -19,4 +19,4 @@ spack:
- esmf@8.4.2~debug~xerces+external-parallelio
view: true
concretizer:
unify: true
unify: when_possible
11 changes: 11 additions & 0 deletions cmake/LibMPI.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,17 @@ function (platform_name RETURN_VARIABLE)

set (${RETURN_VARIABLE} "orion" PARENT_SCOPE)

elseif (SITENAME MATCHES "^Hercules-login-1.HPC.MsState.Edu" OR
SITENAME MATCHES "^Hercules-login-2.HPC.MsState.Edu" OR
SITENAME MATCHES "^Hercules-login-3.HPC.MsState.Edu" OR
SITENAME MATCHES "^Hercules-login-4.HPC.MsState.Edu" OR
SITENAME MATCHES "^hercules-login-1.hpc.msstate.edu" OR
SITENAME MATCHES "^hercules-login-2.hpc.msstate.edu" OR
SITENAME MATCHES "^hercules-login-3.hpc.msstate.edu" OR
SITENAME MATCHES "^hercules-login-4.hps.msstate.edu")

set (${RETURN_VARIABLE} "hercules" PARENT_SCOPE)

elseif (SITENAME MATCHES "^cheyenne1.cheyenne.ucar.edu" OR
SITENAME MATCHES "^cheyenne1.cheyenne.ucar.edu" OR
SITENAME MATCHES "^cheyenne2.cheyenne.ucar.edu" OR
Expand Down
15 changes: 15 additions & 0 deletions cmake/mpiexec.hercules
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/bin/bash
#
# Arguments:
#
# $1 - Number of MPI Tasks
# $2+ - Executable and its arguments
#

ACCOUNT=
QOS=debug

NP=$1
shift

srun -A $ACCOUNT -q $QOS -n $NP $@
20 changes: 12 additions & 8 deletions docs/main.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ https://github.com/ufs-community/UFS_UTILS.
- <a href="chgres_cube/index.html">chgres_cube</a> - Creates cold
start initial conditions for FV3 model runs.

- <a href="cpld_gridgen/index.html">cpld_gridgen</a> - Utility to
create the Fix and IC files for the S2SW and S2S applications

- <a href="emcsfc_ice_blend/index.html">emcsfc_ice_blend</a> - Blends
National Ice Center sea ice cover and EMC sea ice concentration data
to create a global sea ice analysis used to update the GFS once per
Expand Down Expand Up @@ -57,6 +60,15 @@ https://github.com/ufs-community/UFS_UTILS.
gnomonic grids, and to compute the equivalent global resolution of a
regional grid.

- <a href="lsm_routines/index.html">lsm_routines</a> - Land surface
model-specific routines that are utilised elsewhere within UFS_UTILS.
Currently, contains the routines required by global_cycle to
perform data assimilation updates to land model states

- <a href="ocean_merge/index.html">ocean_merge</a> - Utility to
determine the water mask by merging a lake mask with a mapped
ocean mask from MOM6.

- <a href="orog_mask_tools/index.html">orog_mask_tools</a> - Utilities
to create land mask, terrain and gravity wave drag fields; set lake
fraction and depth; creates an inland land mask.
Expand All @@ -70,14 +82,6 @@ https://github.com/ufs-community/UFS_UTILS.
top and the number of vertical levels. Outputs the 'ak' and 'bk'
parameters used by the forecast model to define the hybrid levels.

- <a href="lsm_routines/index.html">lsm_routines</a> - Land surface
model-specific routines that are utilised elsewhere within UFS_UTILS.
Currently, contains the routines required by global_cycle to
perform data assimilation updates to land model states

- <a href="cpld_gridgen/index.html">cpld_gridgen</a> - Utility to
create the Fix and IC files for the S2SW and S2S applications

- <a href="weight_gen/index.html">weight_gen</a> - Utility to
create gaussian grid ESMF 'scrip' files for use in creating
ESMF interpolation weight files.
10 changes: 6 additions & 4 deletions driver_scripts/driver_grid.hera.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@
#SBATCH --open-mode=truncate
#SBATCH -o log.fv3_grid_driver
#SBATCH -e log.fv3_grid_driver
#SBATCH --nodes=1 --ntasks-per-node=24
#SBATCH --nodes=6 --ntasks-per-node=12
#SBATCH --partition=bigmem
#SBATCH -q debug
#SBATCH -t 00:20:00
#SBATCH -t 00:30:00

#-----------------------------------------------------------------------
# Driver script to create a cubic-sphere based model grid on Hera.
Expand Down Expand Up @@ -103,7 +103,7 @@ export soil_type_src="bnu.v3.30s" # Soil type data.
# 3) "statsgo.conus.30s" for CONUS 30s data
# 4) "statsgo.nh.30s" for NH 30s data
# 5) "statsgo.30s" for global 30s data
# For Beijing Norm. Univ. data
# For Beijing Norm. Univ. data
# 1) "bnu.v3.30s" for global 30s data.

# choose dataset sources for lakefrac & lakedepth so that lake_data_srce=LakeFrac_LakeDepth;
Expand All @@ -115,6 +115,7 @@ if [ $gtype = uniform ]; then
export add_lake=true # Add lake frac and depth to orography data.
export lake_cutoff=0.50 # return 0 if lake_frac < lake_cutoff & add_lake=T
export binary_lake=1 # return 1 if lake_frac >= lake_cutoff & add_lake=T
export ocn=${ocn:-"025"} # use one of "025", "050", "100", "500". Cannot be empty
elif [ $gtype = stretch ]; then
export res=96
export stretch_fac=1.5 # Stretching factor for the grid
Expand Down Expand Up @@ -150,14 +151,15 @@ fi

#-----------------------------------------------------------------------
# Check paths.
#
# home_dir - location of repository.
# TEMP_DIR - working directory.
# out_dir - where files will be placed upon completion.
#-----------------------------------------------------------------------

export home_dir=$SLURM_SUBMIT_DIR/..
export TEMP_DIR=/scratch2/NCEPDEV/stmp1/$LOGNAME/fv3_grid.$gtype
export out_dir=/scratch2/NCEPDEV/stmp1/$LOGNAME/my_grids
export out_dir=/scratch2/NCEPDEV/stmp1/$LOGNAME/my_grids_ocean_fixed/

#-----------------------------------------------------------------------
# Should not need to change anything below here.
Expand Down
178 changes: 178 additions & 0 deletions driver_scripts/driver_grid.hercules.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
#!/bin/bash

#SBATCH -J fv3_grid_driver
#SBATCH -A fv3-cpu
#SBATCH --open-mode=truncate
#SBATCH -o log.fv3_grid_driver
#SBATCH -e log.fv3_grid_driver
#SBATCH --nodes=2 --ntasks-per-node=15
#SBATCH -q debug
#SBATCH -t 00:30:00

#-----------------------------------------------------------------------
# Driver script to create a cubic-sphere based model grid on Hercules.
#
# Produces the following files (netcdf, each tile in separate file):
# 1) 'mosaic' and 'grid' files containing lat/lon and other
# records that describe the model grid.
# 2) 'oro' files containing land mask, terrain and gravity
# wave drag fields.
# 3) Surface climo fields, such as soil type, vegetation
# greenness and albedo.
#
# Note: The sfc_climo_gen program only runs with an
# mpi task count that is a multiple of six. This is
# an ESMF library requirement. Large grids may require
# tasks spread across multiple nodes. The orography code
# benefits from threads.
#
# To run, do the following:
#
# 1) Set "C" resolution, "res" - Example: res=96.
# 2) Set grid type ("gtype"). Valid choices are
# "uniform" - global uniform grid
# "stretch" - global stretched grid
# "nest" - global stretched grid with nest
# "regional_gfdl" - stand-alone gfdl regional grid
# "regional_esg" - stand-alone extended Schmidt gnomonic
# (esg) regional grid
# 3) For "uniform" and "regional_gfdl" grids - to include lake
# fraction and depth, set "add_lake" to true, and the
# "lake_cutoff" value.
# 4) For "stretch" and "nest" grids, set the stretching factor -
# "stretch_fac", and center lat/lon of highest resolution
# tile - "target_lat" and "target_lon".
# 5) For "nest" grids, set the refinement ratio - "refine_ratio",
# the starting/ending i/j index location within the parent
# tile - "istart_nest", "jstart_nest", "iend_nest", "jend_nest"
# 6) For "regional_gfdl" grids, set the "halo". Default is three
# rows/columns.
# 7) For "regional_esg" grids, set center lat/lon of grid,
# - "target_lat/lon" - the i/j dimensions - "i/jdim", the
# x/y grid spacing - "delx/y", and halo.
# 8) Set working directory - TEMP_DIR - and path to the repository
# clone - home_dir.
# 9) To use the GSL orographic drag suite, set 'make_gsl_orog' to true.
# 10) Set 'soil_veg_src' and 'veg_type_src' to choose the
# soil type and vegetation type data.
# 11) Submit script: "sbatch $script".
# 12) All files will be placed in "out_dir".
#
#-----------------------------------------------------------------------

set -x

source ../sorc/machine-setup.sh > /dev/null 2>&1
module use ../modulefiles
module load build.$target.intel
module list

#-----------------------------------------------------------------------
# Set grid specs here.
#-----------------------------------------------------------------------

export gtype=uniform # 'uniform', 'stretch', 'nest',
# 'regional_gfdl', 'regional_esg'

export make_gsl_orog=false # When 'true' will output 'oro' files for
# the GSL orographic drag suite.

export vegsoilt_frac='.false.' # When true, outputs percent of each
# soil and veg type category and a
# dominant category. When false, only
# outputs the dominant category. A
# Fortran logical, so include the dots.

export veg_type_src="viirs.v3.igbp.30s" # Vegetation type data.
# For viirs-based vegetation type data, set to:
# 1) "viirs.v3.igbp.30s" for global 30s data
# For the modis-based data, set to:
# 1) "modis.igbp.0.05" for global 0.05-deg data
# 2) "modis.igbp.0.03" for global 0.03-deg data
# 3) "modis.igbp.conus.30s" for CONUS 30s data
# 4) "modis.igbp.nh.30s" for N Hemis 30s data
# 5) "modis.igbp.30s" for global 30s data

export soil_type_src="bnu.v3.30s" # Soil type data.
# For Beijing Normal Univ. data, set to:
# 1) "bnu.v3.30s" for global 30s data.
# For STATSGO soil type data, set to:
# 1) "statsgo.0.05" for global 0.05-deg data
# 2) "statsgo.0.03" for global 0.03-deg data
# 3) "statsgo.conus.30s" for CONUS 30s data
# 4) "statsgo.nh.30s" for NH 30s data
# 5) "statsgo.30s" for global 30s data

# choose dataset sources for lakefrac & lakedepth so that lake_data_srce=LakeFrac_LakeDepth;
# available options are 'MODISP_GLDBV3', 'MODISP_GLOBATHY', 'VIIRS_GLDBV3', 'VIIRS_GLOBATHY' & 'GLDBV3'
export lake_data_srce=MODISP_GLDBV3

if [ $gtype = uniform ]; then
export res=96
export add_lake=true # Add lake frac and depth to orography data.
export lake_cutoff=0.50 # return 0 if lake_frac < lake_cutoff & add_lake=T
export binary_lake=1 # return 1 if lake_frac >= lake_cutoff & add_lake=T
export ocn=${ocn:-"025"} # use one of "025", "050", "100", "500". Cannot be empty
elif [ $gtype = stretch ]; then
export res=96
export stretch_fac=1.5 # Stretching factor for the grid
export target_lon=-97.5 # Center longitude of the highest resolution tile
export target_lat=35.5 # Center latitude of the highest resolution tile
elif [ $gtype = nest ] || [ $gtype = regional_gfdl ]; then
export add_lake=false # Add lake frac and depth to orography data.
export lake_cutoff=0.20 # lake frac < lake_cutoff ignored when add_lake=T
export res=768
export stretch_fac=1.5 # Stretching factor for the grid
export target_lon=-97.5 # Center longitude of the highest resolution tile
export target_lat=38.5 # Center latitude of the highest resolution tile
export refine_ratio=3 # The refinement ratio
export istart_nest=123 # Starting i-direction index of nest grid in parent tile supergrid
export jstart_nest=331 # Starting j-direction index of nest grid in parent tile supergrid
export iend_nest=1402 # Ending i-direction index of nest grid in parent tile supergrid
export jend_nest=1194 # Ending j-direction index of nest grid in parent tile supergrid
export halo=3 # Lateral boundary halo
elif [ $gtype = regional_esg ] ; then
export res=-999 # equivalent resolution is computed
export target_lon=-97.5 # Center longitude of grid
export target_lat=35.5 # Center latitude of grid
export idim=301 # Dimension of grid in 'i' direction
export jdim=200 # Dimension of grid in 'j' direction
export delx=0.0585 # Grid spacing (in degrees) in the 'i' direction
# on the SUPERGRID (which has twice the resolution of
# the model grid). The physical grid spacing in the 'i'
# direction is related to delx as follows:
# distance = 2*delx*(circumf_Earth/360 deg)
export dely=0.0585 # Grid spacing (in degrees) in the 'j' direction.
export halo=3 # number of row/cols for halo
fi

#-----------------------------------------------------------------------
# Check paths.
# home_dir - location of repository.
# TEMP_DIR - working directory.
# out_dir - where files will be placed upon completion.
#-----------------------------------------------------------------------

export home_dir=$SLURM_SUBMIT_DIR/..
export TEMP_DIR=/work/noaa/stmp/$LOGNAME/fv3_grid.$gtype
export out_dir=/work/noaa/stmp/$LOGNAME/my_grids

#-----------------------------------------------------------------------
# Should not need to change anything below here.
#-----------------------------------------------------------------------

export APRUN=time
export APRUN_SFC=srun
export OMP_NUM_THREADS=24
export OMP_STACKSIZE=2048m

ulimit -a
ulimit -s unlimited

#-----------------------------------------------------------------------
# Start script.
#-----------------------------------------------------------------------

$home_dir/ush/fv3gfs_driver_grid.sh

exit
2 changes: 1 addition & 1 deletion driver_scripts/driver_grid.jet.sh
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ if [ $gtype = uniform ]; then
export add_lake=true # Add lake frac and depth to orography data.
export lake_cutoff=0.50 # return 0 if lake_frac < lake_cutoff & add_lake=T
export binary_lake=1 # return 1 if lake_frac >= lake_cutoff & add_lake=T
export ocn=${ocn:-"025"} # use one of "025", "050", "100", "500". Cannot be empty
elif [ $gtype = stretch ]; then
export res=96
export stretch_fac=1.5 # Stretching factor for the grid
Expand Down Expand Up @@ -156,7 +157,6 @@ fi
export home_dir=$SLURM_SUBMIT_DIR/..
export TEMP_DIR=/lfs4/HFIP/emcda/$LOGNAME/stmp/fv3_grid.$gtype
export out_dir=/lfs4/HFIP/emcda/$LOGNAME/stmp/my_grids

#-----------------------------------------------------------------------
# Should not need to change anything below here.
#-----------------------------------------------------------------------
Expand Down
1 change: 1 addition & 0 deletions driver_scripts/driver_grid.orion.sh
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ if [ $gtype = uniform ]; then
export add_lake=true # Add lake frac and depth to orography data.
export lake_cutoff=0.50 # return 0 if lake_frac < lake_cutoff & add_lake=T
export binary_lake=1 # return 1 if lake_frac >= lake_cutoff & add_lake=T
export ocn=${ocn:-"025"} # use one of "025", "050", "100", "500". Cannot be empty
elif [ $gtype = stretch ]; then
export res=96
export stretch_fac=1.5 # Stretching factor for the grid
Expand Down
Loading

0 comments on commit ee79c49

Please sign in to comment.