Run grid_gen consistency tests in parallel. (#697)

Fixes #696.
ufs-community · Oct 14, 2022 · bc65edc · bc65edc
1 parent 3c74c99
commit bc65edc
Show file tree

Hide file tree

Showing 4 changed files with 82 additions and 63 deletions.
diff --git a/reg_tests/grid_gen/driver.hera.sh b/reg_tests/grid_gen/driver.hera.sh
@@ -8,16 +8,16 @@
 # as appropriate.  To see which projects you are authorized to use,
 # type "account_params".
 #
-# Invoke the script with no arguments.  A series of daily-
-# chained jobs will be submitted.  To check the queue, type:
+# Invoke the script with no arguments.  A set of tests will
+# be submitted to run in parallel.  To check the queue, type:
 # "squeue -u USERNAME".
 #
-# Log output from the suite will be in LOG_FILE.  Once the suite
-# has completed, a summary is placed in SUM_FILE.
+# Log output from each tests will be in its own LOG_FILE.  Once 
+# the tests have completed, a summary is placed in SUM_FILE.
 #
 # A test fails when its output does not match the baseline files as
 # determined by the "nccmp" utility.  The baseline files are stored in
-# HOMEreg
+# HOMEreg.
 #
 #-----------------------------------------------------------------------------
 
@@ -65,43 +65,48 @@ export OMP_NUM_THREADS=24
 # C96 uniform grid
 #-----------------------------------------------------------------------------
 
-TEST1=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.uniform \
-      -o $LOG_FILE -e $LOG_FILE ./c96.uniform.sh)
+LOG_FILE1=${LOG_FILE}01
+TEST1=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J c96.uniform \
+      -o $LOG_FILE1 -e $LOG_FILE1 ./c96.uniform.sh)
 
 #-----------------------------------------------------------------------------
 # C96 uniform grid using viirs vegetation data.
 #-----------------------------------------------------------------------------
 
-TEST2=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.viirs.vegt \
-      -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST1 ./c96.viirs.vegt.sh)
+LOG_FILE2=${LOG_FILE}02
+TEST2=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J c96.viirs.vegt \
+      -o $LOG_FILE2 -e $LOG_FILE2 ./c96.viirs.vegt.sh)
 
 #-----------------------------------------------------------------------------
 # gfdl regional grid
 #-----------------------------------------------------------------------------
 
-TEST3=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J gfdl.regional \
-      -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST2 ./gfdl.regional.sh)
+LOG_FILE3=${LOG_FILE}03
+TEST3=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:07:00 -A $PROJECT_CODE -q $QUEUE -J gfdl.regional \
+      -o $LOG_FILE3 -e $LOG_FILE3 ./gfdl.regional.sh)
 
 #-----------------------------------------------------------------------------
 # esg regional grid
 #-----------------------------------------------------------------------------
 
-TEST4=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J esg.regional \
-      -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST3 ./esg.regional.sh)
+LOG_FILE4=${LOG_FILE}04
+TEST4=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:07:00 -A $PROJECT_CODE -q $QUEUE -J esg.regional \
+      -o $LOG_FILE4 -e $LOG_FILE4 ./esg.regional.sh)
 
 #-----------------------------------------------------------------------------
 # Regional GSL gravity wave drag test.
 #-----------------------------------------------------------------------------
 
-TEST5=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J reg.gsl.gwd \
-      -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST4 ./regional.gsl.gwd.sh)
+LOG_FILE5=${LOG_FILE}05
+TEST5=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:07:00 -A $PROJECT_CODE -q $QUEUE -J reg.gsl.gwd \
+      -o $LOG_FILE5 -e $LOG_FILE5 ./regional.gsl.gwd.sh)
 
 #-----------------------------------------------------------------------------
 # Create summary log.
 #-----------------------------------------------------------------------------
 
 sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J grid_summary -o $LOG_FILE -e $LOG_FILE \
-       --open-mode=append -q $QUEUE -d afterok:$TEST5 << EOF
+       --open-mode=append -q $QUEUE -d afterok:$TEST1:$TEST2:$TEST3:$TEST4:$TEST5 << EOF
 #!/bin/bash
-grep -a '<<<' $LOG_FILE  > $SUM_FILE
+grep -a '<<<' ${LOG_FILE}*  > $SUM_FILE
 EOF
diff --git a/reg_tests/grid_gen/driver.jet.sh b/reg_tests/grid_gen/driver.jet.sh
@@ -8,12 +8,12 @@
 # as appropriate.  To see which projects you are authorized to use,
 # type "account_params".
 #
-# Invoke the script with no arguments.  A series of daily-
-# chained jobs will be submitted.  To check the queue, type:
+# Invoke the script with no arguments.  Several tests will
+# be submitted to run in parallel. To check the queue, type:
 # "squeue -u USERNAME".
 #
-# Log output from the suite will be in LOG_FILE.  Once the suite
-# has completed, a summary is placed in SUM_FILE.
+# Log output from each test will be placed in its own LOG_FILE.
+# Once the suite has completed, a summary is placed in SUM_FILE.
 #
 # A test fails when its output does not match the baseline files as
 # determined by the "nccmp" utility.  The baseline files are stored in
@@ -28,8 +28,8 @@ module list
 
 set -x
 
-QUEUE="${QUEUE:-windfall}"
-PROJECT_CODE="${PROJECT_CODE:-emcda}"
+QUEUE="${QUEUE:-batch}"
+PROJECT_CODE="${PROJECT_CODE:-hfv3gfs}"
 export WORK_DIR="${WORK_DIR:-/lfs4/HFIP/emcda/$LOGNAME/stmp}"
 export WORK_DIR="${WORK_DIR}/reg-tests/grid-gen"
 
@@ -63,43 +63,48 @@ export OMP_NUM_THREADS=24
 # C96 uniform grid
 #-----------------------------------------------------------------------------
 
-TEST1=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.uniform \
-      --partition=xjet -o $LOG_FILE -e $LOG_FILE ./c96.uniform.sh)
+LOG_FILE1=${LOG_FILE}01
+TEST1=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J c96.uniform \
+      --partition=xjet -o $LOG_FILE1 -e $LOG_FILE1 ./c96.uniform.sh)
 
 #-----------------------------------------------------------------------------
 # C96 uniform grid using viirs vegetation type data.
 #-----------------------------------------------------------------------------
 
-TEST2=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.viirs.vegt \
-      --partition=xjet -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST1 ./c96.viirs.vegt.sh)
+LOG_FILE2=${LOG_FILE}02
+TEST2=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J c96.viirs.vegt \
+      --partition=xjet -o $LOG_FILE2 -e $LOG_FILE2 ./c96.viirs.vegt.sh)
 
 #-----------------------------------------------------------------------------
 # gfdl regional grid
 #-----------------------------------------------------------------------------
 
-TEST3=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J gfdl.regional \
-      --partition=xjet -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST2 ./gfdl.regional.sh)
+LOG_FILE3=${LOG_FILE}03
+TEST3=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:07:00 -A $PROJECT_CODE -q $QUEUE -J gfdl.regional \
+      --partition=xjet -o $LOG_FILE3 -e $LOG_FILE3 ./gfdl.regional.sh)
 
 #-----------------------------------------------------------------------------
 # ESG regional grid
 #-----------------------------------------------------------------------------
 
-TEST4=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J esg.regional \
-      --partition=xjet -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST3 ./esg.regional.sh)
+LOG_FILE4=${LOG_FILE}04
+TEST4=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:07:00 -A $PROJECT_CODE -q $QUEUE -J esg.regional \
+      --partition=xjet -o $LOG_FILE4 -e $LOG_FILE4 ./esg.regional.sh)
 
 #-----------------------------------------------------------------------------
 # Regional GSL gravity wave drag.
 #-----------------------------------------------------------------------------
 
-TEST5=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J reg.gsl.gwd \
-      --partition=xjet -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST4 ./regional.gsl.gwd.sh)
+LOG_FILE5=${LOG_FILE}05
+TEST5=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:07:00 -A $PROJECT_CODE -q $QUEUE -J reg.gsl.gwd \
+      --partition=xjet -o $LOG_FILE5 -e $LOG_FILE5 ./regional.gsl.gwd.sh)
 
 #-----------------------------------------------------------------------------
 # Create summary log.
 #-----------------------------------------------------------------------------
 
 sbatch --partition=xjet --nodes=1  -t 0:01:00 -A $PROJECT_CODE -J grid_summary -o $LOG_FILE -e $LOG_FILE \
-       --open-mode=append -q $QUEUE -d afterok:$TEST5 << EOF
+       -q $QUEUE -d afterok:$TEST1:$TEST2:$TEST3:$TEST4:$TEST5 << EOF
 #!/bin/bash
-grep -a '<<<' $LOG_FILE  > $SUM_FILE
+grep -a '<<<' ${LOG_FILE}*  > $SUM_FILE
 EOF
diff --git a/reg_tests/grid_gen/driver.orion.sh b/reg_tests/grid_gen/driver.orion.sh
@@ -6,14 +6,14 @@
 #
 # Set WORK_DIR to your working directory. Set the PROJECT_CODE and QUEUE
 # as appropriate.  To see which projects you are authorized to use,
-# type "saccount_params".
+# type "saccount_params" (after a 'module load contrib noaatools').
 #
-# Invoke the script with no arguments.  A series of daily-
-# chained jobs will be submitted.  To check the queue, type:
+# Invoke the script with no arguments.  A set of tests will
+# be submitted to run in parallel. To check the queue, type:
 # "squeue -u $LOGNAME".
 #
-# Log output from the suite will be in LOG_FILE.  Once the suite
-# has completed, a summary is placed in SUM_FILE.
+# Log output from each test will be placed in its own LOG_FILE.  
+# Once the suite has completed, a summary is placed in SUM_FILE.
 #
 # A test fails when its output does not match the baseline files as
 # determined by the "nccmp" utility.  The baseline files are stored in
@@ -61,43 +61,48 @@ rm -fr $WORK_DIR
 # C96 uniform grid
 #-----------------------------------------------------------------------------
 
+LOG_FILE1=${LOG_FILE}01
 TEST1=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.uniform \
-      -o $LOG_FILE -e $LOG_FILE ./c96.uniform.sh)
+      -o $LOG_FILE1 -e $LOG_FILE1 ./c96.uniform.sh)
 
 #-----------------------------------------------------------------------------
 # C96 uniform grid using viirs vegetation type data.
 #-----------------------------------------------------------------------------
 
+LOG_FILE2=${LOG_FILE}02
 TEST2=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:15:00 -A $PROJECT_CODE -q $QUEUE -J c96.viirs.vegt \
-      --open-mode=append -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST1 ./c96.viirs.vegt.sh)
+      -o $LOG_FILE2 -e $LOG_FILE2 ./c96.viirs.vegt.sh)
 
 #-----------------------------------------------------------------------------
 # GFDL regional grid
 #-----------------------------------------------------------------------------
 
+LOG_FILE3=${LOG_FILE}03
 TEST3=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J gfdl.regional \
-      --open-mode=append -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST2 ./gfdl.regional.sh)
+      -o $LOG_FILE3 -e $LOG_FILE3 ./gfdl.regional.sh)
 
 #-----------------------------------------------------------------------------
 # ESG regional grid
 #-----------------------------------------------------------------------------
 
+LOG_FILE4=${LOG_FILE}04
 TEST4=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J esg.regional \
-      --open-mode=append -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST3 ./esg.regional.sh)
+      -o $LOG_FILE4 -e $LOG_FILE4 ./esg.regional.sh)
 
 #-----------------------------------------------------------------------------
 # Regional grid with GSL gravity wave drag fields.
 #-----------------------------------------------------------------------------
 
+LOG_FILE5=${LOG_FILE}05
 TEST5=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_CODE -q $QUEUE -J reg.gsl.gwd \
-      --open-mode=append -o $LOG_FILE -e $LOG_FILE -d afterok:$TEST4 ./regional.gsl.gwd.sh)
+      -o $LOG_FILE5 -e $LOG_FILE5 ./regional.gsl.gwd.sh)
 
 #-----------------------------------------------------------------------------
 # Create summary log.
 #-----------------------------------------------------------------------------
 
 sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J grid_summary -o $LOG_FILE -e $LOG_FILE \
-       --open-mode=append -q $QUEUE -d afterok:$TEST5 << EOF
+       -q $QUEUE -d afterok:$TEST1:$TEST2:$TEST3:$TEST4:$TEST5 << EOF
 #!/bin/bash
-grep -a '<<<' $LOG_FILE  > $SUM_FILE
+grep -a '<<<' ${LOG_FILE}*  > $SUM_FILE
 EOF
diff --git a/reg_tests/grid_gen/driver.wcoss2.sh b/reg_tests/grid_gen/driver.wcoss2.sh
@@ -7,12 +7,11 @@
 # Set WORK_DIR to your working directory. Set the PROJECT_CODE and QUEUE
 # as appropriate.
 #
-# Invoke the script with no arguments.  A series of daily-
-# chained jobs will be submitted.  To check the queue, type:
-# "qstat -u USERNAME".
+# Invoke the script with no arguments. Several tests will be started
+# in parallel.  To check the queue, type: "qstat -u USERNAME".
 #
-# Log output from the suite will be in LOG_FILE.  Once the suite
-# has completed, a summary is placed in SUM_FILE.
+# Log output from the tests will be stored in their own LOG_FILE.  
+# Once the tests have completed, a summary is placed in SUM_FILE.
 #
 # A test fails when its output does not match the baseline files as
 # determined by the "nccmp" utility.  The baseline files are stored in
@@ -67,44 +66,49 @@ rm -fr $WORK_DIR
 # C96 uniform grid
 #-----------------------------------------------------------------------------
 
-TEST1=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:10:00 \
+LOG_FILE1=${LOG_FILE}01
+TEST1=$(qsub -V -o $LOG_FILE1 -e $LOG_FILE1 -q $QUEUE -A $PROJECT_CODE -l walltime=00:10:00 \
         -N c96.uniform -l select=1:ncpus=30:mem=40GB $PWD/c96.uniform.sh)
 
 #-----------------------------------------------------------------------------
 # C96 uniform grid using viirs vegetation data.
 #-----------------------------------------------------------------------------
 
-TEST2=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:10:00 \
-        -N c96.viirs.vegt -l select=1:ncpus=30:mem=40GB -W depend=afterok:$TEST1 $PWD/c96.viirs.vegt.sh)
+LOG_FILE2=${LOG_FILE}02
+TEST2=$(qsub -V -o $LOG_FILE2 -e $LOG_FILE2 -q $QUEUE -A $PROJECT_CODE -l walltime=00:10:00 \
+        -N c96.viirs.vegt -l select=1:ncpus=30:mem=40GB $PWD/c96.viirs.vegt.sh)
 
 #-----------------------------------------------------------------------------
 # gfdl regional grid
 #-----------------------------------------------------------------------------
 
-TEST3=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:10:00 \
-        -N gfdl.regional -l select=1:ncpus=30:mem=40GB -W depend=afterok:$TEST2 $PWD/gfdl.regional.sh)
+LOG_FILE3=${LOG_FILE}03
+TEST3=$(qsub -V -o $LOG_FILE3 -e $LOG_FILE3 -q $QUEUE -A $PROJECT_CODE -l walltime=00:07:00 \
+        -N gfdl.regional -l select=1:ncpus=30:mem=40GB  $PWD/gfdl.regional.sh)
 
 #-----------------------------------------------------------------------------
 # esg regional grid
 #-----------------------------------------------------------------------------
 
-TEST4=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:10:00 \
-        -N esg.regional -l select=1:ncpus=30:mem=40GB -W depend=afterok:$TEST3 $PWD/esg.regional.sh)
+LOG_FILE4=${LOG_FILE}04
+TEST4=$(qsub -V -o $LOG_FILE4 -e $LOG_FILE4 -q $QUEUE -A $PROJECT_CODE -l walltime=00:07:00 \
+        -N esg.regional -l select=1:ncpus=30:mem=40GB $PWD/esg.regional.sh)
 
 #-----------------------------------------------------------------------------
 # Regional GSL gravity wave drag test.
 #-----------------------------------------------------------------------------
 
-TEST5=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:10:00 \
-        -N rsg.gsl.gwd -l select=1:ncpus=30:mem=40GB -W depend=afterok:$TEST4 $PWD/regional.gsl.gwd.sh)
+LOG_FILE5=${LOG_FILE}05
+TEST5=$(qsub -V -o $LOG_FILE5 -e $LOG_FILE5 -q $QUEUE -A $PROJECT_CODE -l walltime=00:07:00 \
+        -N rsg.gsl.gwd -l select=1:ncpus=30:mem=40GB $PWD/regional.gsl.gwd.sh)
 
 #-----------------------------------------------------------------------------
 # Create summary log.
 #-----------------------------------------------------------------------------
 
 qsub -V -o ${LOG_FILE} -e ${LOG_FILE} -q $QUEUE -A $PROJECT_CODE -l walltime=00:02:00 \
-        -N grid_summary -l select=1:ncpus=1:mem=100MB -W depend=afterok:$TEST5 << EOF
+        -N grid_summary -l select=1:ncpus=1:mem=100MB -W depend=afterok:$TEST1:$TEST2:$TEST3:$TEST4:$TEST5 << EOF
 #!/bin/bash
 cd ${this_dir}
-grep -a '<<<' $LOG_FILE | grep -v echo > $SUM_FILE
+grep -a '<<<' ${LOG_FILE}* | grep -v echo > $SUM_FILE
 EOF