feature/gfsv16_nco

Merge branch 'develop' into feature/gfsv16_nco

Issue ufs-community#138

driver_scripts/driver_grid.cray.sh

@@ -49,7 +49,7 @@
 #   7) For "regional_esg" grids, set center lat/lon of grid,
 #      - "target_lat/lon" - the i/j dimensions - "i/jdim", the
 #      x/y grid spacing - "delx/y", and halo.
-#   8) Set working directory - TMPDIR - and path to the repository
+#   8) Set working directory - TEMP_DIR - and path to the repository
 #      clone - home_dir.
 #   9) Submit script: "cat $script | bsub".
 #  10) All files will be placed in "out_dir".
@@ -106,12 +106,12 @@ fi
 #-----------------------------------------------------------------------
 # Check paths.
 #   home_dir - location of repository.
-#   TMPDIR   - working directory.
+#   TEMP_DIR - working directory.
 #   out_dir  - where files will be placed upon completion.
 #-----------------------------------------------------------------------
 
 export home_dir=$LS_SUBCWD/..
-export TMPDIR=/gpfs/hps3/stmp/$LOGNAME/fv3_grid.$gtype
+export TEMP_DIR=/gpfs/hps3/stmp/$LOGNAME/fv3_grid.$gtype
 export out_dir=/gpfs/hps3/stmp/$LOGNAME/my_grids
 
 #-----------------------------------------------------------------------

driver_scripts/driver_grid.dell.sh

@@ -51,7 +51,7 @@
 #   7) For "regional_esg" grids, set center lat/lon of grid,
 #      - "target_lat/lon" - the i/j dimensions - "i/jdim", the 
 #      x/y grid spacing - "delx/y", and halo.
-#   8) Set working directory - TMPDIR - and path to the repository
+#   8) Set working directory - TEMP_DIR - and path to the repository
 #      clone - home_dir.
 #   9) Submit script: "cat $script | bsub".
 #  10) All files will be placed in "out_dir".
@@ -108,12 +108,12 @@ fi
 #-----------------------------------------------------------------------
 # Check paths.
 #   home_dir - location of repository.
-#   TMPDIR   - working directory.
+#   TEMP_DIR   - working directory.
 #   out_dir  - where files will be placed upon completion.
 #-----------------------------------------------------------------------
 
 export home_dir=$LS_SUBCWD/..
-export TMPDIR=/gpfs/dell1/stmp/$LOGNAME/fv3_grid.$gtype
+export TEMP_DIR=/gpfs/dell1/stmp/$LOGNAME/fv3_grid.$gtype
 export out_dir=/gpfs/dell1/stmp/$LOGNAME/my_grids
 
 #-----------------------------------------------------------------------

driver_scripts/driver_grid.hera.sh

@@ -49,7 +49,7 @@
 #   7) For "regional_esg" grids, set center lat/lon of grid,
 #      - "target_lat/lon" - the i/j dimensions - "i/jdim", the
 #      x/y grid spacing - "delx/y", and halo.
-#   8) Set working directory - TMPDIR - and path to the repository
+#   8) Set working directory - TEMP_DIR - and path to the repository
 #      clone - home_dir.
 #   9) Submit script: "sbatch $script".
 #  10) All files will be placed in "out_dir".
@@ -108,12 +108,12 @@ fi
 #-----------------------------------------------------------------------
 # Check paths.  
 #   home_dir - location of repository.
-#   TMPDIR   - working directory.
+#   TEMP_DIR   - working directory.
 #   out_dir  - where files will be placed upon completion. 
 #-----------------------------------------------------------------------
 
 export home_dir=$SLURM_SUBMIT_DIR/..
-export TMPDIR=/scratch2/NCEPDEV/stmp1/$LOGNAME/fv3_grid.$gtype
+export TEMP_DIR=/scratch2/NCEPDEV/stmp1/$LOGNAME/fv3_grid.$gtype
 export out_dir=/scratch2/NCEPDEV/stmp1/$LOGNAME/my_grids
 
 #-----------------------------------------------------------------------

driver_scripts/driver_grid.jet.sh

@@ -50,7 +50,7 @@
 #   7) For "regional_esg" grids, set center lat/lon of grid,
 #      - "target_lat/lon" - the i/j dimensions - "i/jdim", the
 #      x/y grid spacing - "delx/y", and halo.
-#   8) Set working directory - TMPDIR - and path to the repository
+#   8) Set working directory - TEMP_DIR - and path to the repository
 #      clone - home_dir.
 #   9) Submit script: "sbatch $script".
 #  10) All files will be placed in "out_dir".
@@ -109,12 +109,12 @@ fi
 #-----------------------------------------------------------------------
 # Check paths.  
 #   home_dir - location of repository.
-#   TMPDIR   - working directory.
+#   TEMP_DIR   - working directory.
 #   out_dir  - where files will be placed upon completion. 
 #-----------------------------------------------------------------------
 
 export home_dir=$SLURM_SUBMIT_DIR/..
-export TMPDIR=/lfs4/HFIP/emcda/$LOGNAME/stmp/fv3_grid.$gtype
+export TEMP_DIR=/lfs4/HFIP/emcda/$LOGNAME/stmp/fv3_grid.$gtype
 export out_dir=/lfs4/HFIP/emcda/$LOGNAME/stmp/my_grids
 
 #-----------------------------------------------------------------------

driver_scripts/driver_grid.orion.sh

@@ -49,7 +49,7 @@
 #   7) For "regional_esg" grids, set center lat/lon of grid,
 #      - "target_lat/lon" - the i/j dimensions - "i/jdim", the
 #      x/y grid spacing - "delx/y", and halo.
-#   8) Set working directory - TMPDIR - and path to the repository
+#   8) Set working directory - TEMP_DIR - and path to the repository
 #      clone - home_dir.
 #   9) Submit script: "sbatch $script".
 #  10) All files will be placed in "out_dir".
@@ -108,12 +108,12 @@ fi
 #-----------------------------------------------------------------------
 # Check paths.  
 #   home_dir - location of repository.
-#   TMPDIR   - working directory.
+#   TEMP_DIR   - working directory.
 #   out_dir  - where files will be placed upon completion. 
 #-----------------------------------------------------------------------
 
 export home_dir=$SLURM_SUBMIT_DIR/..
-export TMPDIR=/work/noaa/stmp/$LOGNAME/fv3_grid.$gtype
+export TEMP_DIR=/work/noaa/stmp/$LOGNAME/fv3_grid.$gtype
 export out_dir=/work/noaa/stmp/$LOGNAME/my_grids
 
 #-----------------------------------------------------------------------

modulefiles/build.hera

@@ -24,6 +24,6 @@ module load landsfcutil/2.4.1
 module load wgrib2/2.0.8
 
 module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles
-module load hdf5_parallel/1.10.6
-module load netcdf_parallel/4.7.4
+module load hdf5_parallel/1.10.6.release
+module load netcdf_parallel/4.7.4.release
 module load esmf/8.0.0_ParallelNetCDF

reg_tests/chgres_cube/driver.hera.sh

@@ -123,7 +123,7 @@ TEST8=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_C
 
 sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J chgres_summary -o $LOG_FILE -e $LOG_FILE \
        --open-mode=append -q $QUEUE -d afterok:$TEST8 << EOF
-#!/bin/sh
+#!/bin/bash
 grep -a '<<<' $LOG_FILE  > $SUM_FILE
 EOF
 

reg_tests/chgres_cube/driver.jet.sh

@@ -123,7 +123,7 @@ TEST8=$(sbatch --parsable --partition=xjet --nodes=1 --ntasks-per-node=6 -t 0:05
 
 sbatch --partition=xjet --nodes=1  -t 0:01:00 -A $PROJECT_CODE -J chgres_summary -o $LOG_FILE -e $LOG_FILE \
        --open-mode=append -q $QUEUE -d afterok:$TEST8 << EOF
-#!/bin/sh
+#!/bin/bash
 grep -a '<<<' $LOG_FILE  > $SUM_FILE
 EOF
 

reg_tests/chgres_cube/driver.orion.sh

@@ -126,7 +126,7 @@ TEST8=$(sbatch --parsable --ntasks-per-node=6 --nodes=2 -t 0:10:00 -A $PROJECT_C
 
 sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J chgres_summary -o $LOG_FILE -e $LOG_FILE \
        --open-mode=append -q $QUEUE -d afterok:$TEST8 << EOF
-#!/bin/sh
+#!/bin/bash
 grep -a '<<<' $LOG_FILE  > $SUM_FILE
 EOF
 

reg_tests/grid_gen/c96.uniform.sh

@@ -8,7 +8,7 @@
 
 set -x
 
-export TMPDIR=${WORK_DIR}/c96.uniform.work
+export TEMP_DIR=${WORK_DIR}/c96.uniform.work
 export out_dir=${WORK_DIR}/c96.uniform
 
 export res=96

reg_tests/grid_gen/driver.hera.sh

@@ -71,6 +71,6 @@ TEST2=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_
 
 sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J grid_summary -o $LOG_FILE -e $LOG_FILE \
        --open-mode=append -q $QUEUE -d afterok:$TEST2 << EOF
-#!/bin/sh
+#!/bin/bash
 grep -a '<<<' $LOG_FILE  > $SUM_FILE
 EOF

reg_tests/grid_gen/driver.jet.sh

@@ -71,6 +71,6 @@ TEST2=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_
 
 sbatch --partition=xjet --nodes=1  -t 0:01:00 -A $PROJECT_CODE -J grid_summary -o $LOG_FILE -e $LOG_FILE \
        --open-mode=append -q $QUEUE -d afterok:$TEST2 << EOF
-#!/bin/sh
+#!/bin/bash
 grep -a '<<<' $LOG_FILE  > $SUM_FILE
 EOF

reg_tests/grid_gen/driver.orion.sh

@@ -67,6 +67,6 @@ TEST2=$(sbatch --parsable --ntasks-per-node=24 --nodes=1 -t 0:10:00 -A $PROJECT_
 
 sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J grid_summary -o $LOG_FILE -e $LOG_FILE \
        --open-mode=append -q $QUEUE -d afterok:$TEST2 << EOF
-#!/bin/sh
+#!/bin/bash
 grep -a '<<<' $LOG_FILE  > $SUM_FILE
 EOF

reg_tests/grid_gen/gfdl.regional.sh

@@ -8,7 +8,7 @@
 
 set -x
 
-export TMPDIR=${WORK_DIR}/gfdl.regional.work
+export TEMP_DIR=${WORK_DIR}/gfdl.regional.work
 export out_dir=${WORK_DIR}/gfdl.regional
 
 export res=96                # global resolution in which grid is embedded.

scripts/exemcsfc_global_sfc_prep.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 ####  UNIX Script Documentation Block ###################################
 #                      .                                             .

sorc/CMakeLists.txt

@@ -17,3 +17,5 @@ add_subdirectory(grid_tools.fd)
 add_subdirectory(chgres_cube.fd)
 add_subdirectory(orog_mask_tools.fd)
 add_subdirectory(sfc_climo_gen.fd)
+add_subdirectory(vcoord_gen.fd)
+add_subdirectory(fvcom_tools.fd)

sorc/chgres_cube.fd/atmosphere.F90

@@ -144,9 +144,9 @@ module atmosphere
 
  subroutine atmosphere_driver(localpet)
 
- implicit none
+ use mpi
 
- include 'mpif.h'
+ implicit none
 
  integer, intent(in)                :: localpet
 
@@ -1407,9 +1407,9 @@ SUBROUTINE VINTG
 !   LANGUAGE: FORTRAN
 !
 !
- IMPLICIT NONE
+ use mpi
 
- include 'mpif.h'
+ IMPLICIT NONE
 
  REAL(ESMF_KIND_R8), PARAMETER   :: DLTDZ=-6.5E-3*287.05/9.80665
  REAL(ESMF_KIND_R8), PARAMETER   :: DLPVDRT=-2.5E6/461.50

sorc/chgres_cube.fd/chgres.F90

@@ -9,6 +9,7 @@ program chgres
 !
 !-------------------------------------------------------------------------
 
+ use mpi
  use esmf
 
  use atmosphere, only          : atmosphere_driver
@@ -34,8 +35,6 @@ program chgres
 ! Initialize mpi and esmf environment.
 !-------------------------------------------------------------------------
 
- include 'mpif.h'
-
  call mpi_init(ierr)
 
  print*,"- INITIALIZE ESMF"

sorc/chgres_cube.fd/input_data.F90

@@ -1728,9 +1728,9 @@ end subroutine read_input_atm_restart_file
 
  subroutine read_input_atm_gaussian_netcdf_file(localpet)
 
- implicit none
+ use mpi
 
- include 'mpif.h'
+ implicit none
 
  integer, intent(in)               :: localpet
 
@@ -2111,9 +2111,9 @@ end subroutine read_input_atm_gaussian_netcdf_file
 
  subroutine read_input_atm_tiled_history_file(localpet)
 
- implicit none
+ use mpi
 
- include 'mpif.h'
+ implicit none
 
  integer, intent(in)             :: localpet
 

sorc/chgres_cube.fd/model_grid.F90

@@ -602,6 +602,8 @@ end subroutine define_input_grid_mosaic
 
  subroutine define_input_grid_gfs_grib2(localpet, npets)
 
+ use mpi
+
  use wgrib2api
 
  use program_setup, only       : data_dir_input_grid, &
@@ -613,7 +615,7 @@ subroutine define_input_grid_gfs_grib2(localpet, npets)
 
  character(len=250)               :: the_file
 
- integer                          :: i, j, rc, clb(2), cub(2)
+ integer                          :: i, j, rc, clb(2), cub(2), ierr
 
  real(esmf_kind_r8), allocatable  :: latitude(:,:)
  real(esmf_kind_r8), allocatable  :: longitude(:,:)
@@ -631,10 +633,15 @@ subroutine define_input_grid_gfs_grib2(localpet, npets)
  num_tiles_input_grid = 1
 
  the_file = trim(data_dir_input_grid) // "/" // grib2_file_input_grid
- print*,'- OPEN AND INVENTORY GRIB2 FILE: ',trim(the_file)
- rc=grb2_mk_inv(the_file,inv_file)
- if (rc /=0) call error_handler("OPENING GRIB2 FILE",rc)
+ if(localpet == 0) then
+   print*,'- OPEN AND INVENTORY GRIB2 FILE: ',trim(the_file)
+   rc=grb2_mk_inv(the_file,inv_file)
+   if (rc /=0) call error_handler("OPENING GRIB2 FILE",rc)
+ endif
 
+! Wait for localpet 0 to create inventory.
+ call mpi_barrier(mpi_comm_world, ierr)
+
  rc = grb2_inq(the_file,inv_file,':PRES:',':surface:',nx=i_input, ny=j_input, &
     lat=lat4, lon=lon4)
  if (rc /= 1) call error_handler("READING GRIB2 FILE", rc)

sorc/chgres_cube.fd/search_util.f90

@@ -34,12 +34,11 @@ subroutine search (field, mask, idim, jdim, tile, field_num, latitude)
 ! future upgrade.
 !-----------------------------------------------------------------------
 
+ use mpi
  use esmf
 
  implicit none
 
- include 'mpif.h'
-
  integer, intent(in)               :: idim, jdim, tile, field_num
  integer(esmf_kind_i8), intent(in) :: mask(idim,jdim)
 

sorc/chgres_cube.fd/surface.F90

@@ -242,6 +242,7 @@ end subroutine surface_driver
 
  subroutine interp(localpet)
 
+ use mpi
  use esmf
 
  use input_data, only                : canopy_mc_input_grid,  &
@@ -301,8 +302,6 @@ subroutine interp(localpet)
 
  implicit none
 
- include 'mpif.h'
-
  integer, intent(in)                :: localpet
 
  integer                            :: l(1), u(1)

sorc/chgres_cube.fd/utils.f90

@@ -1,8 +1,8 @@
  subroutine error_handler(string, rc)
 
- implicit none
+ use mpi
 
- include 'mpif.h'
+ implicit none
 
  character(len=*), intent(in)    :: string
 
@@ -18,6 +18,7 @@ end subroutine error_handler
 
  subroutine netcdf_err( err, string )
 
+ use mpi
  use netcdf
 
  implicit none
@@ -26,8 +27,6 @@ subroutine netcdf_err( err, string )
  character(len=256) :: errmsg
  integer :: iret
 
- include "mpif.h"
-
  if( err.EQ.NF90_NOERR )return
  errmsg = NF90_STRERROR(err)
  print*,''

sorc/fre-nctools.fd/tools/make_solo_mosaic/make_solo_mosaic.c

@@ -78,7 +78,7 @@ main (int argc, char *argv[])
   int contact_tile1_jstart[MAXCONTACT], contact_tile1_jend[MAXCONTACT];
   int contact_tile2_istart[MAXCONTACT], contact_tile2_iend[MAXCONTACT];
   int contact_tile2_jstart[MAXCONTACT], contact_tile2_jend[MAXCONTACT];
-  char mosaic_name[128] = "solo_mosaic";
+  char mosaic_name[STRING] = "solo_mosaic";
   char grid_descriptor[128] = "";
   int c, i, n, m, l, errflg;